doix-db 1.0.18 → 1.0.20

package/README.md

@@ -2,4 +2,11 @@
 ![Jest coverage](./badges/coverage-jest%20coverage.svg)
 
 # node-doix-db
-Shared database related code for doix
+`doix-db` is a plug in for [doix](https://github.com/do-/node-doix) framework implementing a common interface to relational databases. It features:
+* [DbClient](DbClient) — the database API available to `doix` [Job](https://github.com/do-/node-doix/wiki/Job)s;
+* [DbModel](DbModel) — the set of classes representing the database structure;
+* [DbQuery](DbQuery) — a `DbModel` based `SELECT` builder;
+* [DbMigrationPlan](DbMigrationPlan) — a diff/patch tool to compare existing database structure with the required `DbModel`;
+* [DbLang](DbLang) — a set of SQL generating functions for miscellaneous application tasks.
+
+* More information is available at https://github.com/do-/node-doix-db/wiki

package/index.js

@@ -1,7 +1,7 @@
 module.exports = {
 
-	DbEventLogger : require ('./lib/DbEventLogger.js'),
-	DbClient:       require ('./lib/DbClient.js'), 
+	DbCallTracker : require ('./lib/DbCallTracker.js'),
+	DbClient:       require ('./lib/DbClient.js'),
 	DbPool:         require ('./lib/DbPool.js'), 
 	DbLang:         require ('./lib/DbLang.js'),
 	DbColumn:       require ('./lib/model/DbColumn.js'),

package/lib/DbCall.js

@@ -0,0 +1,255 @@
+const EV_FINISH = 'finish'
+const EV_ERROR = 'error'
+const EV_END_STREAM = [
+	'end',
+	'close',
+]
+
+const OPT_REQUIRING_MAX_ROW = [
+	'rowMode',
+	'checkOverflow',
+	'minRows',
+]
+
+const NULL = Symbol.for ('NULL'), mayBeNull = v => v === NULL ? null : v
+
+const MD_ARRAY = 'array'
+const MD_OBJECT = 'object'
+const MD_SCALAR = 'scalar'
+
+const EventEmitter = require ('events')
+const {Transform} = require ('stream')
+
+class DbCall extends EventEmitter {
+
+	constructor (db, sql, params = [], options = {}) {
+
+		if (!db) throw Error ('DbCall: db must be defined')
+
+		super ()
+
+		this.ord = ++ db.count
+
+		if (typeof sql !== 'string') throw Error ('DbCall: sql must be a string, found: ' + sql)
+
+		if (!('maxRows' in options)) options.maxRows = 0
+
+		if (typeof options.maxRows !== 'number') throw Error ('DbCall: maxRows must be a number, found: ' + options.maxRows)
+
+		if (options.maxRows < 0) throw Error ('DbCall: maxRows cannot be negative, found: ' + options.maxRows)
+
+		if (options.maxRows > 0) {
+
+			if (!('rowMode' in options)) options.rowMode = MD_OBJECT
+
+			switch (options.rowMode) {
+
+				case MD_ARRAY:
+				case MD_OBJECT:
+				case MD_SCALAR:
+					break
+				
+				default:
+					throw Error ('DbCall: rowMode must be one of (scalar|array|object) found' + options.rowMode)
+
+			}
+
+			if (!('checkOverflow' in options)) options.checkOverflow = false
+
+			if (typeof options.checkOverflow !== 'boolean') throw Error ('DbCall: checkOverflow must be a boolean, found: ' + options.maxRows)
+
+			if (options.maxRows === Infinity) {
+
+				if ('minRows' in options) throw Error ('DbCall: minRows requires a finite maxRows')
+
+			}
+			else {
+
+				if (!('minRows' in options)) options.minRows = 0
+
+				if (typeof options.minRows !== 'number') throw Error ('DbCall: minRows must be a number, found: ' + options.minRows)
+
+				if (options.minRows < 0) throw Error ('DbCall: minRows cannot be negative, found: ' + options.minRows)
+
+				if (options.minRows > options.maxRows) throw Error ('DbCall: minRows cannot exceed maxRows=' + options.maxRows)
+
+			}
+
+		}
+		else {
+
+			for (const k of OPT_REQUIRING_MAX_ROW) if (k in options) throw Error (`DbCall: ${k} cannot be set with maxRows=${options.maxRows}`)
+
+		}
+
+		if ('notFound' in options && !(options.minRows > 0)) throw Error ('DbCall: notFound requires a positive minRows')
+
+		this.db = db
+		this.sql = sql
+		this.params = params
+		this.options = options
+
+	}
+
+	get objectMode () {
+
+		return this.options.rowMode === MD_OBJECT
+
+	}
+
+	get result () {
+
+		const {rows, options} = this, {minRows, maxRows} = options
+
+		if (maxRows === 0) return
+
+		if (minRows > rows.length) {
+
+			if ('notFound' in options) return options.notFound
+
+			throw Error (`Missing data: ${rows.length} found, ${minRows} required`)
+
+		}
+
+		return maxRows === 1 ? rows [0] : rows
+
+	}
+
+	finish () {
+
+		this.emit (EV_FINISH)
+
+		this.removeAllListeners (EV_FINISH)
+
+	}
+
+	flattenArray () {
+
+		const {rows} = this
+
+		for (let i = 0; i < rows.length; i ++) rows [i] = rows [i] [0]
+
+	}
+	
+	flattenStream () {
+
+		const {rows} = this
+		
+		const xform = new Transform ({objectMode: true,
+			transform (r, __, cb) {
+				const v = r [0]
+				cb (null, v == null ? NULL : v)
+			}
+		})
+
+		rows.once ('error', e => xform.destroy (e))
+
+		this.rows = this.rows.pipe (xform)
+
+	}
+
+	observeStream () {
+
+		const {rows} = this
+
+		rows.on (EV_ERROR, e => this.emit (EV_ERROR, e))
+
+		const finish = () => this.finish ()
+
+		for (const event of EV_END_STREAM) rows.once (event, finish)
+
+	}
+	
+	processArray () {
+
+		if (this.options.rowMode === MD_SCALAR) this.flattenArray ()
+
+		this.finish ()
+
+		return this.result
+
+	}
+
+	processStream () {
+
+		const {rowMode} = this.options
+
+		if (rowMode === MD_SCALAR) this.flattenStream ()
+
+		this.observeStream ()
+
+	}
+
+	async fetchStream () {
+
+		const a = [], {rows, options: {maxRows, checkOverflow}} = this; await new Promise ((ok, fail) => {
+
+			rows.once ('error', fail)
+
+			let stop = false; rows.on ('end', () => ok (stop = true))
+
+			rows.on ('data',
+
+				checkOverflow ? r => {
+					if (a.length === maxRows) return rows.destroy (Error ('Result set overflow, maxRows = ' + maxRows))
+					a.push (mayBeNull (r))
+				}
+
+				: r => {
+					if (stop) return
+					a.push (mayBeNull (r))
+					if (a.length === maxRows) rows.emit ('end')
+				}
+
+			)
+
+		}).finally (() => rows.destroy ())
+
+		this.rows = a
+		
+	}
+
+	async exec () {
+
+		const {db} = this
+
+		db.lang.normalizeSQL (this)
+
+		this.once ('error', () => this.finish ())
+
+		try {
+
+			const {maxRows} = this.options
+
+			this.emit ('start')
+
+			await db.exec (this) 
+			
+			this.emit ('result')
+
+			if (maxRows === 0) return
+
+			if (Array.isArray (this.rows)) return this.processArray ()
+
+			this.processStream ()
+
+			if (maxRows === Infinity) return this.result
+
+			await this.fetchStream ()
+
+			return this.result
+
+		}
+		catch (error) {
+
+			this.emit ('error', this.error = error)
+
+			throw error
+
+		}
+
+	}
+
+}
+
+module.exports = DbCall

package/lib/DbCallTracker.js

@@ -0,0 +1,70 @@
+const {LifeCycleTracker} = require ('doix')
+const stringEscape  = require ('string-escape-map')
+
+const ESC_PARAMS = new stringEscape ([
+  ['\t', '\\t'],
+  ['\n', '\\n'],
+  ['\r', '\\r'],
+  [ "'", "''"],
+])
+
+const stringifyParams = params => {
+	
+	if (!Array.isArray (params)) return JSON.stringify (params)
+	
+	if (params.length === 0) return ''
+
+	let s = '['; for (const p of params) {
+	
+		if (s.length !== 1) s += ', '
+		
+		if (typeof p === 'string') {
+		
+			s += "'"
+			s += ESC_PARAMS.escape (p)
+			s += "'"
+		
+		}
+		else {
+
+			s += p
+
+		}
+	
+	}
+	
+	return s + ']'
+
+}
+
+class DbCallTracker extends LifeCycleTracker {
+
+	constructor (call) {
+
+		const {db} = call
+
+		super (call, db.pool.logger)
+
+		this.call = call
+
+		this.prefix = db.job.tracker.prefix + '/' + db.uuid + '/' + call.ord
+
+	}
+	
+	startMessage () {
+
+		const {sql, params} = this.call
+
+		const s = super.startMessage () + ' ' + sql
+
+		if (params.length === 0) return s
+
+		return s + ' ' + stringifyParams (params)
+
+	}
+
+}
+
+DbCallTracker.stringifyParams = stringifyParams
+
+module.exports = DbCallTracker

package/lib/DbClient.js

@@ -1,16 +1,34 @@
 const EventEmitter    = require ('events')
 const {randomUUID}    = require ('crypto')
 
+const DbCall          = require ('./DbCall.js')
 const DbQuery         = require ('./query/DbQuery.js')
 const DbMigrationPlan = require ('./migration/DbMigrationPlan.js')
 
-const NULL = Symbol.for ('NULL')
+const PR_QUERY        = Symbol.for ('query')
+const PR_CALL         = Symbol.for ('call')
+const PR_COLUMNS      = Symbol.for ('columns')
+const PR_COUNT        = Symbol.for ('count')
+
+const set = (o, k, v) => Object.defineProperty (o, k, {
+	configurable: false,
+	enumerable: false,
+	get: () => v
+})
+
+const setAll = (rows, call, q) => {
+	set (rows, PR_CALL, call)
+	set (rows, PR_COLUMNS, call.columns)
+	if (q instanceof DbQuery) set (rows, PR_QUERY, q)	
+}
 
 class DbClient extends EventEmitter {
 
 	constructor () {
 	
 		super ()
+
+		this.count = 0
 		
 		this.uuid = randomUUID ()
 	
@@ -22,106 +40,122 @@ class DbClient extends EventEmitter {
 	
 	}
 
-	async getArrayBySql (sql, params = [], options = {}) {
+	async getArrayOnly (q, params = [], o = {}) {
 
-		const maxRows = options.maxRows || 1000
-		const isPartial = options.isPartial === true
-		
-		const s = await this.getStream (sql, params, options)
-		const rows = []
+		const options = {...o}
 
-		Object.defineProperty (rows, Symbol.for ('columns'), {
-			configurable: false,
-			enumerable: false,
-			get: () => s [Symbol.for ('columns')]
-		})
-		
-		for await (const r of s) {
+		if (!('maxRows' in options)) {
+			options.maxRows = 1000
+			options.checkOverflow = true
+		}
 
-			if (rows.length === maxRows && !isPartial) {
+		const call = this.call (q, params, options)
 
-				const err = Error (maxRows + ' rows limit exceeded. Please fix the request or consider using getStream instead of getArray')
+		await call.exec ()
 
-				s.destroy (err)
+		const {rows} = call
 
-				throw err
+		setAll (rows, call, q)
 
-			}				
+		return rows
 
-			rows.push (r === NULL ? null : r)
+	}
 
-			if (rows.length === maxRows && isPartial) {
-				
-				s.destroy ()
+	async getArray (q, p, o) {
 
-				break
-				
-			}
+		const arrayOnly = this.getArrayOnly (q, p, o)
 
-		}
+		if (!(q instanceof DbQuery) || !('offset' in q.options)) return arrayOnly
 
+		const [rows, cnt] = await Promise.all ([
+			arrayOnly,
+			this.getScalar (q.toQueryCount ())
+		])
+		
+		set (rows, PR_COUNT, parseInt (cnt))
+		
 		return rows
 
 	}
+
+	async getObject (sql, params = [], options = {}) {
 	
-	async getArray (q, p, o) {
+		const {model} = this; if (model) {
 
-		if (!(q instanceof DbQuery)) return this.getArrayBySql (q, p, o)
-		
-		const addCount = 'offset' in q.options, todo = addCount ? [null, this.getScalar (q.toQueryCount ())] : []
-				
-		const params = this.lang.toParamsSql (q), sql = params.pop (); todo [0] = this.getArrayBySql (sql, params, o)
+			const relation = model.find (sql); if (relation) {
 
-		const done = await Promise.all (todo), rows = done [0]
+				const {name, pk} = relation, filters = []; for (let i = 0; i < pk.length; i ++) filters.push ([pk [i], '=', params [i]])
 
-		Object.defineProperty (rows, Symbol.for ('query'), {
-			configurable: false,
-			enumerable: false,
-			get: () => q
-		})
+				sql = model.createQuery ([[name, {filters}]])
 
-		if (addCount) {
+			}
 
-			const count = parseInt (done [1]), get = () => count
+		}
 
-			Object.defineProperty (rows, Symbol.for ('count'), {
-				configurable: false,
-				enumerable: false,
-				get
-			})
+		return this.call (sql, params, {
+			...options,
+			minRows: 1,
+			maxRows: 1,
+		}).exec ()
 
-		}
+	}
+
+	async getScalar (sql, params = [], options = {}) {
+
+		return this.call (sql, params, {
+			notFound: undefined,
+			...options,
+			rowMode: 'scalar',
+			minRows: 1,
+			maxRows: 1,
+		}).exec ()
 		
-		return rows
-	
-	}		
+	}
 
-	async getObject (sqlOrName, p = [], options = {}) {
-	
-		const params = this.lang.genSelectObjectParamsSql (sqlOrName, p), sql = params.pop ()
+	async do (sql, params = [], options = {}) {
 
-		const a = await this.getArray (sql, params, {
+		const call = this.call (sql, params, {
 			...options,
-			maxRows: 1,		
-			isPartial: true,
+			maxRows: 0,
 		})
 		
-		if (a.length === 1) return a [0]
-	
-		const {notFound} = options; if (notFound instanceof Error) throw notFound
-		
-		return 'notFound' in options ? notFound : {}
+		await call.exec ()
+
+		return call
 
 	}
 
-	async getScalar (sql, params = [], options = {}) {
+	async getStream (q, p = [], options = {}) {
 
-		return this.getObject (sql, params, {
-			notFound: undefined,
+		const call = this.call (q, p, {
 			...options,
-			rowMode: 'scalar',
+			maxRows: Infinity,
 		})
 		
+		const rows = await call.exec ()
+
+		setAll (rows, call, q)
+
+		return rows
+		
+	}
+
+	call (q, params, options) {
+
+		if (q instanceof DbQuery) {
+
+			params = this.lang.toParamsSql (q)
+			
+			q = params.pop ()
+
+		}
+
+		const call = new DbCall (this, q, params, options)
+
+		call.tracker = new this.pool.trackerClass (call)
+
+		return call
+
 	}
 
 }

package/lib/DbLang.js

@@ -263,37 +263,6 @@ class DbLang {
 	
 	}
 
-	genSelectObjectParamsSql (sqlOrName, params) {
-			
-		if (!Array.isArray (params)) params = [params]
-
-		const {model} = this
-
-		const relation = model ? model.find (sqlOrName) : undefined; if (relation) {
-
-			const {qName, pk, columns} = relation
-
-			let filter = ''; for (const name of pk) {
-
-				if (filter.length !== 0) filter += ' AND '
-
-				filter += columns [name].qName + '=?'
-
-			}
-
-			params.push (`SELECT * FROM ${qName} WHERE ${filter}`)
-
-		}
-		else {
-		
-			params.push (sqlOrName)
-		
-		}
-
-		return params
-
-	}
-	
 	genInsertParamsSql (name, data) {
 
 		const {model} = this; if (!model) throw Error ('Model not set')
@@ -510,6 +479,30 @@ class DbLang {
 	
 	}
 
+	normalizeSQL (call) {
+
+		call.sql = call.sql.trim ()
+
+		const {sql} = call, {length} = sql; if (length === 0) return
+	
+		let r = '', from = 0
+
+		while (from < length) {
+
+			let to = from + 1; while (to < length && sql.charCodeAt (to) > 32) to ++
+
+			if (r.length !== 0) r += ' '
+
+			r += sql.slice (from, to)
+
+			from = to; while (from < length && sql.charCodeAt (from) <= 32) from ++
+	
+		}
+	
+		call.sql = r
+
+	}
+
 }
 
 module.exports = DbLang

package/lib/DbPool.js

@@ -1,5 +1,5 @@
 const {ResourcePool} = require ('doix')
-const DbEventLogger = require ('./DbEventLogger.js')
+const DbCallTracker = require ('./DbCallTracker.js')
 
 class DbPool extends ResourcePool {
 
@@ -12,7 +12,7 @@ class DbPool extends ResourcePool {
 		
 		this.logger = o.logger
 
-		this.eventLoggerClass = o.eventLoggerClass || DbEventLogger
+		this.trackerClass = o.trackerClass || DbCallTracker
 
 	}
 	

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "doix-db",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "description": "Shared database related code for doix",
   "main": "index.js",
   "files": [
@@ -40,7 +40,7 @@
   },
   "homepage": "https://github.com/do-/node-doix-db#readme",
   "peerDependencies": {
-    "doix": "^1.0.3"
+    "doix": "^1.0.5"
   },
   "devDependencies": {
     "jest": "^29.6.1"

package/lib/DbEventLogger.js

@@ -1,101 +0,0 @@
-const {EventLogger} = require ('doix')
-const stringEscape  = require ('string-escape-map')
-
-const ESC_PARAMS = new stringEscape ([
-  ['\t', '\\t'],
-  ['\n', '\\n'],
-  ['\r', '\\r'],
-  [ "'", "''"],
-])
-
-const normalizeSpace = s => {
-	
-	s = s.trim (); 
-	
-	const {length} = s; if (length === 0) return s
-
-	let r = '', from = 0
-
-	while (from < length) {
-
-		let to = from + 1; while (to < length && s.charCodeAt (to) > 32) to ++
-
-		if (r.length !== 0) r += ' '
-
-		r += s.slice (from, to)
-
-		from = to; while (from < length && s.charCodeAt (from) <= 32) from ++
-
-	}
-
-	return r
-		
-}
-
-const stringifyParams = params => {
-	
-	if (!Array.isArray (params)) return JSON.stringify (params)
-	
-	if (params.length === 0) return ''
-
-	let s = '['; for (const p of params) {
-	
-		if (s.length !== 1) s += ', '
-		
-		if (typeof p === 'string') {
-		
-			s += "'"
-			s += ESC_PARAMS.escape (p)
-			s += "'"
-		
-		}
-		else {
-
-			s += p
-
-		}
-	
-	}
-	
-	return s + ']'
-
-}
-
-class DbEventLogger extends EventLogger {
-
-	constructor (client) {
-
-		super (client)
-
-		this.client = client
-
-		this.logger = client.logger
-
-	}
-
-	get prefix () {
-	
-		let {client} = this, {job} = client
-
-		let p = job.uuid + '/' + client.uuid
-
-		while (job = job.parent) p = job.uuid + '/' + p
-
-		return p
-
-	}
-	
-	startMessage ({sql, params}) {
-
-		const s = '> ' + normalizeSpace (sql), p = stringifyParams (params)
-
-		return this.message (p.length === 0 ? s : s + ' ' + p)
-
-	}
-
-}
-
-DbEventLogger.normalizeSpace = normalizeSpace 
-DbEventLogger.stringifyParams = stringifyParams
-
-module.exports = DbEventLogger