@cap-js/postgres 1.1.0 → 1.2.1

package/CHANGELOG.md

@@ -4,6 +4,23 @@
 - The format is based on [Keep a Changelog](http://keepachangelog.com/).
 - This project adheres to [Semantic Versioning](http://semver.org/).
 
+## Version 1.2.1 - 2023-09-08
+
+### Changed
+
+- Bump minimum required version of `@cap-js/db-service` 
+
+## Version 1.2.0 - 2023-09-06
+
+### Added
+
+- Reduced the usage of `is not distinct [not] from`. #157
+
+### Fixed
+
+- [Reserved words](https://www.postgresql.org/docs/current/sql-keywords-appendix.html) are now used to automatically escape reserved words which are used as identifier. #178
+- Remove column count limitation. #150
+
 ## Version 1.1.0 - 2023-08-01
 
 ### Added

package/lib/PostgresService.js

@@ -1,7 +1,9 @@
 const { SQLService } = require('@cap-js/db-service')
-const { Client } = require('pg')
+const { Client, Query } = require('pg')
 const cds = require('@sap/cds/lib')
 const crypto = require('crypto')
+const { Writable, Readable } = require('stream')
+const sessionVariableMap = require('./session.json')
 
 class PostgresService extends SQLService {
   init() {
@@ -61,18 +63,18 @@ class PostgresService extends SQLService {
   }
 
   async set(variables) {
-    // REVISIT: remove when all environment variables are aligned
     // RESTRICTIONS: 'Custom parameter names must be two or more simple identifiers separated by dots.'
-    const nameMap = {
-      '$user.id': 'cap.applicationuser',
-      '$user.locale': 'cap.locale',
-      '$valid.from': 'cap.valid_from',
-      '$valid.to': 'cap.valid_to',
-    }
-
     const env = {}
+
+    // Check all properties on the variables object
     for (let name in variables) {
-      env[nameMap[name]] = variables[name]
+      env[sessionVariableMap[name] || name] = variables[name]
+    }
+
+    // Explicitly check for the default session variable properties
+    // As they are getters and not own properties of the object
+    for (let name in sessionVariableMap) {
+      if (variables[name]) env[sessionVariableMap[name]] = variables[name]
     }
 
     return Promise.all([
@@ -106,14 +108,14 @@ class PostgresService extends SQLService {
      * The group by is done by the key column to make sure that only one collation per key is returned
      */
     const cSQL = `
-SELECT 
+SELECT
   SUBSTRING(collname, 1, 2) AS K,
-  MIN(collname) AS V 
-FROM 
-  pg_collation 
-WHERE 
-  collprovider = 'c' AND 
-  collname LIKE '__>___' ESCAPE '>' 
+  MIN(collname) AS V
+FROM
+  pg_collation
+WHERE
+  collprovider = 'c' AND
+  collname LIKE '__>___' ESCAPE '>'
 GROUP BY k
 `
 
@@ -134,6 +136,7 @@ GROUP BY k
 
   prepare(sql) {
     const query = {
+      _streams: 0,
       text: sql,
       // Track queries name for postgres referencing prepare statements
       // sha1 as it needs to be less then 63 characters
@@ -142,7 +145,9 @@ GROUP BY k
     return {
       run: async values => {
         // REVISIT: SQLService provides empty values as {} for plain SQL statements - PostgreSQL driver expects array or nothing - see issue #78
-        const result = await this.dbc.query({ ...query, values: this._getValues(values) })
+        let newQuery = this._prepareStreams(query, values)
+        if (typeof newQuery.then === 'function') newQuery = await newQuery
+        const result = await this.dbc.query(newQuery)
         return { changes: result.rowCount }
       },
       get: async values => {
@@ -159,6 +164,14 @@ GROUP BY k
           throw Object.assign(e, { sql: sql + '\n' + new Array(e.position).fill(' ').join('') + '^' })
         }
       },
+      stream: async (values, one) => {
+        try {
+          const streamQuery = new QueryStream({ ...query, values: this._getValues(values) }, one)
+          return await this.dbc.query(streamQuery)
+        } catch (e) {
+          throw Object.assign(e, { sql: sql + '\n' + new Array(e.position).fill(' ').join('') + '^' })
+        }
+      },
     }
   }
 
@@ -170,6 +183,60 @@ GROUP BY k
     return values
   }
 
+  _prepareStreams(query, values) {
+    values = this._getValues(values)
+    if (!values) return query
+
+    const streams = []
+    const newValues = []
+    let sql = query.text
+    if (Array.isArray(values)) {
+      values.forEach((value, i) => {
+        if (value instanceof Readable) {
+          const streamID = query._streams++
+          const isBinary = value.type === 'binary'
+          const paramStream = new ParameterStream(query.name, streamID)
+          if (isBinary) value.setEncoding('base64')
+          value.pipe(paramStream)
+          value.on('error', err => paramStream.emit('error', err))
+          streams[i] = paramStream
+          newValues[i] = streamID
+          sql = sql.replace(
+            new RegExp(`\\$${i + 1}`, 'g'),
+            // Don't ask about the dollar signs
+            `(SELECT ${isBinary ? `DECODE(PARAM,'base64')` : 'PARAM'} FROM "$$$$PARAMETER_BUFFER$$$$" WHERE NAME='${
+              query.name
+            }' AND ID=$${i + 1})`,
+          )
+          return
+        }
+        newValues[i] = value
+      })
+    }
+
+    if (streams.length > 0) {
+      return (async () => {
+        const newQuery = {
+          text: sql,
+          // Even with the changed SQL it might be common to call this statement with the same parameters as streams
+          // As the streams are selected with their ID as prepared statement parameter, the sql is the same
+          name: crypto.createHash('sha1').update(sql).digest('hex'),
+          values: newValues,
+        }
+        await this.dbc.query({
+          text: 'CREATE TEMP TABLE IF NOT EXISTS "$$PARAMETER_BUFFER$$" (PARAM TEXT, NAME TEXT, ID INT) ON COMMIT DROP',
+        })
+        const proms = []
+        for (const stream of streams) {
+          proms.push(this.dbc.query(stream))
+        }
+        await Promise.all(proms)
+        return newQuery
+      })()
+    }
+    return { ...query, values }
+  }
+
   async exec(sql) {
     return this.dbc.query(sql)
   }
@@ -249,18 +316,9 @@ GROUP BY k
       return super.from(from)
     }
 
-    // REVISIT: pg requires alias for {val}
-    SELECT_columns({ SELECT }) {
-      // REVISIT: Genres cqn has duplicate ID column
-      if (!SELECT.columns) return '*'
-      const unique = {}
-      return SELECT.columns
-        .map(x => `${this.column_expr(x)} as ${this.quote(this.column_name(x))}`)
-        .filter(x => {
-          if (unique[x]) return false
-          unique[x] = true
-          return true
-        })
+    column_alias4(x, q) {
+      if (!x.as && 'val' in x) return String(x.val)
+      return super.column_alias4(x, q)
     }
 
     SELECT_expand({ SELECT }, sql) {
@@ -268,19 +326,25 @@ GROUP BY k
       const queryAlias = this.quote(SELECT.from?.as || (SELECT.expand === 'root' && 'root'))
       const cols = SELECT.columns.map(x => {
         const name = this.column_name(x)
-        let col = `${this.string(name)},${this.output_converter4(x.element, queryAlias + '.' + this.quote(name))}`
+        const outputConverter = this.output_converter4(x.element, `${queryAlias}.${this.quote(name)}`)
+        let col = `${outputConverter} as ${this.doubleQuote(name)}`
 
         if (x.SELECT?.count) {
           // Return both the sub select and the count for @odata.count
           const qc = cds.ql.clone(x, { columns: [{ func: 'count' }], one: 1, limit: 0, orderBy: 0 })
-          col += `, '${name}@odata.count',${this.expr(qc)}`
+          col += `,${this.expr(qc)} as ${this.doubleQuote(`${name}@odata.count`)}`
         }
         return col
       })
-      let obj = `json_build_object(${cols})`
+      // REVISIT: Remove SELECT ${cols} by adjusting SELECT_columns
+      let obj = `row_to_json(${queryAlias}.*)`
       return `SELECT ${
         SELECT.one || SELECT.expand === 'root' ? obj : `coalesce(json_agg(${obj}),'[]'::json)`
-      } as _json_ FROM (${sql}) as ${queryAlias}`
+      } as _json_ FROM (SELECT ${cols} FROM (${sql}) as ${queryAlias}) as ${queryAlias}`
+    }
+
+    doubleQuote(name) {
+      return `"${name.replace(/"/g, '""')}"`
     }
 
     INSERT(q, isUpsert = false) {
@@ -291,20 +355,24 @@ GROUP BY k
         // Adjusts json path expressions to be postgres specific
         .replace(/->>'\$(?:(?:\."(.*?)")|(?:\[(\d*)\]))'/g, (a, b, c) => (b ? `->>'${b}'` : `->>${c}`))
         // Adjusts json function to be postgres specific
-        .replace('json_each(?)', 'json_array_elements($1)')
+        .replace('json_each(?)', 'json_array_elements($1::JSON)')
         .replace(/json_type\((\w+),'\$\."(\w+)"'\)/g, (_a, b, c) => `json_typeof(${b}->'${c}')`))
     }
 
+    param({ ref }) {
+      this._paramCount = this._paramCount || 1
+      if (ref.length > 1) throw cds.error`Unsupported nested ref parameter: ${ref}`
+      return ref[0] === '?' ? `$${this._paramCount++}` : `:${ref}`
+    }
+
     val(val) {
       const ret = super.val(val)
       return ret === '?' ? `$${this.values.length}` : ret
     }
 
-    operator(x) {
+    operator(x, i, xpr) {
       if (x === 'regexp') return '~'
-      if (x === '=') return 'is not distinct from'
-      if (x === '!=') return 'is distinct from'
-      else return x
+      else return super.operator(x, i, xpr)
     }
 
     defaultValue(defaultValue = this.context.timestamp.toISOString()) {
@@ -346,14 +414,14 @@ GROUP BY k
       // REVISIT: Remove that with upcomming fixes in cds.linked
       Double: (e, t) => `CAST(${e} as decimal${t.precision && t.scale ? `(${t.precision},${t.scale})` : ''})`,
       DecimalFloat: (e, t) => `CAST(${e} as decimal${t.precision && t.scale ? `(${t.precision},${t.scale})` : ''})`,
-      Binary: e => `CAST(${e} as bytea)`,
-      LargeBinary: e => `CAST(${e} as bytea)`,
+      Binary: e => `DECODE(${e},'base64')`,
+      LargeBinary: e => `DECODE(${e},'base64')`,
     }
 
     static OutputConverters = {
       ...super.OutputConverters,
-      Binary: e => e,
-      LargeBinary: e => e,
+      Binary: e => `ENCODE(${e},'base64')`,
+      LargeBinary: e => `ENCODE(${e},'base64')`,
       Date: e => `to_char(${e}, 'YYYY-MM-DD')`,
       Time: e => `to_char(${e}, 'HH24:MI:SS')`,
       DateTime: e => `to_char(${e}, 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`,
@@ -403,7 +471,7 @@ GROUP BY k
     }
   }
 
-  async tenant({ database, tenant }) {
+  async tenant({ database, tenant }, clean = false) {
     const creds = {
       database: database,
       usergroup: `${database}_USERS`,
@@ -413,18 +481,20 @@ GROUP BY k
     creds.password = creds.user
 
     try {
-      await this.tx(async tx => {
-        // await tx.run(`DROP USER IF EXISTS "${creds.user}"`)
-        await tx
-          .run(`CREATE USER "${creds.user}" IN GROUP "${creds.usergroup}" PASSWORD '${creds.password}'`)
-          .catch(e => {
-            if (e.code === '42710') return
-            throw e
-          })
-      })
-      await this.tx(async tx => {
-        await tx.run(`GRANT CREATE, CONNECT ON DATABASE "${creds.database}" TO "${creds.user}";`)
-      })
+      if (!clean) {
+        await this.tx(async tx => {
+          // await tx.run(`DROP USER IF EXISTS "${creds.user}"`)
+          await tx
+            .run(`CREATE USER "${creds.user}" IN GROUP "${creds.usergroup}" PASSWORD '${creds.password}'`)
+            .catch(e => {
+              if (e.code === '42710') return
+              throw e
+            })
+        })
+        await this.tx(async tx => {
+          await tx.run(`GRANT CREATE, CONNECT ON DATABASE "${creds.database}" TO "${creds.user}";`)
+        })
+      }
 
       // Update credentials to new Schema owner
       await this.disconnect()
@@ -433,7 +503,7 @@ GROUP BY k
       // Create new schema using schema owner
       await this.tx(async tx => {
         await tx.run(`DROP SCHEMA IF EXISTS "${creds.schema}" CASCADE`)
-        await tx.run(`CREATE SCHEMA "${creds.schema}" AUTHORIZATION "${creds.user}"`).catch(() => {})
+        if (!clean) await tx.run(`CREATE SCHEMA "${creds.schema}" AUTHORIZATION "${creds.user}"`).catch(() => {})
       })
     } finally {
       await this.disconnect()
@@ -441,4 +511,204 @@ GROUP BY k
   }
 }
 
+class QueryStream extends Query {
+  constructor(config, one) {
+    // REVISIT: currently when setting the row chunk size
+    // it results in an inconsistent connection state
+    // if (!one) config.rows = 1000
+    super(config)
+
+    this._one = one || config.one
+
+    this.stream = new Readable({
+      read: this.rows
+        ? () => {
+            this.stream.pause()
+            // Request more rows
+            this.connection.execute({
+              portal: this.portal,
+              rows: this.rows,
+            })
+            this.connection.flush()
+          }
+        : () => {},
+    })
+    this.push = this.stream.push.bind(this.stream)
+
+    this._prom = new Promise((resolve, reject) => {
+      this.once('error', reject)
+      this.once('end', () => {
+        if (!this._one) this.push(this.constructor.close)
+        this.push(null)
+        if (this.stream.isPaused()) this.stream.resume()
+        resolve(null)
+      })
+      this.once('row', row => {
+        if (row == null) return resolve(null)
+        resolve(this.stream)
+      })
+    })
+  }
+
+  static sep = Buffer.from(',')
+  static open = Buffer.from('[')
+  static close = Buffer.from(']')
+
+  // Trigger query initialization
+  _getRows(connection) {
+    this.connection = connection
+    connection.execute({
+      portal: this.portal,
+      rows: this.rows ? 1 : undefined,
+    })
+    if (this.rows) {
+      connection.flush()
+    } else {
+      connection.sync()
+    }
+  }
+
+  // Delay requesting more rows until next is called
+  handlePortalSuspended() {
+    this.stream.resume()
+  }
+
+  // Provides metadata information from the database
+  handleRowDescription(msg) {
+    // Use default parser for binary results
+    if (msg.fields.length === 1 && msg.fields[0].dataTypeID === 17) {
+      this.handleDataRow = this.handleBinaryRow
+    } else {
+      this.handleDataRow = msg => {
+        const val = msg.fields[0]
+        if (!this._one && val !== null) this.push(this.constructor.open)
+        this.emit('row', val)
+        this.push(val)
+        delete this.handleDataRow
+      }
+    }
+    return super.handleRowDescription(msg)
+  }
+
+  // Called when a new row is received
+  handleDataRow(msg) {
+    this.push(this.constructor.sep)
+    this.push(msg.fields[0])
+  }
+
+  // Called when a new binary row is received
+  handleBinaryRow(msg) {
+    const val = msg.fields[0] === null ? null : this._result._parsers[0](msg.fields[0])
+    this.push(val)
+    this.emit('row', val)
+  }
+
+  then(resolve, reject) {
+    return this._prom.then(resolve, reject)
+  }
+}
+
+class ParameterStream extends Writable {
+  constructor(queryName, id) {
+    super({})
+    this.queryName = queryName
+    this.id = id
+    this.text = `COPY "$$PARAMETER_BUFFER$$"(param,name,id) FROM STDIN DELIMITER ',' QUOTE '${this.constructor.sep}' CSV`
+    this.lengthBuffer = Buffer.from([0x64, 0, 0, 0, 0])
+
+    // Flush quote character before input stream
+    this.flushChunk = chunk => {
+      delete this.flushChunk
+
+      this.lengthBuffer.writeUInt32BE(chunk.length + 5, 1)
+      this.connection.stream.write(this.lengthBuffer)
+      this.connection.stream.write(Buffer.from(this.constructor.sep))
+      return this.connection.stream.write(chunk)
+    }
+  }
+
+  static sep = String.fromCharCode(31) // Separator One
+  static done = Buffer.from([0x63, 0, 0, 0, 4])
+
+  then(resolve, reject) {
+    this.on('error', reject)
+    this.on('finish', resolve)
+  }
+
+  /**
+   * Indicates that the query was started by the connection
+   * @param {Object} connection
+   */
+  submit(connection) {
+    this.connection = connection
+    // Initialize query to be executed
+    connection.query(this.text)
+  }
+
+  // Used by the client to handle timeouts
+  callback() {}
+
+  _write(chunk, enc, cb) {
+    return this.flush(chunk, cb)
+  }
+
+  _construct(cb) {
+    this.handleCopyInResponse = () => cb()
+  }
+
+  _destroy(err, cb) {
+    this.handleError = () => {
+      this.callback()
+      this.connection = null
+      cb(err)
+    }
+    this.connection.sendCopyFail(err ? err.message : 'ParameterStream early destroy')
+  }
+
+  _final(cb) {
+    const sep = this.constructor.sep
+    this.flush(Buffer.from(`${sep},${this.queryName},${this.id}`), err => {
+      if (err) return cb(err)
+      this._finish = () => {
+        this.emit('finish')
+        cb()
+      }
+      this._destroy = (err, cb) => cb(err)
+      this.connection.stream.write(this.constructor.done)
+    })
+  }
+
+  flush(chunk, callback) {
+    if (this.flushChunk(chunk)) {
+      return callback()
+    }
+    this.connection.stream.once('drain', callback)
+  }
+
+  flushChunk(chunk) {
+    this.lengthBuffer.writeUInt32BE(chunk.length + 4, 1)
+    this.connection.stream.write(this.lengthBuffer)
+    return this.connection.stream.write(chunk)
+  }
+
+  handleError(e) {
+    this.callback()
+    this.emit('error', e)
+    this.connection = null
+  }
+
+  handleCommandComplete(msg) {
+    const match = /COPY (\d+)/.exec((msg || {}).text)
+    if (match) {
+      this.rowCount = parseInt(match[1], 10)
+    }
+  }
+
+  handleReadyForQuery() {
+    this.callback()
+    this._finish()
+    this.connection = null
+  }
+}
+
 module.exports = PostgresService

package/lib/ReservedWords.json

@@ -1,4 +1,102 @@
 {
+  "ALL": 1,
+  "ANALYSE": 1,
+  "ANALYZE": 1,
+  "AND": 1,
+  "ANY": 1,
+  "ARRAY": 1,
+  "AS": 1,
+  "ASC": 1,
+  "ASYMMETRIC": 1,
+  "AUTHORIZATION": 1,
   "BINARY": 1,
-  "ARRAY": 1
+  "BOTH": 1,
+  "CASE": 1,
+  "CAST": 1,
+  "CHECK": 1,
+  "COLLATE": 1,
+  "COLLATION": 1,
+  "COLUMN": 1,
+  "CONCURRENTLY": 1,
+  "CONSTRAINT": 1,
+  "CREATE": 1,
+  "CROSS": 1,
+  "CURRENT_CATALOG": 1,
+  "CURRENT_DATE": 1,
+  "CURRENT_ROLE": 1,
+  "CURRENT_SCHEMA": 1,
+  "CURRENT_TIME": 1,
+  "CURRENT_TIMESTAMP": 1,
+  "CURRENT_USER": 1,
+  "DEFAULT": 1,
+  "DEFERRABLE": 1,
+  "DESC": 1,
+  "DISTINCT": 1,
+  "DO": 1,
+  "ELSE": 1,
+  "END": 1,
+  "EXCEPT": 1,
+  "FALSE": 1,
+  "FETCH": 1,
+  "FOR": 1,
+  "FOREIGN": 1,
+  "FREEZE": 1,
+  "FROM": 1,
+  "FULL": 1,
+  "GRANT": 1,
+  "GROUP": 1,
+  "HAVING": 1,
+  "ILIKE": 1,
+  "IN": 1,
+  "INITIALLY": 1,
+  "INNER": 1,
+  "INTERSECT": 1,
+  "INTO": 1,
+  "IS": 1,
+  "ISNULL": 1,
+  "JOIN": 1,
+  "LATERAL": 1,
+  "LEADING": 1,
+  "LEFT": 1,
+  "LIKE": 1,
+  "LIMIT": 1,
+  "LOCALTIME": 1,
+  "LOCALTIMESTAMP": 1,
+  "NATURAL": 1,
+  "NOT": 1,
+  "NOTNULL": 1,
+  "NULL": 1,
+  "OFFSET": 1,
+  "ON": 1,
+  "ONLY": 1,
+  "OR": 1,
+  "ORDER": 1,
+  "OUTER": 1,
+  "OVERLAPS": 1,
+  "PLACING": 1,
+  "PRIMARY": 1,
+  "REFERENCES": 1,
+  "RETURNING": 1,
+  "RIGHT": 1,
+  "SELECT": 1,
+  "SESSION_USER": 1,
+  "SIMILAR": 1,
+  "SOME": 1,
+  "SYMMETRIC": 1,
+  "TABLE": 1,
+  "TABLESAMPLE": 1,
+  "THEN": 1,
+  "TO": 1,
+  "TRAILING": 1,
+  "TRUE": 1,
+  "UNION": 1,
+  "UNIQUE": 1,
+  "USER": 1,
+  "USING": 1,
+  "VARIADIC": 1,
+  "VERBOSE": 1,
+  "WHEN": 1,
+  "WHERE": 1,
+  "WINDOW": 1,
+  "WITH": 1
 }

package/lib/func.js

@@ -1,4 +1,11 @@
+const session = require('./session.json')
+
 const StandardFunctions = {
+  session_context: x => {
+    let sql = `current_setting('${ session[x.val] || x.val }')`
+    if (x.val === '$now') sql += '::timestamp'
+    return sql
+  },
   countdistinct: x => `count(distinct ${x || '*'})`,
   contains: (...args) => `(coalesce(strpos(${args}),0) > 0)`,
   indexof: (x, y) => `strpos(${x},${y}) - 1`, // sqlite instr is 1 indexed

package/lib/session.json

@@ -0,0 +1,7 @@
+{
+  "$user.id": "cap.applicationuser",
+  "$user.locale": "cap.locale",
+  "$now": "cap.now",
+  "$valid.from": "cap.valid_from",
+  "$valid.to": "cap.valid_to"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js/postgres",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "CDS database service for Postgres",
   "homepage": "https://github.com/cap-js/cds-dbs/tree/main/postgres#cds-database-service-for-postgres",
   "repository": {
@@ -27,11 +27,11 @@
     "npm": ">=8"
   },
   "scripts": {
-    "setup": "docker-compose -f pg-stack.yml up -d",
-    "test": "npm run setup && jest --silent"
+    "test": "npm start && jest --silent",
+    "start": "docker-compose -f pg-stack.yml up -d"
   },
   "dependencies": {
-    "@cap-js/db-service": "^1.1.0",
+    "@cap-js/db-service": "^1.2.1",
     "pg": "^8"
   },
   "peerDependencies": {