expressa 2.0.7 → 2.0.9

package/controllers/collections.js

@@ -101,7 +101,7 @@ exports.get = async function (req) {
     }
     req.query.pageitems = parseInt(req.query.pageitems) || parseInt(req.query.limit) || 10
     req.query.offset = util.getDatabaseOffset(req.query.page, req.query.pageitems)
-    req.query.orderby = req.query.orderby || util.normalizeOrderBy({ 'meta.created': 1 }) // paging requires orderby to ensure consistent results with offset and limit
+    req.query.orderby = util.orderByForPagedRequests(req.query.orderby)
     delete req.query.limit
     delete req.query.skip
     let pageData, totalItems

package/db/postgres.js

@@ -1,100 +1,93 @@
-const mongoToPostgres = require('mongo-query-to-postgres-jsonb')
-const util = require('../util')
-
-module.exports = function (settings, collectionId, collection) {
-  const pool = util.getPgPool(settings.postgresql_uri)
-
-  return {
-    init: async function () {
-      if (collection.plainStringIds) {
-        await pool.query('CREATE TABLE IF NOT EXISTS ' + collectionId + ' (id text primary key, data jsonb)')
-      } else {
-        await pool.query('CREATE TABLE IF NOT EXISTS ' + collectionId + ' (id uuid primary key, data jsonb)')
-      }
-    },
-    all: async function () {
-      return this.find({})
-    },
-    find: async function (rawQuery, offset, limit, orderby, fields) {
-      const arrayFields = util.getArrayPaths('', collection.schema)
-      const pgQuery = mongoToPostgres('data', rawQuery || {}, arrayFields)
-      const select = fields ? mongoToPostgres.convertSelect('data', fields, arrayFields) : '*'
-      let query = 'SELECT ' + select + ' FROM ' + collectionId + (pgQuery ? ' WHERE ' + pgQuery : '')
-      if (typeof orderby !== 'undefined') {
-        query += ' ORDER BY '
-        query += orderby.map((ordering) => {
-          return mongoToPostgres.convertDotNotation('data', ordering[0]) + (ordering[1] > 0 ? ' ASC' : ' DESC')
-        }).join(', ')
-      }
-      if (typeof offset !== 'undefined') {
-        query += ' OFFSET ' + offset
-      }
-      if (typeof limit !== 'undefined') {
-        query += ' LIMIT ' + limit
-      }
-      const result = await pool.query(query)
-      return result.rows.map((row) => row.data)
-    },
-    count: async function(rawQuery, offset, limit) {
-      const arrayFields = util.getArrayPaths('', collection.schema)
-      const pgQuery = mongoToPostgres('data', rawQuery || {}, arrayFields)
-      let query = 'SELECT COUNT(*) FROM ' + collectionId + (pgQuery ? ' WHERE ' + pgQuery : '')
-      if (typeof offset !== 'undefined') {
-        query += ' OFFSET ' + offset
-      }
-      if (typeof limit !== 'undefined') {
-        query += ' LIMIT ' + limit
-      }
-      const result = await pool.query(query)
-      return parseInt(result.rows[0].count)
-    },
-    get: async function (id, fields) {
-      const arrayFields = util.getArrayPaths('', collection.schema)
-      const select = fields ? mongoToPostgres.convertSelect('data', fields, arrayFields) : '*'
-      const result = await pool.query(`SELECT ${select} FROM ${collectionId} WHERE id = $1`, [id])
-      if (result.rowCount === 0) {
-        throw new util.ApiError(404, 'document not found')
-      }
-      return result.rows[0].data
-    },
-    create: async function (data) {
-      util.addIdIfMissing(data)
-      try {
-        await pool.query('INSERT INTO ' + collectionId + ' (id, data) VALUES ($1, $2)', [data._id, data])
-      } catch (e) {
-        if (e.message.includes('duplicate key')) {
-          throw new util.ApiError(409, 'document already exists')
-        }
-        throw e
-      }
-      return data._id
-    },
-    update: async function (id, data) {
-      if (typeof data._id === 'undefined') {
-        data._id = id
-      }
-      const result = await pool.query('UPDATE ' + collectionId + ' SET data=$2,id=$3 WHERE id=$1', [id, data, data._id])
-      if (result.rowCount === 0) {
-        throw new util.ApiError(404, 'document not found')
-      }
-      return data
-    },
-    // eslint-disable-next-line no-unused-vars
-    updateWithQuery: async function (query, update, options) {
-      const arrayFields = util.getArrayPaths('', collection.schema)
-      const pgQuery = mongoToPostgres('data', query || {}, arrayFields)
-      const updateSql = mongoToPostgres.convertUpdate('data', update,false)
-      const result = await pool.query('UPDATE ' + collectionId + ' SET data=' + updateSql + ' WHERE ' + pgQuery)
-      return {
-        matchedCount: result.rowCount
-      }
-    },
-    delete: async function (id) {
-      const result = await pool.query('DELETE FROM ' + collectionId + ' WHERE id=$1', [id])
-      if (result.rowCount === 0) {
-        throw new util.ApiError(404, 'document not found')
-      }
-      return 'OK'
-    }
-  }
-}
+const util = require('../util')
+
+module.exports = function (settings, collectionId, collection) {
+  const pool = util.getPgPool(settings.postgresql_uri)
+
+  return {
+    init: async function () {
+      if (collection.plainStringIds) {
+        await pool.query('CREATE TABLE IF NOT EXISTS ' + collectionId + ' (id text primary key, data jsonb)')
+      } else {
+        await pool.query('CREATE TABLE IF NOT EXISTS ' + collectionId + ' (id uuid primary key, data jsonb)')
+      }
+    },
+    all: async function () {
+      return this.find({})
+    },
+    find: async function (rawQuery, offset, limit, orderby, fields) {
+      const pgSelect = util.mongoToPostgresSelect(collectionId, fields)
+      const pgWhere = util.mongoToPostgresWhere(collectionId, rawQuery)
+      let query = 'SELECT ' + pgSelect + ' FROM ' + collectionId + (pgWhere ? ' WHERE ' + pgWhere : '')
+      if (typeof orderby !== 'undefined') {
+        query += ' ORDER BY '
+        query += util.mongoToPostgresOrderBy(collectionId, orderby)
+      }
+      if (typeof offset !== 'undefined') {
+        query += ' OFFSET ' + offset
+      }
+      if (typeof limit !== 'undefined') {
+        query += ' LIMIT ' + limit
+      }
+      const result = await pool.query(query)
+      return result.rows.map((row) => row.data)
+    },
+    count: async function(rawQuery, offset, limit) {
+      const pgWhere = util.mongoToPostgresWhere(collectionId, rawQuery)
+      let query = 'SELECT COUNT(*) FROM ' + collectionId + (pgWhere ? ' WHERE ' + pgWhere : '')
+      if (typeof offset !== 'undefined') {
+        query += ' OFFSET ' + offset
+      }
+      if (typeof limit !== 'undefined') {
+        query += ' LIMIT ' + limit
+      }
+      const result = await pool.query(query)
+      return parseInt(result.rows[0].count)
+    },
+    get: async function (id, fields) {
+      const pgSelect = util.mongoToPostgresSelect(collectionId, fields)
+      const result = await pool.query(`SELECT ${pgSelect} FROM ${collectionId} WHERE id = $1`, [id])
+      if (result.rowCount === 0) {
+        throw new util.ApiError(404, 'document not found')
+      }
+      return result.rows[0].data
+    },
+    create: async function (data) {
+      util.addIdIfMissing(data)
+      try {
+        await pool.query('INSERT INTO ' + collectionId + ' (id, data) VALUES ($1, $2)', [data._id, data])
+      } catch (e) {
+        if (e.message.includes('duplicate key')) {
+          throw new util.ApiError(409, 'document already exists')
+        }
+        throw e
+      }
+      return data._id
+    },
+    update: async function (id, data) {
+      if (typeof data._id === 'undefined') {
+        data._id = id
+      }
+      const result = await pool.query('UPDATE ' + collectionId + ' SET data=$2,id=$3 WHERE id=$1', [id, data, data._id])
+      if (result.rowCount === 0) {
+        throw new util.ApiError(404, 'document not found')
+      }
+      return data
+    },
+    // eslint-disable-next-line no-unused-vars
+    updateWithQuery: async function (query, update, options) {
+      const pgWhere = util.mongoToPostgresWhere(collectionId, query)
+      const updateSql = util.mongoToPostgresUpdate(collectionId, update)
+      const result = await pool.query('UPDATE ' + collectionId + ' SET data=' + updateSql + ' WHERE ' + pgWhere)
+      return {
+        matchedCount: result.rowCount
+      }
+    },
+    delete: async function (id) {
+      const result = await pool.query('DELETE FROM ' + collectionId + ' WHERE id=$1', [id])
+      if (result.rowCount === 0) {
+        throw new util.ApiError(404, 'document not found')
+      }
+      return 'OK'
+    }
+  }
+}

package/listeners_validation.js

@@ -18,13 +18,13 @@ module.exports = async function (api) {
   const collections = await api.db.collection.all()
   collections.forEach((collection) => {
     const schema = addImplicitFields(collection.schema)
-    util.addSchemaValidator(collection._id, schema)
+    util.addSchema(collection._id, schema)
   })
 
   // Load new and update validators as necessary
   api.addCollectionListener('changed', 'collection', function updateSchemaValidators (req, collection, data) {
     const schema = addImplicitFields(data.schema)
-    util.addSchemaValidator(data._id, schema)
+    util.addSchema(data._id, schema)
   })
 
   api.addCollectionListener('get', ['collection', 'schemas'], function ensureIdAdded (req, collection, data) {

package/modules/admin/package.json

@@ -59,7 +59,7 @@
     "friendly-errors-webpack-plugin": "1.7.0",
     "html-webpack-plugin": "^4.0.0",
     "mini-css-extract-plugin": "0.4.1",
-    "node-notifier": "5.2.1",
+    "node-notifier": "8.0.1",
     "node-sass": "^6.0.1",
     "optimize-css-assets-webpack-plugin": "5.0.0",
     "ora": "3.0.0",
@@ -72,7 +72,7 @@
     "sass-loader": "7.0.3",
     "script-ext-html-webpack-plugin": "^2.1.5",
     "semver": "5.5.0",
-    "shelljs": "0.8.2",
+    "shelljs": "0.8.5",
     "svg-sprite-loader": "3.8.0",
     "svgo": "^1.3.2",
     "terser-webpack-plugin": "^2.2.3",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "expressa",
-  "version": "2.0.7",
+  "version": "2.0.9",
   "description": "API framework using JSON schemas",
   "main": "index.js",
   "repository": "https://github.com/thomas4019/expressa",

package/util.js

@@ -1,4 +1,3 @@
-const randomstring = require('randomstring')
 const {v4} = require('uuid')
 const debug = require('debug')('expressa')
 const crypto = require('crypto')
@@ -6,6 +5,7 @@ const pg = require('pg')
 const pgPools = {}
 const dot = require('dot-object')
 const mongoQuery = require('mongo-query')
+const mongoToPostgres = require('mongo-query-to-postgres-jsonb')
 const sift = require('sift')
 const auth = require('./auth/index')
 const Ajv = require('ajv')
@@ -33,9 +33,11 @@ ajv.addKeyword({
 const formatKey = ajv.getKeyword('format')
 formatKey.type = formatKey.type.concat(['array', 'boolean', 'object'])
 
+const schemas = {}
 const schemaValidators = {}
 
-exports.addSchemaValidator = function(collection, schema) {
+exports.addSchema = function(collection, schema) {
+  schemas[collection] = schema
   schemaValidators[collection] = ajv.compile(schema)
 }
 
@@ -141,6 +143,27 @@ function _exclude(obj, source) {
   return data
 }
 
+exports.mongoToPostgresSelect = function(collection, fields) {
+  const arrayFields = exports.getArrayPaths('', schemas[collection])
+  return fields ? mongoToPostgres.convertSelect('data', fields, arrayFields) : '*'
+}
+
+exports.mongoToPostgresUpdate = function(collection, query) {
+  return mongoToPostgres.convertUpdate('data', query,false)
+}
+
+exports.mongoToPostgresWhere = function(collection, query) {
+  const arrayFields = exports.getArrayPaths('', schemas[collection])
+  return mongoToPostgres('data', query || {}, arrayFields)
+}
+
+exports.mongoToPostgresOrderBy = function(collection, orderby) {
+  const normalizedOrderBy = exports.normalizeOrderBy(orderby)
+  return normalizedOrderBy.map((ordering) => {
+    return mongoToPostgres.convertDotNotation('data', ordering[0]) + (ordering[1] > 0 ? ' ASC' : ' DESC')
+  }).join(', ')
+}
+
 exports.mongoProject = function(record, projection) {
   if (!projection || Object.keys(projection).length < 1) {
     return record
@@ -162,6 +185,18 @@ exports.mongoUpdate = function(doc, update) {
   return mongoQuery(doc, {}, update)
 }
 
+// paging requires orderby to include a field that is known to be unique and constant
+// to ensure consistent results from database with offset and limit. Chosen field should
+// also represent a preferred ordering when doing final sort
+exports.orderByForPagedRequests = function(orderby, finalSortField = 'meta.created') {
+  orderby = exports.normalizeOrderBy(orderby || {})
+  const allFields = orderby.map(([field]) => field)
+  if (!allFields.includes(finalSortField)) {
+    orderby.push([finalSortField, 1])
+  }
+  return orderby
+}
+
 exports.normalizeOrderBy = function(orderby) {
   if (Array.isArray(orderby)) {
     orderby = orderby.map(function (ordering) {
@@ -172,6 +207,7 @@ exports.normalizeOrderBy = function(orderby) {
           return [ordering[0], 1] // add 1 (default to ascending sort)
         }
       }
+      // if reaches here, is already normalized
       return ordering
     })
   } else if (typeof orderby === 'object') {