@vida-global/core 2.1.1 → 2.2.0

package/lib/activeRecord/README.md

@@ -98,7 +98,7 @@ module.exports = {
 Available migration helpers:
 
 ```js
-createTable(tableName, details, { timestamps })
+createTable(tableName, details, { timestamps, paranoid })
 dropTable(tableName)
 addColumn(tableName, columnName, columnDetails)
 removeColumn(tableName, columnName)
@@ -111,6 +111,7 @@ changeColumn(tableName, columnName, dataTypeOrOptions)
 **Conventions:**
 
 - `createTable` adds an auto-increment `id` primary key and `created_at` / `updated_at` columns by default. Pass `{ timestamps: false }` to skip the timestamps.
+- Pass `{ paranoid: true }` to add a nullable `deleted_at` column. Models over that table soft delete: `initializationOptions` turns on `paranoid` for any table whose schema has `deleted_at`, so `destroy()` sets the column and ordinary queries skip the row. Unique indexes on a paranoid table should be partial — `where: { deleted_at: null }` — or a deleted row will still block re-creating the same one.
 - Column names are automatically converted to snake_case, so `teamId` in the details object becomes `team_id` in the database.
 - Each migration runs inside a transaction.
 
@@ -307,8 +308,3 @@ class User extends ActiveRecord.BaseRecord {
 ```
 
 See [Sequelize validators](https://sequelize.org/docs/v6/core-concepts/validations-and-constraints/).
-
-
-## TODO
-
-- **Paranoid (soft delete).** The infrastructure for a `deleted_at` column is wired in `initializationOptions`, but `paranoid: true` is not set, so soft deletes are not yet usable out of the box.

package/lib/activeRecord/baseRecord.js

@@ -33,21 +33,74 @@ class BaseRecord extends Model {
     /***********************************************************************************************
     * QUERIES
     ***********************************************************************************************/
-    static async find(ids) {
+    static async find(ids, { useWriteDB=false, ...options }={}) {
         if (this.isCacheable) {
-            return await this._cachedFind(...arguments);
+            return await this._cachedFind(ids, options);
         } else if (Array.isArray(ids)) {
             const pk = this.primaryKeyAttribute
-            return await this.where({[pk]: ids});
+            return await this.where({[pk]: ids}, { useMaster: useWriteDB });
         } else {
-            return await this.findByPk(ids);
+            return await this.findByPk(ids, { useMaster: useWriteDB });
         }
     }
 
 
     static async where(where, options={}) {
-        const parameters = {...options, where };
-        return await this.findAll(parameters);
+        const { page, pageSize, ...queryOptions } = options;
+
+        if (!page && !pageSize) {
+            const parameters = {...queryOptions, where };
+            return await this.findAll(parameters);
+        }
+
+        const offset = pageSize * (page - 1);
+        const rows = await this.findAll({
+            ...queryOptions,
+            where,
+            order: queryOptions.order || [this.defaultSort()],
+            limit: pageSize,
+            offset,
+        });
+
+        const totalCount = await this.paginatedTotalCount(rows, where,
+            {
+                offset,
+                pageSize,
+                queryOptions,
+            })
+
+        return { rows, totalCount }
+    }
+
+
+    static async transaction(callback) {
+        return await this.connection.queryInterface.transaction(callback);
+    }
+
+
+    static defaultSort() {
+        return ['created_at', 'DESC'];
+    }
+
+
+    static async paginatedTotalCount(rows, where, { offset, pageSize, queryOptions }) {
+        const isFullPage = rows.length === pageSize
+        const isEmptyPageAfterFirst = rows.length === 0 && offset > 0
+
+        if (isFullPage || isEmptyPageAfterFirst) {
+            return await this.count({...queryOptions, where })
+        }
+
+        return offset + rows.length
+    }
+
+
+    static async execute(query, { raw=false, ...options }={}) {
+        return await this.sequelize.query(query, {
+            type: Sequelize.QueryTypes.SELECT,
+            ...(raw ? {} : { model: this, mapToModel: true }),
+            ...options
+        });
     }
 
 
@@ -78,10 +131,16 @@ class BaseRecord extends Model {
             updatedAt: 'updated_at',
         }
 
-        if (!schema.created_at || !schema.updated_at) {
+        const hasTimestamps = schema.created_at && schema.updated_at;
+
+        if (!hasTimestamps) {
             options.timestamps = false;
         }
 
+        if (hasTimestamps && schema.deleted_at) {
+            options.paranoid = true;
+        }
+
         this.configureOverridenAccessors(schema);
         this.configureVirtualAccessors(schema);
 
@@ -278,30 +337,39 @@ class BaseRecord extends Model {
     }
 
 
-    static _marshallDataForCaching(value) {
-      let marshalledValue;
-      if (value instanceof this) {
-          marshalledValue = JSON.stringify(value.toJSON());
-          marshalledValue = `${this.recordCachingPrefix}:${marshalledValue}`;
-      } else {
-          marshalledValue = JSON.stringify(value);
-      }
-      return marshalledValue;
+    static async clearCacheKey(key) {
+        const client = await this._getRedisClient();
+        await client.del(key);
     }
 
 
-    static _unmarshallCachedData(value) {
-        if (value == null)            return null;
-        if (typeof value != 'string') return value;
+    static async forEachCacheKeyByPattern(pattern, callback, count=100) {
+        const client = await this._getRedisClient();
+        let   batch  = [];
 
-        if (value.startsWith(this.recordCachingPrefix)) {
-            const regExp = new RegExp(`^${this.recordCachingPrefix}:`);
-            value        = value.replace(regExp, '');
-            const data   = JSON.parse(value);
-            return new this(data, {isNewRecord: false});
-        } else {
-            return JSON.parse(value);
+        for await (const scanned of client.scanIterator({ MATCH: pattern, COUNT: count })) {
+            batch.push(...this._scannedKeysAsArray(scanned));
+            if (batch.length < count) continue;
+
+            await callback(batch, client);
+            batch = [];
         }
+
+        if (batch.length) await callback(batch, client);
+    }
+
+
+    // redis 4 yields one key at a time; redis 5 yields an array of keys per scan.
+    static _scannedKeysAsArray(scanned) {
+        if (Array.isArray(scanned)) return scanned;
+        return [scanned];
+    }
+
+
+    static async clearCacheKeysByPattern(pattern) {
+        await this.forEachCacheKeyByPattern(pattern, async (keys, client) => {
+            await client.del(keys);
+        });
     }
 
 
@@ -310,9 +378,8 @@ class BaseRecord extends Model {
 
         const pk     = this.constructor.primaryKeyAttribute;
         const key    = this.constructor._recordCacheKey(this[pk]);
-        const client = await this.constructor._getRedisClient();
+        await this.constructor.clearCacheKey(key);
 
-        await client.del(key);
         this.constructor.debugLog('Cache Del', key);
     }
 
@@ -330,18 +397,6 @@ class BaseRecord extends Model {
     }
 
 
-    static get redisClientFactory() {
-        return redisClientFactory;
-    }
-
-
-    static async _getRedisClient() {
-      const client = this.redisClientFactory();
-      await client.connect();
-      return client;
-    }
-
-
     static get isCacheable() { return false; }
 
 
@@ -357,6 +412,45 @@ class BaseRecord extends Model {
     }
 
 
+    static _marshallDataForCaching(value) {
+      let marshalledValue;
+      if (value instanceof this) {
+          marshalledValue = JSON.stringify(value.toJSON());
+          marshalledValue = `${this.recordCachingPrefix}:${marshalledValue}`;
+      } else {
+          marshalledValue = JSON.stringify(value);
+      }
+      return marshalledValue;
+    }
+
+
+    static _unmarshallCachedData(value) {
+        if (value == null)            return null;
+        if (typeof value != 'string') return value;
+
+        if (value.startsWith(this.recordCachingPrefix)) {
+            const regExp = new RegExp(`^${this.recordCachingPrefix}:`);
+            value        = value.replace(regExp, '');
+            const data   = JSON.parse(value);
+            return new this(data, {isNewRecord: false});
+        } else {
+            return JSON.parse(value);
+        }
+    }
+
+
+    static get _redisClientFactory() {
+        return redisClientFactory;
+    }
+
+
+    static async _getRedisClient() {
+      const client = this._redisClientFactory();
+      await client.connect();
+      return client;
+    }
+
+
     /***********************************************************************************************
     * MISC
     ***********************************************************************************************/
@@ -376,17 +470,33 @@ class BaseRecord extends Model {
     toApiResponse() {
       const response = {};
       for (const column of this.apiResponseColumns) {
-        response[column] = this[column]; // Use overridden getters when available
+        response[this.apiResponseKey(column)] = this[column]; // Use overridden getters when available
       }
       return response;
     }
 
 
+    toObject() {
+      return this.toApiResponse();
+    }
+
+
     get apiResponseColumns() {
       return Object.keys(this.constructor.documentationSchema());
     }
 
 
+    apiResponseKey(column) {
+      if (!this.camelizeFieldsForApiResponse) return column;
+      return camelize(column, true);
+    }
+
+
+    get camelizeFieldsForApiResponse() {
+      return false;
+    }
+
+
     /***********************************************************************************************
     * DOCUMENTATION
     ***********************************************************************************************/

package/lib/activeRecord/db/migrator.js

@@ -72,6 +72,9 @@ class Migrator {
         if (options.timestamps !== false) {
             details = {...details, ...this.defaultTimestampColumns};
         }
+        if (options.paranoid) {
+            details = {...details, ...this.deletedAtColumn};
+        }
 
         details = this.normalizeTableDetails(details);
         await this.#sequelizeQueryInterface.createTable(tableName, details);
@@ -113,6 +116,17 @@ class Migrator {
     }
 
 
+    // Records use soft deletes when the table has this column: initializationOptions turns paranoid
+    // on for any model whose schema includes it.
+    get deletedAtColumn() {
+        return {
+            deleted_at: {
+                allowNull: true,
+                type: this.DataTypes.DATE }
+        };
+    }
+
+
     async dropTable(tableName) {
         await this.#sequelizeQueryInterface.dropTable(tableName);
     }

package/lib/activeRecord/db/queryInterface.js

@@ -25,7 +25,7 @@ class QueryInterface {
 
 
     async transaction(callback) {
-        await this.#sequelize.transaction(callback); 
+        return await this.#sequelize.transaction(callback); 
     }
 
 

package/lib/cache/memoryCache.js

@@ -14,6 +14,16 @@ class MemoryCache {
     }
 
 
+    async delete({ namespace, key } = {}) {
+        this.#validateNamespace(namespace);
+
+        const cacheKey         = this.#createCacheKey({ namespace, key });
+        const entryDeleted     = this.#entries.delete(cacheKey);
+        const fetchInvalidated = this.#removePendingFetch(cacheKey);
+        return entryDeleted || fetchInvalidated;
+    }
+
+
     async fetch(options = {}, fetcher) {
         this.#validateNamespace(options.namespace);
         this.#validateTtl(options.ttlSeconds);
@@ -26,13 +36,15 @@ class MemoryCache {
         const pending  = this.#pendingFetches.get(cacheKey);
         if (pending) return this.#buildFetchResult(await pending, false);
 
-        const fetch = this.#fetch(options, fetcher, cacheKey);
+        const fetch = this.#runFetcher(fetcher);
         this.#pendingFetches.set(cacheKey, fetch);
 
         try {
-            return this.#buildFetchResult(await fetch, false);
+            const value = await fetch;
+            this.#cacheFetchedValue(cacheKey, options.ttlSeconds, fetch, value);
+            return this.#buildFetchResult(value, false);
         } finally {
-            this.#pendingFetches.delete(cacheKey);
+            this.#removePendingFetch(cacheKey, fetch);
         }
     }
 
@@ -46,10 +58,22 @@ class MemoryCache {
     }
 
 
-    async #fetch(options, fetcher, cacheKey) {
-        const value = await fetcher();
-        this.#set(cacheKey, options.ttlSeconds, value);
-        return value;
+    async #runFetcher(fetcher) {
+        return fetcher();
+    }
+
+
+    #cacheFetchedValue(cacheKey, ttlSeconds, fetch, value) {
+        if (this.#pendingFetches.get(cacheKey) !== fetch) return;
+        this.#set(cacheKey, ttlSeconds, value);
+    }
+
+
+    #removePendingFetch(cacheKey, expectedFetch) {
+        const currentFetch = this.#pendingFetches.get(cacheKey);
+        if (expectedFetch !== undefined && currentFetch !== expectedFetch) return false;
+
+        return this.#pendingFetches.delete(cacheKey);
     }
 
 

package/lib/server/controllerMixins/renderer.js

@@ -51,6 +51,9 @@ function setSseHeaders(controller) {
 
 
 const InstanceMethods = {
+    MIME_TYPES: CONTENT_TYPE_BY_EXTENSION,
+
+
     async render(body, options={}) {
         if (this.rendered)    return;
         if (this.isStreaming) return;
@@ -76,21 +79,21 @@ const InstanceMethods = {
 
 
     renderJSONResponse(body) {
-        this.setHeader('content-type', CONTENT_TYPE_BY_EXTENSION.json);
+        if (!this.responseContentType) this.responseContentType = CONTENT_TYPE_BY_EXTENSION.json;
         this._response.json(body);
     },
 
 
     renderXMLResponse(body) {
-        this.setHeader('content-type', CONTENT_TYPE_BY_EXTENSION.xml);
-        const doc = create({ response: body });
+        if (!this.responseContentType) this.responseContentType = CONTENT_TYPE_BY_EXTENSION.xml;
+        const doc = create({ [this.xmlRootElement]: body });
         const xml = doc.end();
         this._response.send(xml);
     },
 
 
     renderTextResponse(body) {
-        this.setHeader('content-type', CONTENT_TYPE_BY_EXTENSION.txt);
+        if (!this.responseContentType) this.responseContentType = CONTENT_TYPE_BY_EXTENSION.txt;
         this._response.send(body);
     },
 
@@ -163,7 +166,7 @@ const InstanceMethods = {
 
 
     _serializeDateForResponseBody(date, options) {
-        return date.getTime();
+        return Math.floor(date.getTime() / 1000);
     },
 
 
@@ -186,33 +189,6 @@ const InstanceMethods = {
     },
 
 
-    _setupStatusRenderers() {
-        const successStatuses = {
-            CreationSuccessful: 201,
-            Accepted:           202,
-            NoConent:           204,
-            MovedPermanently:   301,
-            Found:              302,
-            NotModified:        304,
-            TemporaryRedirect:  307,
-            PermanentRedirect:  308,
-        }
-
-        for (let [type, statusCode] of Object.entries(Errors.Statuses)) {
-            if (type == 'Server') type = 'ServerError';
-            this[`render${type}`] = async function(message=null, body=null) {
-                return await this._renderError(statusCode, message, body);
-            }
-        }
-
-        for (let [type, statusCode] of Object.entries(successStatuses)) {
-            this[`render${type}`] = async function(body=null) {
-                return await this._renderSuccess(statusCode, body);
-            }
-        }
-    },
-
-
     async _renderError(statusCode, message, body) {
         this.statusCode     = statusCode;
         body                = structuredClone(body || {});
@@ -305,7 +281,37 @@ const InstanceMethods = {
 }
 
 
+const StatusRenderSuccessStatuses = {
+    CreationSuccessful: 201,
+    Accepted:           202,
+    NoContent:          204,
+    MovedPermanently:   301,
+    Found:              302,
+    NotModified:        304,
+    TemporaryRedirect:  307,
+    PermanentRedirect:  308,
+}
+
+
 const StaticMethods = {
+    // Define render<Status> helpers on the prototype (not per instance) so subclasses and mixins
+    // can override them with a normal method of the same name.
+    setupStatusRenderers() {
+        for (let [type, statusCode] of Object.entries(Errors.Statuses)) {
+            if (type == 'Server') type = 'ServerError';
+            this.prototype[`render${type}`] = async function(message=null, body=null) {
+                return await this._renderError(statusCode, message, body);
+            }
+        }
+
+        for (const [type, statusCode] of Object.entries(StatusRenderSuccessStatuses)) {
+            this.prototype[`render${type}`] = async function(body=null) {
+                return await this._renderSuccess(statusCode, body);
+            }
+        }
+    },
+
+
     formatResponseBody(body, errors, { statusCode, statusText }) {
         const response = {data: body, status: statusText};
         if (errors) response.errors = errors;
@@ -325,7 +331,6 @@ const Accessors = {
         get() {
             const override = `${this.params._format || ''}`.toLowerCase();
             if (override === 'xml' || override === 'json') return override;
-            if (/\bxml\b/i.test(this.requestHeaders.accept || '')) return 'xml';
             return 'json';
         }
     },
@@ -341,6 +346,19 @@ const Accessors = {
     },
 
 
+    responseContentType: {
+        get()     { return this._response.getHeader('content-type') },
+        set(type) { return this._response.setHeader('content-type', type) }
+    },
+
+
+    // Root element wrapped around XML response bodies. Override in a subclass to emit a custom
+    // root (e.g. a TwiML `<Response>` for a telecom webhook).
+    xmlRootElement: {
+        get() { return 'response' }
+    },
+
+
     statusText: {
         get() {
             return STATUS_TEXTS[this.statusCode];

package/lib/server/doc/renderer.md

@@ -47,7 +47,7 @@ Don't set `this.statusCode` directly — use the helper that pairs the status wi
 |---|---|
 | 201 Created | `this.renderCreationSuccessful(body)` |
 | 202 Accepted | `this.renderAccepted(body)` |
-| 204 No Content | `this.renderNoConent(body)` |
+| 204 No Content | `this.renderNoContent(body)` |
 | 301 Moved Permanently | `this.renderMovedPermanently(body)` |
 | 302 Found | `this.renderFound(body)` |
 | 304 Not Modified | `this.renderNotModified(body)` |

package/lib/server/serverController.js

@@ -23,7 +23,6 @@ class VidaServerController {
 
         this._applyCallbacks();
         this._applyDocumentation();
-        this._setupStatusRenderers();
     }
 
 
@@ -260,6 +259,8 @@ VidaServerController.loadMixin(RendererMixin);
 VidaServerController.loadMixin(RoutingMixin);
 VidaServerController.loadMixin(ValidationMixin);
 
+VidaServerController.setupStatusRenderers();
+
 
 /***************************************************************************************************
 * ERRORS

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vida-global/core",
-    "version": "2.1.1",
+    "version": "2.2.0",
     "description": "Core libraries for supporting Vida development",
     "author": "",
     "license": "ISC",