@vida-global/core 2.1.2 → 2.2.0

package/lib/activeRecord/README.md

@@ -98,7 +98,7 @@ module.exports = {
 Available migration helpers:
 
 ```js
-createTable(tableName, details, { timestamps })
+createTable(tableName, details, { timestamps, paranoid })
 dropTable(tableName)
 addColumn(tableName, columnName, columnDetails)
 removeColumn(tableName, columnName)
@@ -111,6 +111,7 @@ changeColumn(tableName, columnName, dataTypeOrOptions)
 **Conventions:**
 
 - `createTable` adds an auto-increment `id` primary key and `created_at` / `updated_at` columns by default. Pass `{ timestamps: false }` to skip the timestamps.
+- Pass `{ paranoid: true }` to add a nullable `deleted_at` column. Models over that table soft delete: `initializationOptions` turns on `paranoid` for any table whose schema has `deleted_at`, so `destroy()` sets the column and ordinary queries skip the row. Unique indexes on a paranoid table should be partial — `where: { deleted_at: null }` — or a deleted row will still block re-creating the same one.
 - Column names are automatically converted to snake_case, so `teamId` in the details object becomes `team_id` in the database.
 - Each migration runs inside a transaction.
 
@@ -307,8 +308,3 @@ class User extends ActiveRecord.BaseRecord {
 ```
 
 See [Sequelize validators](https://sequelize.org/docs/v6/core-concepts/validations-and-constraints/).
-
-
-## TODO
-
-- **Paranoid (soft delete).** The infrastructure for a `deleted_at` column is wired in `initializationOptions`, but `paranoid: true` is not set, so soft deletes are not yet usable out of the box.

package/lib/activeRecord/baseRecord.js

@@ -337,30 +337,39 @@ class BaseRecord extends Model {
     }
 
 
-    static _marshallDataForCaching(value) {
-      let marshalledValue;
-      if (value instanceof this) {
-          marshalledValue = JSON.stringify(value.toJSON());
-          marshalledValue = `${this.recordCachingPrefix}:${marshalledValue}`;
-      } else {
-          marshalledValue = JSON.stringify(value);
-      }
-      return marshalledValue;
+    static async clearCacheKey(key) {
+        const client = await this._getRedisClient();
+        await client.del(key);
     }
 
 
-    static _unmarshallCachedData(value) {
-        if (value == null)            return null;
-        if (typeof value != 'string') return value;
+    static async forEachCacheKeyByPattern(pattern, callback, count=100) {
+        const client = await this._getRedisClient();
+        let   batch  = [];
 
-        if (value.startsWith(this.recordCachingPrefix)) {
-            const regExp = new RegExp(`^${this.recordCachingPrefix}:`);
-            value        = value.replace(regExp, '');
-            const data   = JSON.parse(value);
-            return new this(data, {isNewRecord: false});
-        } else {
-            return JSON.parse(value);
+        for await (const scanned of client.scanIterator({ MATCH: pattern, COUNT: count })) {
+            batch.push(...this._scannedKeysAsArray(scanned));
+            if (batch.length < count) continue;
+
+            await callback(batch, client);
+            batch = [];
         }
+
+        if (batch.length) await callback(batch, client);
+    }
+
+
+    // redis 4 yields one key at a time; redis 5 yields an array of keys per scan.
+    static _scannedKeysAsArray(scanned) {
+        if (Array.isArray(scanned)) return scanned;
+        return [scanned];
+    }
+
+
+    static async clearCacheKeysByPattern(pattern) {
+        await this.forEachCacheKeyByPattern(pattern, async (keys, client) => {
+            await client.del(keys);
+        });
     }
 
 
@@ -369,9 +378,8 @@ class BaseRecord extends Model {
 
         const pk     = this.constructor.primaryKeyAttribute;
         const key    = this.constructor._recordCacheKey(this[pk]);
-        const client = await this.constructor._getRedisClient();
+        await this.constructor.clearCacheKey(key);
 
-        await client.del(key);
         this.constructor.debugLog('Cache Del', key);
     }
 
@@ -389,18 +397,6 @@ class BaseRecord extends Model {
     }
 
 
-    static get redisClientFactory() {
-        return redisClientFactory;
-    }
-
-
-    static async _getRedisClient() {
-      const client = this.redisClientFactory();
-      await client.connect();
-      return client;
-    }
-
-
     static get isCacheable() { return false; }
 
 
@@ -416,6 +412,45 @@ class BaseRecord extends Model {
     }
 
 
+    static _marshallDataForCaching(value) {
+      let marshalledValue;
+      if (value instanceof this) {
+          marshalledValue = JSON.stringify(value.toJSON());
+          marshalledValue = `${this.recordCachingPrefix}:${marshalledValue}`;
+      } else {
+          marshalledValue = JSON.stringify(value);
+      }
+      return marshalledValue;
+    }
+
+
+    static _unmarshallCachedData(value) {
+        if (value == null)            return null;
+        if (typeof value != 'string') return value;
+
+        if (value.startsWith(this.recordCachingPrefix)) {
+            const regExp = new RegExp(`^${this.recordCachingPrefix}:`);
+            value        = value.replace(regExp, '');
+            const data   = JSON.parse(value);
+            return new this(data, {isNewRecord: false});
+        } else {
+            return JSON.parse(value);
+        }
+    }
+
+
+    static get _redisClientFactory() {
+        return redisClientFactory;
+    }
+
+
+    static async _getRedisClient() {
+      const client = this._redisClientFactory();
+      await client.connect();
+      return client;
+    }
+
+
     /***********************************************************************************************
     * MISC
     ***********************************************************************************************/

package/lib/activeRecord/db/migrator.js

@@ -72,6 +72,9 @@ class Migrator {
         if (options.timestamps !== false) {
             details = {...details, ...this.defaultTimestampColumns};
         }
+        if (options.paranoid) {
+            details = {...details, ...this.deletedAtColumn};
+        }
 
         details = this.normalizeTableDetails(details);
         await this.#sequelizeQueryInterface.createTable(tableName, details);
@@ -113,6 +116,17 @@ class Migrator {
     }
 
 
+    // Records use soft deletes when the table has this column: initializationOptions turns paranoid
+    // on for any model whose schema includes it.
+    get deletedAtColumn() {
+        return {
+            deleted_at: {
+                allowNull: true,
+                type: this.DataTypes.DATE }
+        };
+    }
+
+
     async dropTable(tableName) {
         await this.#sequelizeQueryInterface.dropTable(tableName);
     }

package/lib/cache/memoryCache.js

@@ -14,6 +14,16 @@ class MemoryCache {
     }
 
 
+    async delete({ namespace, key } = {}) {
+        this.#validateNamespace(namespace);
+
+        const cacheKey         = this.#createCacheKey({ namespace, key });
+        const entryDeleted     = this.#entries.delete(cacheKey);
+        const fetchInvalidated = this.#removePendingFetch(cacheKey);
+        return entryDeleted || fetchInvalidated;
+    }
+
+
     async fetch(options = {}, fetcher) {
         this.#validateNamespace(options.namespace);
         this.#validateTtl(options.ttlSeconds);
@@ -26,13 +36,15 @@ class MemoryCache {
         const pending  = this.#pendingFetches.get(cacheKey);
         if (pending) return this.#buildFetchResult(await pending, false);
 
-        const fetch = this.#fetch(options, fetcher, cacheKey);
+        const fetch = this.#runFetcher(fetcher);
         this.#pendingFetches.set(cacheKey, fetch);
 
         try {
-            return this.#buildFetchResult(await fetch, false);
+            const value = await fetch;
+            this.#cacheFetchedValue(cacheKey, options.ttlSeconds, fetch, value);
+            return this.#buildFetchResult(value, false);
         } finally {
-            this.#pendingFetches.delete(cacheKey);
+            this.#removePendingFetch(cacheKey, fetch);
         }
     }
 
@@ -46,10 +58,22 @@ class MemoryCache {
     }
 
 
-    async #fetch(options, fetcher, cacheKey) {
-        const value = await fetcher();
-        this.#set(cacheKey, options.ttlSeconds, value);
-        return value;
+    async #runFetcher(fetcher) {
+        return fetcher();
+    }
+
+
+    #cacheFetchedValue(cacheKey, ttlSeconds, fetch, value) {
+        if (this.#pendingFetches.get(cacheKey) !== fetch) return;
+        this.#set(cacheKey, ttlSeconds, value);
+    }
+
+
+    #removePendingFetch(cacheKey, expectedFetch) {
+        const currentFetch = this.#pendingFetches.get(cacheKey);
+        if (expectedFetch !== undefined && currentFetch !== expectedFetch) return false;
+
+        return this.#pendingFetches.delete(cacheKey);
     }
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vida-global/core",
-    "version": "2.1.2",
+    "version": "2.2.0",
     "description": "Core libraries for supporting Vida development",
     "author": "",
     "license": "ISC",

package/test/activeRecord/baseRecord.test.js

@@ -3,7 +3,7 @@ const { Connection }            = require('../../lib/activeRecord/db/connection'
 const { getActiveRecordSchema } = require('../../lib/activeRecord/db/schema');
 const   importSchema            = require('../../lib/activeRecord/db/importSchema');
 const { Model, Sequelize }      = require('sequelize');
-const { mockRedisClientFactory } = require('./helpers/baseRecordMocks');
+const { mockRedisClientFactory, mockScannedKeys, randomCacheKeys } = require('./helpers/baseRecordMocks');
 const   TestHelpers             = require('@vida-global/test-helpers');
 const { camelize }              = require('inflection');
 const   Helpers                 = require('./helpers/baseRecord');
@@ -503,7 +503,7 @@ describe('BaseRecord', () => {
     describe('Caching', () => {
         class Person extends BaseRecord {
             static get isCacheable()       { return true; }
-            static get redisClientFactory() { return mockRedisClientFactory; }
+            static get _redisClientFactory() { return mockRedisClientFactory; }
         }
         let redisClient
         beforeEach(() => {
@@ -772,6 +772,116 @@ describe('BaseRecord', () => {
         });
 
 
+        describe('BaseRecord.clearCacheKey', () => {
+            it ('deletes the given key from cache', async () => {
+                const key = `Person#find:${Math.random()}`;
+
+                await Person.clearCacheKey(key);
+                expect(redisClient.del).toHaveBeenCalledTimes(1);
+                expect(redisClient.del).toHaveBeenCalledWith(key);
+            });
+        });
+
+
+        describe('BaseRecord.forEachCacheKeyByPattern', () => {
+            const pattern = `Person#find:*`;
+
+            it ('scans with the given pattern and a default count of 100', async () => {
+                mockScannedKeys([]);
+
+                await Person.forEachCacheKeyByPattern(pattern, jest.fn());
+
+                expect(redisClient.scanIterator).toHaveBeenCalledTimes(1);
+                expect(redisClient.scanIterator).toHaveBeenCalledWith({ MATCH: pattern, COUNT: 100 });
+            });
+
+            it ('passes a custom count to the scan', async () => {
+                const count = TestHelpers.Faker.Math.randomNumber();
+                mockScannedKeys([]);
+
+                await Person.forEachCacheKeyByPattern(pattern, jest.fn(), count);
+
+                expect(redisClient.scanIterator).toHaveBeenCalledWith({ MATCH: pattern, COUNT: count });
+            });
+
+            it ('buffers the individually yielded keys into one batch per count', async () => {
+                const keys     = randomCacheKeys(5);
+                const callback = jest.fn();
+                mockScannedKeys(keys);
+
+                await Person.forEachCacheKeyByPattern(pattern, callback, 2);
+
+                expect(callback).toHaveBeenCalledTimes(3);
+                expect(callback).toHaveBeenNthCalledWith(1, keys.slice(0, 2), redisClient);
+                expect(callback).toHaveBeenNthCalledWith(2, keys.slice(2, 4), redisClient);
+                expect(callback).toHaveBeenNthCalledWith(3, keys.slice(4), redisClient);
+            });
+
+            it ('invokes the callback once when fewer keys than the count are scanned', async () => {
+                const keys     = randomCacheKeys(2);
+                const callback = jest.fn();
+                mockScannedKeys(keys);
+
+                await Person.forEachCacheKeyByPattern(pattern, callback);
+
+                expect(callback).toHaveBeenCalledTimes(1);
+                expect(callback).toHaveBeenCalledWith(keys, redisClient);
+            });
+
+            it ('flattens keys yielded as arrays', async () => {
+                const keys     = randomCacheKeys(3);
+                const callback = jest.fn();
+                mockScannedKeys([keys.slice(0, 2), keys.slice(2)]);
+
+                await Person.forEachCacheKeyByPattern(pattern, callback);
+
+                expect(callback).toHaveBeenCalledTimes(1);
+                expect(callback).toHaveBeenCalledWith(keys, redisClient);
+            });
+
+            it ('does not invoke the callback when the scan yields nothing', async () => {
+                const callback = jest.fn();
+                mockScannedKeys([]);
+
+                await Person.forEachCacheKeyByPattern(pattern, callback);
+
+                expect(callback).not.toHaveBeenCalled();
+            });
+        });
+
+
+        describe('BaseRecord.clearCacheKeysByPattern', () => {
+            const pattern = `Person#find:*`;
+
+            it ('scans for matching keys with the given pattern', async () => {
+                mockScannedKeys([]);
+
+                await Person.clearCacheKeysByPattern(pattern);
+
+                expect(redisClient.scanIterator).toHaveBeenCalledTimes(1);
+                expect(redisClient.scanIterator).toHaveBeenCalledWith({ MATCH: pattern, COUNT: 100 });
+            });
+
+            it ('deletes all scanned keys in a single batched call', async () => {
+                const keys = randomCacheKeys(3);
+                mockScannedKeys(keys);
+
+                await Person.clearCacheKeysByPattern(pattern);
+
+                expect(redisClient.del).toHaveBeenCalledTimes(1);
+                expect(redisClient.del).toHaveBeenCalledWith(keys);
+            });
+
+            it ('does not delete when the scan yields nothing', async () => {
+                mockScannedKeys([]);
+
+                await Person.clearCacheKeysByPattern(pattern);
+
+                expect(redisClient.del).not.toHaveBeenCalled();
+            });
+        });
+
+
         describe('BaseRecord.#clearSelfCache', () => {
             it ('deletes the key from cache', async () => {
                 const pk     = Person.primaryKeyAttribute;
@@ -783,6 +893,18 @@ describe('BaseRecord', () => {
                 expect(redisClient.del).toHaveBeenCalledTimes(1);
                 expect(redisClient.del).toHaveBeenCalledWith(key);
             });
+
+            it ('delegates deletion to clearCacheKey', async () => {
+                const pk       = Person.primaryKeyAttribute;
+                const id       = Math.random();
+                const key      = Person._recordCacheKey(id);
+                const person   = new Person({[pk]: id});
+                const clearSpy = jest.spyOn(Person, 'clearCacheKey');
+
+                await person.clearSelfCache();
+                expect(clearSpy).toHaveBeenCalledTimes(1);
+                expect(clearSpy).toHaveBeenCalledWith(key);
+            });
         });
 
 
@@ -790,7 +912,7 @@ describe('BaseRecord', () => {
         describe.each(hooks)('BaseRecord._after%sCacheHook', (hook) => {
             class Cacheable extends BaseRecord {
                 static get isCacheable()       { return true; }
-                static get redisClientFactory() { return mockRedisClientFactory; }
+                static get _redisClientFactory() { return mockRedisClientFactory; }
             }
             const hookMethod = `_after${hook}CacheHook`;
 

package/test/activeRecord/db/migrator.test.js

@@ -61,6 +61,27 @@ describe('Migrator', () => {
             expect(Helpers.mockQueryInterface.createTable).toHaveBeenCalledWith(tableName, details);
         });
 
+        it ('adds a deleted_at column for paranoid tables', async () => {
+            const tableName = Helpers.randomString();
+            const details = { ...defaultColumns };
+
+            await Helpers.migrator.createTable(tableName, details, { paranoid: true });
+
+            expect(Helpers.mockQueryInterface.createTable).toHaveBeenCalledWith(tableName, {
+                ...defaultColumns,
+                deleted_at: { allowNull: true, type: 'DATE' },
+            });
+        });
+
+        it ('leaves deleted_at off tables that are not paranoid', async () => {
+            const tableName = Helpers.randomString();
+            const details = { ...defaultColumns };
+
+            await Helpers.migrator.createTable(tableName, details);
+
+            expect(Helpers.mockQueryInterface.createTable).toHaveBeenCalledWith(tableName, defaultColumns);
+        });
+
         it ('snake_cases created table columns', async () => {
             const tableName = Helpers.randomString();
             const details = { ...defaultColumns, myColumn: 1 };

package/test/activeRecord/helpers/baseRecordMocks.js

@@ -10,6 +10,11 @@ const dataTypeKey2   = TestHelpers.Faker.Text.randomString();
 const dataTypeValue2 = TestHelpers.Faker.Text.randomString();
 
 
+function randomCacheKeys(count) {
+    return Array.from({ length: count }, () => `Person#find:${TestHelpers.Faker.Text.randomString()}`);
+}
+
+
 /***************************************************************************************************
 * MOCKS
 ***************************************************************************************************/
@@ -42,14 +47,26 @@ class MockSequelize {
 
 
 const redisClient = {
-    del:     jest.fn(),
-    mGet:    jest.fn(),
-    mSet:    jest.fn(),
-    connect: jest.fn(),
+    del:          jest.fn(),
+    mGet:         jest.fn(),
+    mSet:         jest.fn(),
+    connect:      jest.fn(),
+    scanIterator: jest.fn(),
 }
 const mockRedisClientFactory = () => redisClient;
 
 
+// redis 4's scanIterator yields one key string at a time. Pass an array element to model the
+// batches redis 5 yields instead.
+function mockScannedKeys(scanned) {
+    redisClient.scanIterator.mockImplementation(async function* () {
+        for (const item of scanned) {
+            yield item;
+        }
+    });
+}
+
+
 module.exports = {
     dataTypeKey1,
     dataTypeValue1,
@@ -57,5 +74,7 @@ module.exports = {
     dataTypeValue2,
     MockModel,
     mockRedisClientFactory,
+    mockScannedKeys,
     MockSequelize,
+    randomCacheKeys,
 }

package/test/cache/memoryCache.test.js

@@ -16,6 +16,82 @@ describe('MemoryCache', () => {
     });
 
 
+    describe('#delete', () => {
+        const options = {
+            namespace: 'squire:shop-data',
+            key: { shopId: 'shop-1' },
+            ttlSeconds: 300,
+        };
+
+
+        it('deletes a cached value', async () => {
+            await cache.fetch(options, async () => ({ version: 1 }));
+
+            await expect(cache.delete(options)).resolves.toBe(true);
+            await expect(cache.fetch(options, async () => ({ version: 2 }))).resolves.toEqual({
+                value: { version: 2 },
+                cacheHit: false,
+            });
+        });
+
+
+        it('deletes values using an equivalent object key', async () => {
+            await cache.fetch({
+                ...options,
+                key: { agentId: 'agent-1', shopId: 'shop-1' },
+            }, async () => ({ version: 1 }));
+
+            await expect(cache.delete({
+                namespace: options.namespace,
+                key: { shopId: 'shop-1', agentId: 'agent-1' },
+            })).resolves.toBe(true);
+        });
+
+
+        it('returns false when the value is not cached', async () => {
+            await expect(cache.delete(options)).resolves.toBe(false);
+        });
+
+
+        it('invalidates an in-flight fetch without interfering with its replacement', async () => {
+            let resolveOldFetch;
+            let resolveNewFetch;
+            const oldFetcher = jest.fn(() => new Promise((resolve) => {
+                resolveOldFetch = resolve;
+            }));
+            const newFetcher = jest.fn(() => new Promise((resolve) => {
+                resolveNewFetch = resolve;
+            }));
+            const oldFetch = cache.fetch(options, oldFetcher);
+            await expect(cache.delete(options)).resolves.toBe(true);
+            const newFetch = cache.fetch(options, newFetcher);
+
+            resolveOldFetch({ version: 1 });
+            await expect(oldFetch).resolves.toEqual({ value: { version: 1 }, cacheHit: false });
+            const joinedFetcher = jest.fn();
+            const joinedFetch   = cache.fetch(options, joinedFetcher);
+            resolveNewFetch({ version: 2 });
+
+            await expect(Promise.all([newFetch, joinedFetch])).resolves.toEqual([
+                { value: { version: 2 }, cacheHit: false },
+                { value: { version: 2 }, cacheHit: false },
+            ]);
+            expect(oldFetcher).toHaveBeenCalledTimes(1);
+            expect(newFetcher).toHaveBeenCalledTimes(1);
+            expect(joinedFetcher).not.toHaveBeenCalled();
+            await expect(cache.fetch(options, joinedFetcher)).resolves.toEqual({
+                value: { version: 2 },
+                cacheHit: true,
+            });
+        });
+
+
+        it('requires a namespace', async () => {
+            await expect(cache.delete()).rejects.toThrow('MemoryCache namespace is required.');
+        });
+    });
+
+
     describe('#fetch', () => {
         const options = {
             namespace: 'squire:shop-data',