@vida-global/core 2.4.4 → 2.5.0

package/index.js

@@ -1,7 +1,7 @@
 const   ActiveRecord      = require('./lib/activeRecord');
 const   httpLibs          = require('./lib/http/client');
 const   APM               = require('./lib/apm');
-const { AppDependencies } = require('./lib/appDependencies');
+const { AppDependencies, DependencyRegistry } = require('./lib/appDependencies');
 const   JobQueue          = require('./lib/jobQueue');
 const { logger }          = require('./lib/logger');
 const   redisLibs         = require('./lib/redis');
@@ -29,6 +29,7 @@ module.exports = {
     ...serverErrorsToExport,
     ApiDocsGenerator,
     AppDependencies,
+    DependencyRegistry,
     VidaServer,
     VidaServerController,
 };

package/lib/activeRecord/README.md

@@ -113,7 +113,7 @@ dropTable(tableName)
 addColumn(tableName, columnName, columnDetails)
 removeColumn(tableName, columnName)
 addIndex(tableName, fields, { concurrently, unique, name, using, where })
-removeIndex(tableName, indexNameOrAttributes, concurrently=false)
+removeIndex(tableName, indexNameOrAttributes, { concurrently })
 renameColumn(tableName, oldName, newName)
 changeColumn(tableName, columnName, dataTypeOrOptions)
 ```
@@ -141,13 +141,15 @@ module.exports = {
     },
 
     down: async function() {
-        await this.removeIndex('users', ['team_id'], true);
+        await this.removeIndex('users', ['team_id'], { concurrently: true });
     }
 }
 ```
 
 An opted-out migration is **not** rolled back if it fails, so keep it to the single statement that needs it and put everything else in its own migration.
 
+`concurrently` is off by default on both methods, because the default transaction rules it out. Asking for it from inside a transaction throws a `MigrationError` before the statement reaches the database, and the message tells you to set `transactional: false`.
+
 
 ## Models
 

package/lib/activeRecord/db/migrationErrors.js

@@ -0,0 +1,6 @@
+class MigrationError extends Error {}
+
+
+module.exports = {
+    MigrationError
+};

package/lib/activeRecord/db/migrator.js

@@ -1,7 +1,8 @@
-const { Connection }    = require('./connection');
-const   utils           = require('../utils');
-const { Sequelize, Op } = require('sequelize');
-const { underscore }    = require('inflection');
+const { Connection }     = require('./connection');
+const { MigrationError } = require('./migrationErrors');
+const   utils            = require('../utils');
+const { Sequelize, Op }  = require('sequelize');
+const { underscore }     = require('inflection');
 
 const ID_TYPES        = ['bigint', 'int', 'uuid'];
 const DEFAULT_ID_TYPE = 'bigint';
@@ -24,7 +25,7 @@ class Migrator {
 
 
     async run() {
-        if (!this.migrationModule.up) throw new Error('Missing migration `up`');
+        if (!this.migrationModule.up) throw new MigrationError('Missing migration `up`');
 
         await this.perform(() => this.migrationModule.up.call(this));
     }
@@ -122,8 +123,8 @@ class Migrator {
         if (idType === false) return {};
 
         if (!ID_TYPES.includes(idType)) {
-            throw new Error(`Unknown id type "${idType}". Use one of: ${ID_TYPES.join(', ')}, or `
-                          + `false for a table that declares its own primary key.`);
+            throw new MigrationError(`Unknown id type "${idType}". Use one of: ${ID_TYPES.join(', ')}, `
+                                   + `or false for a table that declares its own primary key.`);
         }
 
         return this[`${idType}IdColumn`];
@@ -213,8 +214,10 @@ class Migrator {
 
 
     async addIndex(tableName, fields, { concurrently, unique, name, using, where }={}) {
+        if (concurrently) this.assertConcurrentIndexAllowed('addIndex');
+
         const options = {fields: fields, ...this.queryOptions};
-        options.name = name || this.defaultIndexName(tableName, fields);
+        options.name  = name || this.defaultIndexName(tableName, fields);
         if (concurrently) options.concurrently = true;
         if (unique)       options.unique       = true;
         if (using)        options.using        = using;
@@ -224,8 +227,10 @@ class Migrator {
     }
 
 
-    async removeIndex(tableName, indexNameOrAttributes, concurrently=false) {
-        const options   = this.queryOptions;
+    async removeIndex(tableName, indexNameOrAttributes, { concurrently }={}) {
+        if (concurrently) this.assertConcurrentIndexAllowed('removeIndex');
+
+        const options = this.queryOptions;
         if (concurrently) options.concurrently = true;
 
         const indexName = this.indexNameFor(tableName, indexNameOrAttributes);
@@ -233,6 +238,18 @@ class Migrator {
     }
 
 
+    /***********************************************************************************************
+    * INDEX GUARDS
+    ***********************************************************************************************/
+    assertConcurrentIndexAllowed(methodName) {
+        if (!this.#transaction) return;
+
+        throw new MigrationError(`${methodName} cannot use \`concurrently\` inside a transaction. `
+                               + 'Add `transactional: false` to the migration module, and keep that '
+                               + 'migration to the statements that need it.');
+    }
+
+
     /***********************************************************************************************
     * INDEX NAMING
     ***********************************************************************************************/

package/lib/appDependencies/index.js

@@ -1,4 +1,4 @@
-class AppDependencies {
+class DependencyRegistry {
     #dependencies    = {};
     #controllerClass = null;
 
@@ -56,5 +56,6 @@ class AppDependencies {
 
 
 module.exports = {
-    AppDependencies: new AppDependencies()
+    AppDependencies: new DependencyRegistry(),
+    DependencyRegistry
 };

package/lib/http/README.md

@@ -49,14 +49,28 @@ const response = await client.getTickets({ page: 1, pageSize: 25 });
 
 | Method | Options |
 |---|---|
-| `get(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal` |
-| `post(endpoint, opts)` | `body`, `headers`, `timeout`, `signal` |
-| `put(endpoint, opts)` | `body`, `headers`, `timeout`, `signal` |
-| `delete(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal` |
+| `get(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `post(endpoint, opts)` | `body`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `put(endpoint, opts)` | `body`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `delete(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal`, `binaryResponse` |
 
 For `GET`/`DELETE`, `requestParams` becomes the query string. For `POST`/`PUT`, `body` becomes the request body. Object bodies are serialized as JSON; string bodies are sent unchanged.
 
 
+### Binary responses
+
+By default a response body is read as text and parsed as JSON when the `Content-Type` says so. That is wrong for media: decoding bytes as UTF-8 replaces every sequence that is not valid UTF-8, so the content arrives corrupted.
+
+Pass `binaryResponse: true` and `data` is a `Buffer` of the exact bytes instead. It is per call, so one client can fetch JSON metadata and binary media.
+
+```js
+async fetchRecording(path) {
+    const response = await this.get(path, { binaryResponse: true, headers: { Accept: 'audio/*' } });
+    return response.data;   // Buffer
+}
+```
+
+
 ### Query string encoding
 
 For `GET`/`DELETE`, the way `requestParams` is appended to the URL depends on the `Content-Type` header:

package/lib/http/client.js

@@ -7,22 +7,22 @@ logger.addScope('http');
 
 
 class HttpClient {
-    async get(endpoint, { requestParams, headers, timeout, signal }={}) {
+    async get(endpoint, { requestParams, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "GET", arguments[1]);
     }
 
 
-    async post(endpoint, { body, headers, timeout, signal }={}) {
+    async post(endpoint, { body, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "POST", arguments[1]);
     }
 
 
-    async put(endpoint, { body, headers, timeout, signal }={}) {
+    async put(endpoint, { body, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "PUT", arguments[1]);
     }
 
 
-    async delete(endpoint, { requestParams, headers, timeout, signal }={}) {
+    async delete(endpoint, { requestParams, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "DELETE", arguments[1]);
     }
 
@@ -32,7 +32,7 @@ class HttpClient {
     }
 
 
-    async #_makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal }={}) {
+    async #_makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal, binaryResponse }={}) {
         this.#logRequest(method, endpoint);
 
         endpoint            = `${this.urlRoot}${endpoint}`;
@@ -42,7 +42,7 @@ class HttpClient {
 
         try {
             const res = await fetch(url.toString(), payload);
-            return await this.#handleRequestResponse(res, payload);
+            return await this.#handleRequestResponse(res, payload, binaryResponse);
         } catch (err) {
             this.#handleRequestError(err, payload, timeout, signal);
         }
@@ -70,9 +70,9 @@ class HttpClient {
     }
 
 
-    async #handleRequestResponse(res, payload) {
+    async #handleRequestResponse(res, payload, binaryResponse) {
         if (res.status && res.status < 300) {
-            const data = await this.#parseResponseData(res);
+            const data = await this.#parseResponseData(res, binaryResponse);
             return {data, status: res.status};
         } else {
             throw new HttpError(res, payload);
@@ -80,7 +80,11 @@ class HttpClient {
     }
 
 
-    async #parseResponseData(res) {
+    async #parseResponseData(res, binaryResponse) {
+        // Reading bytes as text decodes them as UTF-8, which replaces every sequence that is not
+        // valid UTF-8. Media has to be taken as bytes or it arrives corrupted.
+        if (binaryResponse) return Buffer.from(await res.arrayBuffer());
+
         const body = await res.text();
         if (!this.#isJsonResponse(res)) return body;
 

package/lib/jobQueue/README.md

@@ -179,6 +179,7 @@ await queue.getJob(id);       // one job, or null
 
 await queue.clean(grace, limit, type); // remove finished jobs older than `grace` ms
 await queue.clearQueuedJobs();         // drain all queued jobs
+await queue.obliterate();              // delete the queue and every key it owns in Redis
 
 await queue.setMaximumConcurrency(8);  // usually left to the Worker option above
 await queue.maximumConcurrency();
@@ -188,6 +189,9 @@ await queue.removeMaximumConcurrency();
 `clean` takes BullMQ's argument order, which is `(grace, limit, type)`. Bull's was
 `(grace, type, limit)` — worth checking when porting a call.
 
+`obliterate` forces, so it ends jobs that are still running rather than refusing. It leaves nothing
+behind for the queue to be rebuilt from — use it to retire a queue for good, not to empty one.
+
 Each `get*Jobs` and `getJob` result has this shape:
 
 ```js

package/lib/jobQueue/queue.js

@@ -64,6 +64,13 @@ class Queue extends AbstractJobComponent {
     }
 
 
+    // Forced because bullmq refuses to obliterate a queue that still has active jobs. A caller
+    // tearing a queue down wants those gone too.
+    async obliterate() {
+        await this.#queue.obliterate({ force: true });
+    }
+
+
     /***********************************************************************************************
     * LIMITS
     ***********************************************************************************************/

package/lib/server/controllerMixins/validations.js

@@ -38,6 +38,7 @@ const InstanceMethods = {
         const errors = [];
 
         if (parameterValidations?.optional && value === undefined) return errors;
+        if (parameterValidations?.nullable === true && value === null) return errors;
 
         for (const [validationType, validationOptions] of Object.entries(parameterValidations)) {
             const validationMethod = `validate${camelize(validationType)}`;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vida-global/core",
-    "version": "2.4.4",
+    "version": "2.5.0",
     "description": "Core libraries for supporting Vida development",
     "author": "",
     "license": "ISC",

package/test/activeRecord/db/migrator.test.js

@@ -1,4 +1,5 @@
-const Helpers = require('../helpers/migrator');
+const   Helpers          = require('../helpers/migrator');
+const { MigrationError } = require('../../../lib/activeRecord/db/migrationErrors');
 
 
 afterEach(() => {
@@ -422,13 +423,13 @@ describe('Migrator', () => {
 
     describe('Migrator#removeIndex', () => {
         it.each([
-            ['default options', false, { transaction: null }],
-            ['concurrently', true, { concurrently: true, transaction: null }],
-        ])('passes removeIndex options for %s', async (_label, concurrently, expectedOptions) => {
+            ['default options', {}, { transaction: null }],
+            ['concurrently', { concurrently: true }, { concurrently: true, transaction: null }],
+        ])('passes removeIndex options for %s', async (_label, indexOptions, expectedOptions) => {
             const tableName = Helpers.randomString();
             const fields = [Helpers.randomString(), Helpers.randomString()];
 
-            await Helpers.migrator.removeIndex(tableName, fields, concurrently);
+            await Helpers.migrator.removeIndex(tableName, fields, indexOptions);
 
             expect(Helpers.mockQueryInterface.removeIndex).toHaveBeenCalledTimes(1);
             expect(Helpers.mockQueryInterface.removeIndex).toHaveBeenCalledWith(
@@ -453,6 +454,17 @@ describe('Migrator', () => {
     });
 
 
+    describe('Migrator concurrent indexes inside a transaction', () => {
+        it.each([
+            ['addIndex',    async function() { await this.addIndex('users', ['team_id'], { concurrently: true }); }],
+            ['removeIndex', async function() { await this.removeIndex('users', ['team_id'], { concurrently: true }); }],
+        ])('throws when %s asks for concurrently', async (methodName, operation) => {
+            await expect(Helpers.runInMigration(operation)).rejects.toThrow(MigrationError);
+            expect(Helpers.mockQueryInterface[methodName]).not.toHaveBeenCalled();
+        });
+    });
+
+
     describe('Migrator#defaultIndexName', () => {
         it ('builds idx_<table>_<columns> from a single column', () => {
             expect(Helpers.migrator.defaultIndexName('users', ['team_id'])).toBe('idx_users_team_id');

package/test/appDependencies/appDependencies.test.js

@@ -1,4 +1,4 @@
-const { AppDependencies } = require('../../lib/appDependencies');
+const { AppDependencies, DependencyRegistry } = require('../../lib/appDependencies');
 
 
 class FooController {}
@@ -129,3 +129,40 @@ describe('AppDependencies', () => {
         });
     });
 });
+
+
+describe('DependencyRegistry', () => {
+    it ('builds registries that are independent of the AppDependencies singleton', () => {
+        const registry = new DependencyRegistry();
+
+        registry.setDependency('scopedHelper', 'scoped');
+
+        expect(registry.get('scopedHelper')).toBe('scoped');
+        expect(AppDependencies.get('scopedHelper')).toBeNull();
+    });
+
+
+    it ('builds registries that are independent of each other', () => {
+        const first  = new DependencyRegistry();
+        const second = new DependencyRegistry();
+
+        first.setDependency('myHelper', 'one');
+        second.setDependency('myHelper', 'two');
+
+        expect(first.myHelper).toBe('one');
+        expect(second.myHelper).toBe('two');
+    });
+
+
+    it ('resets one registry without touching another', () => {
+        const first  = new DependencyRegistry();
+        const second = new DependencyRegistry();
+        first.setDependency('myHelper', 'one');
+        second.setDependency('myHelper', 'two');
+
+        first.reset();
+
+        expect(first.get('myHelper')).toBeNull();
+        expect(second.get('myHelper')).toBe('two');
+    });
+});

package/test/http/client.test.js

@@ -181,6 +181,48 @@ describe('HttpClient', () => {
     });
 
 
+    describe('binary responses', () => {
+        const client = new HttpClient();
+
+
+        beforeEach(() => Helpers.mockFetchBinary());
+
+
+        it ('returns the exact bytes as a Buffer', async () => {
+            const result = await client.get(Helpers.apiUrl, { binaryResponse: true });
+
+            expect(result.status).toBe(200);
+            expect(Buffer.isBuffer(result.data)).toBe(true);
+            expect(result.data).toEqual(Helpers.binaryPayload);
+        });
+
+
+        // The point of the option: a text read replaces every byte that is not valid UTF-8.
+        it ('never reads the body as text', async () => {
+            await client.get(Helpers.apiUrl, { binaryResponse: true });
+
+            const response = await fetch.mock.results[0].value;
+            expect(response.text).not.toHaveBeenCalled();
+        });
+
+
+        it ('reads the body as text without the option, losing the bytes', async () => {
+            const result = await client.get(Helpers.apiUrl);
+
+            expect(typeof result.data).toBe('string');
+            expect(Buffer.from(result.data)).not.toEqual(Helpers.binaryPayload);
+        });
+
+
+        const verbs = [['post'], ['put'], ['delete']];
+        it.each(verbs)('applies to %s as well', async (methodName) => {
+            const result = await client[methodName](Helpers.apiUrl, { binaryResponse: true });
+
+            expect(result.data).toEqual(Helpers.binaryPayload);
+        });
+    });
+
+
     describe('abort/timeout support', () => {
         const client = new HttpClient();
 

package/test/http/helpers/client.js

@@ -18,6 +18,10 @@ const apiUrl           = `${baseUrl}${endpoint}`;
 const responsePayload  = randomPayload();
 const responseBody     = JSON.stringify(responsePayload);
 
+// A RIFF header followed by byte sequences that are not valid UTF-8, so a text read cannot
+// reproduce them.
+const binaryPayload    = Buffer.from([0x52, 0x49, 0x46, 0x46, 0xff, 0xfe, 0x00, 0x80]);
+
 
 class ClientWithUrlRoot extends HttpClient {
     get urlRoot() {
@@ -67,11 +71,19 @@ const mockFetchResponse = ({ body, contentType, status = 200 }) => {
             get: jest.fn((headerName) => headerName == 'content-type' ? contentType : undefined),
         },
         status,
-        text: jest.fn().mockResolvedValue(body),
+        // Interpolated rather than passed through, so a Buffer body decodes the way a real text
+        // read would.
+        text:        jest.fn().mockResolvedValue(`${body ?? ''}`),
+        arrayBuffer: jest.fn().mockResolvedValue(Buffer.from(body ?? '')),
     });
 }
 
 
+const mockFetchBinary = () => {
+    mockFetchResponse({ body: binaryPayload, contentType: 'audio/wav' });
+}
+
+
 /***************************************************************************************************
 * ASSERTIONS
 ***************************************************************************************************/
@@ -109,11 +121,13 @@ module.exports = {
     apiUrl,
     basicAuthCredentials,
     bearerToken,
+    binaryPayload,
     ClientWithBasicAuth,
     ClientWithBearerToken,
     ClientWithUrlRoot,
     endpoint,
     expectRequest,
+    mockFetchBinary,
     mockFetchResponse,
     mockFetchSuccess,
     randomPayload,

package/test/jobQueue/helpers/bullmqMock.js

@@ -16,6 +16,7 @@ class MockBullQueue {
         this.close                   = jest.fn();
         this.pause                   = jest.fn();
         this.resume                  = jest.fn();
+        this.obliterate              = jest.fn();
         this.getJobCounts            = jest.fn();
         this.getJobs                 = jest.fn();
         this.getActive               = jest.fn();

package/test/jobQueue/queue.test.js

@@ -218,6 +218,17 @@ describe('Queue', () => {
     });
 
 
+    describe('#obliterate', () => {
+        it ('calls bullmq obliterate() with force so active jobs do not block it', async () => {
+            await queue.obliterate();
+
+            const instance = bullmqMock.instances.queues[0];
+            expect(instance.obliterate).toHaveBeenCalledTimes(1);
+            expect(instance.obliterate).toHaveBeenCalledWith({ force: true });
+        });
+    });
+
+
     describe('maximum concurrency', () => {
         it ('sets the queue maximum', async () => {
             const maximum = TestHelpers.Faker.Math.randomNumber(20);