@vida-global/core 2.4.5 → 3.0.0

package/README.md

@@ -9,3 +9,4 @@ This package contains core elements used across all Vida Apps. It is included in
 - [Logger](lib/logger) — scoped, child-aware logger with environment-driven levels.
 - [VidaServer](lib/server) — Express-based server with auto-loaded controllers, validation, rendering, callbacks, cookies, and SSE streaming.
 - [JobQueue](lib/jobQueue) — BullMQ-backed queue with workers, retries/backoff, progress reporting, and queue inspection helpers.
+- [Search](lib/search) — backend-agnostic search client with a chainable query vocabulary, an OpenSearch Serverless implementation, and index mapping management.

package/index.js

@@ -1,17 +1,19 @@
 const   ActiveRecord      = require('./lib/activeRecord');
 const   httpLibs          = require('./lib/http/client');
 const   APM               = require('./lib/apm');
-const { AppDependencies } = require('./lib/appDependencies');
+const { AppDependencies, DependencyRegistry } = require('./lib/appDependencies');
 const   JobQueue          = require('./lib/jobQueue');
 const { logger }          = require('./lib/logger');
 const   redisLibs         = require('./lib/redis');
 const   cacheLibs         = require('./lib/cache');
+const   searchLibs        = require('./lib/search');
 const   serverLibs        = require('./lib/server');
 
 
 const {
   AbstractServerError,
   ApiDocsGenerator,
+  Authorizer,
   VidaServer,
   VidaServerController,
 } = serverLibs;
@@ -21,14 +23,17 @@ const serverErrorsToExport = Object.values(serverLibs).filter(obj => obj.prototy
 module.exports = {
     ActiveRecord,
     APM,
+    Authorizer,
     ...httpLibs,
     JobQueue,
     logger,
     ...cacheLibs,
     ...redisLibs,
+    ...searchLibs,
     ...serverErrorsToExport,
     ApiDocsGenerator,
     AppDependencies,
+    DependencyRegistry,
     VidaServer,
     VidaServerController,
 };

package/lib/activeRecord/README.md

@@ -113,7 +113,7 @@ dropTable(tableName)
 addColumn(tableName, columnName, columnDetails)
 removeColumn(tableName, columnName)
 addIndex(tableName, fields, { concurrently, unique, name, using, where })
-removeIndex(tableName, indexNameOrAttributes, concurrently=false)
+removeIndex(tableName, indexNameOrAttributes, { concurrently })
 renameColumn(tableName, oldName, newName)
 changeColumn(tableName, columnName, dataTypeOrOptions)
 ```
@@ -141,13 +141,15 @@ module.exports = {
     },
 
     down: async function() {
-        await this.removeIndex('users', ['team_id'], true);
+        await this.removeIndex('users', ['team_id'], { concurrently: true });
     }
 }
 ```
 
 An opted-out migration is **not** rolled back if it fails, so keep it to the single statement that needs it and put everything else in its own migration.
 
+`concurrently` is off by default on both methods, because the default transaction rules it out. Asking for it from inside a transaction throws a `MigrationError` before the statement reaches the database, and the message tells you to set `transactional: false`.
+
 
 ## Models
 

package/lib/activeRecord/baseRecord.js

@@ -379,7 +379,7 @@ class BaseRecord extends Model {
     /***********************************************************************************************
     * CACHING
     ***********************************************************************************************/
-    static async _cachedFind(ids, { clear=false }={}) {
+    static async _cachedFind(ids, { clear=false, expireSeconds=null }={}) {
         const multiFind = Array.isArray(ids)
         const idsToFind = multiFind ? ids : [ids];
         const keyPairs  = idsToFind.map(id => [this._recordCacheKey(id), id]);
@@ -387,7 +387,7 @@ class BaseRecord extends Model {
         const idsToKeys = Object.fromEntries(keyPairs.map(p => p.reverse()));
         const keys      = Object.keys(keysToIds);
 
-        let records = await this.cachedFetch(keys, { clear }, async (missedKeys) => {
+        let records = await this.cachedFetch(keys, { clear, expireSeconds }, async (missedKeys) => {
             const idsToFind = missedKeys.map(key => keysToIds[key]);
             const pk        = this.primaryKeyAttribute
             const records   = await this.where({[pk]: idsToFind});

package/lib/activeRecord/db/migrationErrors.js

@@ -0,0 +1,6 @@
+class MigrationError extends Error {}
+
+
+module.exports = {
+    MigrationError
+};

package/lib/activeRecord/db/migrator.js

@@ -1,7 +1,8 @@
-const { Connection }    = require('./connection');
-const   utils           = require('../utils');
-const { Sequelize, Op } = require('sequelize');
-const { underscore }    = require('inflection');
+const { Connection }     = require('./connection');
+const { MigrationError } = require('./migrationErrors');
+const   utils            = require('../utils');
+const { Sequelize, Op }  = require('sequelize');
+const { underscore }     = require('inflection');
 
 const ID_TYPES        = ['bigint', 'int', 'uuid'];
 const DEFAULT_ID_TYPE = 'bigint';
@@ -24,7 +25,7 @@ class Migrator {
 
 
     async run() {
-        if (!this.migrationModule.up) throw new Error('Missing migration `up`');
+        if (!this.migrationModule.up) throw new MigrationError('Missing migration `up`');
 
         await this.perform(() => this.migrationModule.up.call(this));
     }
@@ -122,8 +123,8 @@ class Migrator {
         if (idType === false) return {};
 
         if (!ID_TYPES.includes(idType)) {
-            throw new Error(`Unknown id type "${idType}". Use one of: ${ID_TYPES.join(', ')}, or `
-                          + `false for a table that declares its own primary key.`);
+            throw new MigrationError(`Unknown id type "${idType}". Use one of: ${ID_TYPES.join(', ')}, `
+                                   + `or false for a table that declares its own primary key.`);
         }
 
         return this[`${idType}IdColumn`];
@@ -213,8 +214,10 @@ class Migrator {
 
 
     async addIndex(tableName, fields, { concurrently, unique, name, using, where }={}) {
+        if (concurrently) this.assertConcurrentIndexAllowed('addIndex');
+
         const options = {fields: fields, ...this.queryOptions};
-        options.name = name || this.defaultIndexName(tableName, fields);
+        options.name  = name || this.defaultIndexName(tableName, fields);
         if (concurrently) options.concurrently = true;
         if (unique)       options.unique       = true;
         if (using)        options.using        = using;
@@ -224,8 +227,10 @@ class Migrator {
     }
 
 
-    async removeIndex(tableName, indexNameOrAttributes, concurrently=false) {
-        const options   = this.queryOptions;
+    async removeIndex(tableName, indexNameOrAttributes, { concurrently }={}) {
+        if (concurrently) this.assertConcurrentIndexAllowed('removeIndex');
+
+        const options = this.queryOptions;
         if (concurrently) options.concurrently = true;
 
         const indexName = this.indexNameFor(tableName, indexNameOrAttributes);
@@ -233,6 +238,18 @@ class Migrator {
     }
 
 
+    /***********************************************************************************************
+    * INDEX GUARDS
+    ***********************************************************************************************/
+    assertConcurrentIndexAllowed(methodName) {
+        if (!this.#transaction) return;
+
+        throw new MigrationError(`${methodName} cannot use \`concurrently\` inside a transaction. `
+                               + 'Add `transactional: false` to the migration module, and keep that '
+                               + 'migration to the statements that need it.');
+    }
+
+
     /***********************************************************************************************
     * INDEX NAMING
     ***********************************************************************************************/

package/lib/http/README.md

@@ -49,14 +49,28 @@ const response = await client.getTickets({ page: 1, pageSize: 25 });
 
 | Method | Options |
 |---|---|
-| `get(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal` |
-| `post(endpoint, opts)` | `body`, `headers`, `timeout`, `signal` |
-| `put(endpoint, opts)` | `body`, `headers`, `timeout`, `signal` |
-| `delete(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal` |
+| `get(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `post(endpoint, opts)` | `body`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `put(endpoint, opts)` | `body`, `headers`, `timeout`, `signal`, `binaryResponse` |
+| `delete(endpoint, opts)` | `requestParams`, `headers`, `timeout`, `signal`, `binaryResponse` |
 
 For `GET`/`DELETE`, `requestParams` becomes the query string. For `POST`/`PUT`, `body` becomes the request body. Object bodies are serialized as JSON; string bodies are sent unchanged.
 
 
+### Binary responses
+
+By default a response body is read as text and parsed as JSON when the `Content-Type` says so. That is wrong for media: decoding bytes as UTF-8 replaces every sequence that is not valid UTF-8, so the content arrives corrupted.
+
+Pass `binaryResponse: true` and `data` is a `Buffer` of the exact bytes instead. It is per call, so one client can fetch JSON metadata and binary media.
+
+```js
+async fetchRecording(path) {
+    const response = await this.get(path, { binaryResponse: true, headers: { Accept: 'audio/*' } });
+    return response.data;   // Buffer
+}
+```
+
+
 ### Query string encoding
 
 For `GET`/`DELETE`, the way `requestParams` is appended to the URL depends on the `Content-Type` header:

package/lib/http/client.js

@@ -7,22 +7,22 @@ logger.addScope('http');
 
 
 class HttpClient {
-    async get(endpoint, { requestParams, headers, timeout, signal }={}) {
+    async get(endpoint, { requestParams, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "GET", arguments[1]);
     }
 
 
-    async post(endpoint, { body, headers, timeout, signal }={}) {
+    async post(endpoint, { body, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "POST", arguments[1]);
     }
 
 
-    async put(endpoint, { body, headers, timeout, signal }={}) {
+    async put(endpoint, { body, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "PUT", arguments[1]);
     }
 
 
-    async delete(endpoint, { requestParams, headers, timeout, signal }={}) {
+    async delete(endpoint, { requestParams, headers, timeout, signal, binaryResponse }={}) {
         return await this.#makeRequest(endpoint, "DELETE", arguments[1]);
     }
 
@@ -32,7 +32,7 @@ class HttpClient {
     }
 
 
-    async #_makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal }={}) {
+    async #_makeRequest(endpoint, method, { requestParams, body, headers, timeout, signal, binaryResponse }={}) {
         this.#logRequest(method, endpoint);
 
         endpoint            = `${this.urlRoot}${endpoint}`;
@@ -42,7 +42,7 @@ class HttpClient {
 
         try {
             const res = await fetch(url.toString(), payload);
-            return await this.#handleRequestResponse(res, payload);
+            return await this.#handleRequestResponse(res, payload, binaryResponse);
         } catch (err) {
             this.#handleRequestError(err, payload, timeout, signal);
         }
@@ -70,9 +70,9 @@ class HttpClient {
     }
 
 
-    async #handleRequestResponse(res, payload) {
+    async #handleRequestResponse(res, payload, binaryResponse) {
         if (res.status && res.status < 300) {
-            const data = await this.#parseResponseData(res);
+            const data = await this.#parseResponseData(res, binaryResponse);
             return {data, status: res.status};
         } else {
             throw new HttpError(res, payload);
@@ -80,7 +80,11 @@ class HttpClient {
     }
 
 
-    async #parseResponseData(res) {
+    async #parseResponseData(res, binaryResponse) {
+        // Reading bytes as text decodes them as UTF-8, which replaces every sequence that is not
+        // valid UTF-8. Media has to be taken as bytes or it arrives corrupted.
+        if (binaryResponse) return Buffer.from(await res.arrayBuffer());
+
         const body = await res.text();
         if (!this.#isJsonResponse(res)) return body;
 

package/lib/jobQueue/README.md

@@ -179,6 +179,7 @@ await queue.getJob(id);       // one job, or null
 
 await queue.clean(grace, limit, type); // remove finished jobs older than `grace` ms
 await queue.clearQueuedJobs();         // drain all queued jobs
+await queue.obliterate();              // delete the queue and every key it owns in Redis
 
 await queue.setMaximumConcurrency(8);  // usually left to the Worker option above
 await queue.maximumConcurrency();
@@ -188,6 +189,9 @@ await queue.removeMaximumConcurrency();
 `clean` takes BullMQ's argument order, which is `(grace, limit, type)`. Bull's was
 `(grace, type, limit)` — worth checking when porting a call.
 
+`obliterate` forces, so it ends jobs that are still running rather than refusing. It leaves nothing
+behind for the queue to be rebuilt from — use it to retire a queue for good, not to empty one.
+
 Each `get*Jobs` and `getJob` result has this shape:
 
 ```js

package/lib/jobQueue/queue.js

@@ -64,6 +64,13 @@ class Queue extends AbstractJobComponent {
     }
 
 
+    // Forced because bullmq refuses to obliterate a queue that still has active jobs. A caller
+    // tearing a queue down wants those gone too.
+    async obliterate() {
+        await this.#queue.obliterate({ force: true });
+    }
+
+
     /***********************************************************************************************
     * LIMITS
     ***********************************************************************************************/

package/lib/search/README.md

@@ -0,0 +1,232 @@
+# Search
+`Search` is a backend-agnostic search client. A caller chains a query in plain vocabulary
+(`equals`, `range`, `search`) and a backend subclass turns it into whatever its engine speaks.
+`OpenSearchClient` is the only backend shipped today; the vocabulary carries no OpenSearch syntax,
+so another engine can be added without changing a single call site.
+
+
+## Setup
+
+An OpenSearch client signs its requests with AWS SigV4 and reads four environment variables:
+
+| Variable | Purpose |
+|---|---|
+| `OPENSEARCH_REGION` | AWS region of the collection. Falls back to `DEFAULT_AWS_REGION`. |
+| `DEFAULT_AWS_REGION` | Region used when `OPENSEARCH_REGION` is unset. |
+| `OPENSEARCH_AWS_KEY` | Access key id. |
+| `OPENSEARCH_AWS_SECRET` | Secret access key. |
+
+The collection endpoint is not an environment variable the library reads. Each subclass declares
+its own, so one application can talk to several collections. Environments are kept apart by
+pointing that endpoint at a different collection per deploy — index names are not namespaced.
+
+
+## Defining a client
+
+One subclass per index. It declares where the index lives, what it is called, and what its mapping
+should be.
+
+```js
+const { OpenSearchClient } = require('@vida-global/core');
+
+class MessageSearchClient extends OpenSearchClient {
+    get endpoint() { return process.env.OPENSEARCH_MESSAGE_NODE; }
+    get index()    { return 'messages'; }
+
+    get mapping() {
+        return {
+            properties: {
+                roomId:    { type: 'keyword' },
+                message:   { type: 'text'    },
+                timestamp: { type: 'date'    }
+            }
+        };
+    }
+}
+```
+
+
+## Searching
+
+`query()` returns a `Query`. Every method on it returns a **new** `Query`, so a part-built query is
+safe to hold onto and reuse. `fetch()` runs it.
+
+```js
+const client = new MessageSearchClient();
+
+const { total, results } = await client.query()
+    .equals('status', 'success')
+    .range('timestamp', { gte: startAt, lte: endAt })
+    .sort('timestamp', 'desc')
+    .limit(50)
+    .fetch();
+```
+
+`fetch` resolves to `{ total, results }`. `total` is the exact number of matching documents, not
+the number on this page. Each entry in `results` is `{ id, score, document }`, where `document` is
+what was stored. Override `_buildResult(hit)` on a subclass to reshape a single result.
+
+`AbstractSearchClient` assembles that envelope from five small hooks the backend implements:
+`_hitsFromResponse`, `_totalFromResponse`, `_idFromHit`, `_scoreFromHit`, and `_documentFromHit`.
+Override whichever one differs rather than rewriting the whole response loop.
+
+
+### Conditions
+
+| Method | Matches |
+|---|---|
+| `equals(field, value)` | The field holds exactly this value |
+| `in(field, values)` | The field holds any one of these values |
+| `range(field, bounds)` | `bounds` is any of `gte`, `gt`, `lte`, `lt` |
+| `exists(field)` | The field is present |
+| `missing(field)` | The field is absent |
+| `matches(field, text, options)` | Text match, fuzzy by default. `options.fuzziness` overrides it |
+| `phrase(field, text)` | The words appear together, in order |
+| `fullText(text, options)` | Text match across several fields at once, with per-field weights |
+| `nested(path, query)` | A sub-query against an array of objects |
+
+Field names are passed through untouched. OpenSearch stores a string field twice — analyzed for
+text search, and exact under a `.keyword` suffix — so an exact match on a text field asks for it
+by name:
+
+```js
+client.query().equals('roomId.keyword', roomId)
+```
+
+
+### Weighting fields in a text search
+
+`fullText` looks in several fields at once. `boost` makes a hit in one field count for more than a
+hit in another. Anything left out of `boost` carries a weight of 1.
+
+```js
+client.query().fullText(queryText, {
+    fields: ['message', 'fromUser', 'summary'],
+    boost:  { fromUser: 5, summary: 5 }
+})
+```
+
+Pass `wildcard: true` to match partial words. It finds `invoice` when the user types `voic`, but it
+is slow on a large index and it weakens relevance scoring, so leave it off unless partial matching
+is what you want.
+
+
+### Combining conditions
+
+Chained conditions are combined with AND. `or` and `not` take either a condition or a whole
+sub-query, and both accept several at once:
+
+```js
+client.query()
+    .equals('status', 'success')
+    .or(
+        client.query().equals('from', userId),
+        client.query().equals('to',   userId)
+    )
+    .not(client.query().exists('deletedAt'))
+```
+
+One `or(a, b)` call builds one group: the record must match `a` or `b`. Calling `or` twice adds to
+that same group. For two independent groups, nest a sub-query in each.
+
+
+### Paging and ordering
+
+`limit`, `offset`, `sort`, and `minScore` chain like conditions and travel with the query.
+
+`minScore` drops weak matches. The engine applies it, so `total` counts only what survived.
+
+
+### Reading across date-partitioned indexes
+
+Pass a suffix to the constructor to write into a dated index. Reads then target that one index
+unless you ask for the wildcard.
+
+```js
+const client = new ApiLogSearchClient('2026-10');
+
+// writes api-logs-2026-10
+await client.append(logObject);
+
+// reads api-logs-2026-10
+await client.query().equals('accountId', id).fetch();
+
+// reads api-logs-*
+await client.query().equals('accountId', id).fetch({ indexWildcard: true });
+```
+
+
+## Writing
+
+| Method | What it does |
+|---|---|
+| `upsert(id, document)` | Creates the document, or merges into it if the id already exists |
+| `append(document)` | Writes a new document and lets the engine assign the id |
+| `bulkUpsert(documents)` | One request for many. Each entry is `{ id, document }` |
+| `delete(id)` | Removes one document |
+| `deleteWhere(query)` | Removes everything the query matches. Returns how many |
+
+Every write throws on failure. `bulkUpsert` throws `BulkWriteError`, which carries a `failures`
+list of `{ id, reason }` — a bulk request answers `200` even when individual documents fail, so
+the error is built from the response body, not the status code.
+
+`deleteWhere` searches a page, deletes each id, and repeats until nothing matches. OpenSearch
+Serverless has no delete-by-query API, so there is no single-request version. A failure stops the
+loop and throws; what was already deleted stays deleted.
+
+
+### Collection types
+
+An OpenSearch Serverless collection has a type, and only a `SEARCH` collection accepts a write
+addressed to a specific id. Nothing is declared up front: when the collection refuses such a
+write, `upsert` and `bulkUpsert` catch the refusal and raise `UnsupportedOperationError` with the
+original error attached as `cause`.
+
+`_isUnsupportedOperation(error)` is what reads a refusal out of an OpenSearch error. Override it
+on a subclass if the message you see does not match.
+
+`append` works on every collection type.
+
+
+## Index mappings
+
+A mapping is the index's schema. `syncIndex` makes the live index match what the subclass
+declares: it creates the index if it is missing, then pushes the mapping.
+
+```js
+await MessageSearchClient.syncIndex();
+```
+
+OpenSearch accepts a new field on an existing index but rejects a changed type on an existing
+field. `syncIndex` does not diff anything — it sends the mapping and lets a conflict throw.
+
+
+## Retries
+
+Nothing retries by default. A subclass opts in by raising `numRetries`:
+
+```js
+class MessageSearchClient extends OpenSearchClient {
+    get numRetries()        { return 3; }
+    get retryBackoffDelay() { return 1000; }
+    get retryMaxDelay()     { return 8000; }
+}
+```
+
+`OpenSearchClient` only retries server errors — status 500 through 599. Override `shouldRetry`
+to change that.
+
+Retries cover `fetch`, `upsert`, `delete`, and `syncIndex`. They never cover `append` or
+`bulkUpsert`: those carry no caller-supplied id, so re-sending a request that actually succeeded
+would write the document twice.
+
+
+## Adding a backend
+
+Subclass `AbstractSearchClient` and implement the hooks it declares: `_serializeConditions`,
+`_execute`, the five response hooks, the four write hooks, and the three index hooks.
+`AbstractSearchClient` owns the parts that are not engine-specific — the retry loop, the
+`deleteWhere` loop, the `syncIndex` order, and the result envelope.
+
+Collaborators are reached through seams, so tests inject doubles by subclassing and overriding
+`openSearchClientClass`, `awsSignerFactory`, or `loggerLibrary`.