@vida-global/core 2.5.0 → 3.0.1

package/README.md

@@ -9,3 +9,4 @@ This package contains core elements used across all Vida Apps. It is included in
 - [Logger](lib/logger) — scoped, child-aware logger with environment-driven levels.
 - [VidaServer](lib/server) — Express-based server with auto-loaded controllers, validation, rendering, callbacks, cookies, and SSE streaming.
 - [JobQueue](lib/jobQueue) — BullMQ-backed queue with workers, retries/backoff, progress reporting, and queue inspection helpers.
+- [Search](lib/search) — backend-agnostic search client with a chainable query vocabulary, an OpenSearch Serverless implementation, and index mapping management.

package/index.js

@@ -6,12 +6,14 @@ const   JobQueue          = require('./lib/jobQueue');
 const { logger }          = require('./lib/logger');
 const   redisLibs         = require('./lib/redis');
 const   cacheLibs         = require('./lib/cache');
+const   searchLibs        = require('./lib/search');
 const   serverLibs        = require('./lib/server');
 
 
 const {
   AbstractServerError,
   ApiDocsGenerator,
+  Authorizer,
   VidaServer,
   VidaServerController,
 } = serverLibs;
@@ -21,11 +23,13 @@ const serverErrorsToExport = Object.values(serverLibs).filter(obj => obj.prototy
 module.exports = {
     ActiveRecord,
     APM,
+    Authorizer,
     ...httpLibs,
     JobQueue,
     logger,
     ...cacheLibs,
     ...redisLibs,
+    ...searchLibs,
     ...serverErrorsToExport,
     ApiDocsGenerator,
     AppDependencies,

package/lib/activeRecord/baseRecord.js

@@ -379,7 +379,7 @@ class BaseRecord extends Model {
     /***********************************************************************************************
     * CACHING
     ***********************************************************************************************/
-    static async _cachedFind(ids, { clear=false }={}) {
+    static async _cachedFind(ids, { clear=false, expireSeconds=null }={}) {
         const multiFind = Array.isArray(ids)
         const idsToFind = multiFind ? ids : [ids];
         const keyPairs  = idsToFind.map(id => [this._recordCacheKey(id), id]);
@@ -387,7 +387,7 @@ class BaseRecord extends Model {
         const idsToKeys = Object.fromEntries(keyPairs.map(p => p.reverse()));
         const keys      = Object.keys(keysToIds);
 
-        let records = await this.cachedFetch(keys, { clear }, async (missedKeys) => {
+        let records = await this.cachedFetch(keys, { clear, expireSeconds }, async (missedKeys) => {
             const idsToFind = missedKeys.map(key => keysToIds[key]);
             const pk        = this.primaryKeyAttribute
             const records   = await this.where({[pk]: idsToFind});

package/lib/search/README.md

@@ -0,0 +1,232 @@
+# Search
+`Search` is a backend-agnostic search client. A caller chains a query in plain vocabulary
+(`equals`, `range`, `search`) and a backend subclass turns it into whatever its engine speaks.
+`OpenSearchClient` is the only backend shipped today; the vocabulary carries no OpenSearch syntax,
+so another engine can be added without changing a single call site.
+
+
+## Setup
+
+An OpenSearch client signs its requests with AWS SigV4 and reads four environment variables:
+
+| Variable | Purpose |
+|---|---|
+| `OPENSEARCH_REGION` | AWS region of the collection. Falls back to `DEFAULT_AWS_REGION`. |
+| `DEFAULT_AWS_REGION` | Region used when `OPENSEARCH_REGION` is unset. |
+| `OPENSEARCH_AWS_KEY` | Access key id. |
+| `OPENSEARCH_AWS_SECRET` | Secret access key. |
+
+The collection endpoint is not an environment variable the library reads. Each subclass declares
+its own, so one application can talk to several collections. Environments are kept apart by
+pointing that endpoint at a different collection per deploy — index names are not namespaced.
+
+
+## Defining a client
+
+One subclass per index. It declares where the index lives, what it is called, and what its mapping
+should be.
+
+```js
+const { OpenSearchClient } = require('@vida-global/core');
+
+class MessageSearchClient extends OpenSearchClient {
+    get endpoint() { return process.env.OPENSEARCH_MESSAGE_NODE; }
+    get index()    { return 'messages'; }
+
+    get mapping() {
+        return {
+            properties: {
+                roomId:    { type: 'keyword' },
+                message:   { type: 'text'    },
+                timestamp: { type: 'date'    }
+            }
+        };
+    }
+}
+```
+
+
+## Searching
+
+`query()` returns a `Query`. Every method on it returns a **new** `Query`, so a part-built query is
+safe to hold onto and reuse. `fetch()` runs it.
+
+```js
+const client = new MessageSearchClient();
+
+const { total, results } = await client.query()
+    .equals('status', 'success')
+    .range('timestamp', { gte: startAt, lte: endAt })
+    .sort('timestamp', 'desc')
+    .limit(50)
+    .fetch();
+```
+
+`fetch` resolves to `{ total, results }`. `total` is the exact number of matching documents, not
+the number on this page. Each entry in `results` is `{ id, score, document }`, where `document` is
+what was stored. Override `_buildResult(hit)` on a subclass to reshape a single result.
+
+`AbstractSearchClient` assembles that envelope from five small hooks the backend implements:
+`_hitsFromResponse`, `_totalFromResponse`, `_idFromHit`, `_scoreFromHit`, and `_documentFromHit`.
+Override whichever one differs rather than rewriting the whole response loop.
+
+
+### Conditions
+
+| Method | Matches |
+|---|---|
+| `equals(field, value)` | The field holds exactly this value |
+| `in(field, values)` | The field holds any one of these values |
+| `range(field, bounds)` | `bounds` is any of `gte`, `gt`, `lte`, `lt` |
+| `exists(field)` | The field is present |
+| `missing(field)` | The field is absent |
+| `matches(field, text, options)` | Text match, fuzzy by default. `options.fuzziness` overrides it |
+| `phrase(field, text)` | The words appear together, in order |
+| `fullText(text, options)` | Text match across several fields at once, with per-field weights |
+| `nested(path, query)` | A sub-query against an array of objects |
+
+Field names are passed through untouched. OpenSearch stores a string field twice — analyzed for
+text search, and exact under a `.keyword` suffix — so an exact match on a text field asks for it
+by name:
+
+```js
+client.query().equals('roomId.keyword', roomId)
+```
+
+
+### Weighting fields in a text search
+
+`fullText` looks in several fields at once. `boost` makes a hit in one field count for more than a
+hit in another. Anything left out of `boost` carries a weight of 1.
+
+```js
+client.query().fullText(queryText, {
+    fields: ['message', 'fromUser', 'summary'],
+    boost:  { fromUser: 5, summary: 5 }
+})
+```
+
+Pass `wildcard: true` to match partial words. It finds `invoice` when the user types `voic`, but it
+is slow on a large index and it weakens relevance scoring, so leave it off unless partial matching
+is what you want.
+
+
+### Combining conditions
+
+Chained conditions are combined with AND. `or` and `not` take either a condition or a whole
+sub-query, and both accept several at once:
+
+```js
+client.query()
+    .equals('status', 'success')
+    .or(
+        client.query().equals('from', userId),
+        client.query().equals('to',   userId)
+    )
+    .not(client.query().exists('deletedAt'))
+```
+
+One `or(a, b)` call builds one group: the record must match `a` or `b`. Calling `or` twice adds to
+that same group. For two independent groups, nest a sub-query in each.
+
+
+### Paging and ordering
+
+`limit`, `offset`, `sort`, and `minScore` chain like conditions and travel with the query.
+
+`minScore` drops weak matches. The engine applies it, so `total` counts only what survived.
+
+
+### Reading across date-partitioned indexes
+
+Pass a suffix to the constructor to write into a dated index. Reads then target that one index
+unless you ask for the wildcard.
+
+```js
+const client = new ApiLogSearchClient('2026-10');
+
+// writes api-logs-2026-10
+await client.append(logObject);
+
+// reads api-logs-2026-10
+await client.query().equals('accountId', id).fetch();
+
+// reads api-logs-*
+await client.query().equals('accountId', id).fetch({ indexWildcard: true });
+```
+
+
+## Writing
+
+| Method | What it does |
+|---|---|
+| `upsert(id, document)` | Creates the document, or merges into it if the id already exists |
+| `append(document)` | Writes a new document and lets the engine assign the id |
+| `bulkUpsert(documents)` | One request for many. Each entry is `{ id, document }` |
+| `delete(id)` | Removes one document |
+| `deleteWhere(query)` | Removes everything the query matches. Returns how many |
+
+Every write throws on failure. `bulkUpsert` throws `BulkWriteError`, which carries a `failures`
+list of `{ id, reason }` — a bulk request answers `200` even when individual documents fail, so
+the error is built from the response body, not the status code.
+
+`deleteWhere` searches a page, deletes each id, and repeats until nothing matches. OpenSearch
+Serverless has no delete-by-query API, so there is no single-request version. A failure stops the
+loop and throws; what was already deleted stays deleted.
+
+
+### Collection types
+
+An OpenSearch Serverless collection has a type, and only a `SEARCH` collection accepts a write
+addressed to a specific id. Nothing is declared up front: when the collection refuses such a
+write, `upsert` and `bulkUpsert` catch the refusal and raise `UnsupportedOperationError` with the
+original error attached as `cause`.
+
+`_isUnsupportedOperation(error)` is what reads a refusal out of an OpenSearch error. Override it
+on a subclass if the message you see does not match.
+
+`append` works on every collection type.
+
+
+## Index mappings
+
+A mapping is the index's schema. `syncIndex` makes the live index match what the subclass
+declares: it creates the index if it is missing, then pushes the mapping.
+
+```js
+await MessageSearchClient.syncIndex();
+```
+
+OpenSearch accepts a new field on an existing index but rejects a changed type on an existing
+field. `syncIndex` does not diff anything — it sends the mapping and lets a conflict throw.
+
+
+## Retries
+
+Nothing retries by default. A subclass opts in by raising `numRetries`:
+
+```js
+class MessageSearchClient extends OpenSearchClient {
+    get numRetries()        { return 3; }
+    get retryBackoffDelay() { return 1000; }
+    get retryMaxDelay()     { return 8000; }
+}
+```
+
+`OpenSearchClient` only retries server errors — status 500 through 599. Override `shouldRetry`
+to change that.
+
+Retries cover `fetch`, `upsert`, `delete`, and `syncIndex`. They never cover `append` or
+`bulkUpsert`: those carry no caller-supplied id, so re-sending a request that actually succeeded
+would write the document twice.
+
+
+## Adding a backend
+
+Subclass `AbstractSearchClient` and implement the hooks it declares: `_serializeConditions`,
+`_execute`, the five response hooks, the four write hooks, and the three index hooks.
+`AbstractSearchClient` owns the parts that are not engine-specific — the retry loop, the
+`deleteWhere` loop, the `syncIndex` order, and the result envelope.
+
+Collaborators are reached through seams, so tests inject doubles by subclassing and overriding
+`openSearchClientClass`, `awsSignerFactory`, or `loggerLibrary`.

package/lib/search/abstractSearchClient.js

@@ -0,0 +1,188 @@
+const { Query }  = require('./query');
+
+
+class AbstractSearchClient {
+    #indexSuffix;
+
+
+    constructor(indexSuffix) {
+        if (this.constructor === AbstractSearchClient) {
+            throw new Error('SearchClients must be subclasses of AbstractSearchClient');
+        }
+        this.#indexSuffix = indexSuffix;
+    }
+
+
+    get indexSuffix()       { return this.#indexSuffix; }
+    get deleteBatchSize()   { return 500; }
+    get numRetries()        { return 0; }
+    get retryBackoffDelay() { return 1000; }
+    get retryMaxDelay()     { return 8000; }
+
+
+    query() {
+        return new Query(this);
+    }
+
+
+    /***********************************************************************************************
+    * READ
+    ***********************************************************************************************/
+    async fetch(conditions, options = {}) {
+        const serializedConditions = this._serializeConditions(conditions);
+        const response = await this._withRetry(() => this._execute(serializedConditions, options));
+        return this._processResponse(response);
+    }
+
+
+    _processResponse(response) {
+        const hits = this._hitsFromResponse(response)
+
+        return {
+            total:   this._totalFromResponse(response),
+            results: hits.hits.map(hit => this._buildResult(hit))
+        };
+    }
+
+
+    _buildResult(hit) {
+        const id       = this._idFromHit(hit);
+        const score    = this._scoreFromHit(hit);
+        const document = this._documentFromHit(hit);
+        return { id, score, document };
+    }
+
+
+    _hitsFromResponse(response)  { throw new Error('Not Implemented') }
+    _totalFromResponse(response) { throw new Error('Not Implemented') }
+    _idFromHit(hit)              { throw new Error('Not Implemented') }
+    _scoreFromHit(hit)           { throw new Error('Not Implemented') }
+    _documentFromHit(hit)        { throw new Error('Not Implemented') }
+
+
+    /***********************************************************************************************
+    * WRITE
+    ***********************************************************************************************/
+    async upsert(id, document) {
+        return await this._withRetry(() => this._upsert(id, document));
+    }
+
+
+    async delete(id) {
+        return await this._withRetry(() => this._delete(id));
+    }
+
+
+    async append(document) {
+        return await this._append(document);
+    }
+
+
+    async bulkUpsert(documents) {
+        return await this._bulkUpsert(documents);
+    }
+
+
+    async deleteWhere(query) {
+        const deletedIds = new Set();
+        let   batch      = await this.#fetchDeletionBatch(query, deletedIds);
+
+        while (batch.length) {
+            await this.#deleteBatch(batch, deletedIds);
+            batch = await this.#fetchDeletionBatch(query, deletedIds);
+        }
+        return deletedIds.size;
+    }
+
+
+    async #fetchDeletionBatch(query, deletedIds) {
+        const { results } = await query.limit(this.deleteBatchSize).fetch();
+        return results.map(result => result.id).filter(id => !deletedIds.has(id));
+    }
+
+
+    async #deleteBatch(ids, deletedIds) {
+        for (const id of ids) {
+            await this.delete(id);
+            deletedIds.add(id);
+        }
+    }
+
+
+    /***********************************************************************************************
+    * ADMIN
+    ***********************************************************************************************/
+    static async syncIndex(indexSuffix) {
+        return await new this(indexSuffix).syncIndex();
+    }
+
+
+    async syncIndex() {
+        return await this._withRetry(() => this._syncIndex());
+    }
+
+
+    async _syncIndex() {
+        if (!await this._indexExists()) await this._createIndex();
+        return await this._putMapping();
+    }
+
+
+    /***********************************************************************************************
+    * RETRIES
+    ***********************************************************************************************/
+    shouldRetry(attemptCount, _error) {
+        return attemptCount - 1 < this.numRetries;
+    }
+
+
+    async _withRetry(operation) {
+        let attemptCount = 0;
+
+        while (true) {
+            attemptCount += 1;
+            try {
+                return await operation();
+            } catch (error) {
+                await this.#handleFailedAttempt(error, attemptCount);
+            }
+        }
+    }
+
+
+    async #handleFailedAttempt(error, attemptCount) {
+        if (!this.shouldRetry(attemptCount, error)) {
+            throw error;
+        }
+        await this.#waitBeforeRetry(attemptCount);
+    }
+
+
+    async #waitBeforeRetry(attemptCount) {
+        const backoff = 2 ** (attemptCount - 1) * this.retryBackoffDelay;
+        const delay   = Math.min(backoff, this.retryMaxDelay);
+        await new Promise(resolve => setTimeout(resolve, delay));
+    }
+
+
+    /***********************************************************************************************
+    * BACKEND HOOKS
+    ***********************************************************************************************/
+    get index()   { throw new Error('Not Implemented') }
+    get mapping() { throw new Error('Not Implemented') }
+
+    _serializeConditions(conditions)        { throw new Error('Not Implemented') }
+    _execute(serializedConditions, options) { throw new Error('Not Implemented') }
+    _upsert(id, document)                   { throw new Error('Not Implemented') }
+    _append(document)                       { throw new Error('Not Implemented') }
+    _bulkUpsert(documents)                  { throw new Error('Not Implemented') }
+    _delete(id)                             { throw new Error('Not Implemented') }
+    _indexExists()                          { throw new Error('Not Implemented') }
+    _createIndex()                          { throw new Error('Not Implemented') }
+    _putMapping()                           { throw new Error('Not Implemented') }
+}
+
+
+module.exports = {
+    AbstractSearchClient
+};

package/lib/search/errors.js

@@ -0,0 +1,31 @@
+class SearchError extends Error {
+
+    constructor(message, options) {
+        super(message, options);
+        this.name = this.constructor.name;
+    }
+}
+
+
+class UnsupportedOperationError extends SearchError {}
+
+
+class BulkWriteError extends SearchError {
+    #failures;
+
+
+    constructor(message, failures) {
+        super(message);
+        this.#failures = failures;
+    }
+
+
+    get failures() { return this.#failures; }
+}
+
+
+module.exports = {
+    BulkWriteError,
+    SearchError,
+    UnsupportedOperationError
+};

package/lib/search/index.js

@@ -0,0 +1,16 @@
+const { AbstractSearchClient } = require('./abstractSearchClient');
+const { OpenSearchClient }     = require('./openSearchClient');
+const { Query }                = require('./query');
+const { SearchError,
+        BulkWriteError,
+        UnsupportedOperationError } = require('./errors');
+
+
+module.exports = {
+    AbstractSearchClient,
+    SearchError,
+    BulkWriteError,
+    OpenSearchClient,
+    Query,
+    UnsupportedOperationError
+};