@vida-global/core 2.4.5 → 3.0.0

package/lib/search/abstractSearchClient.js

@@ -0,0 +1,188 @@
+const { Query }  = require('./query');
+
+
+class AbstractSearchClient {
+    #indexSuffix;
+
+
+    constructor(indexSuffix) {
+        if (this.constructor === AbstractSearchClient) {
+            throw new Error('SearchClients must be subclasses of AbstractSearchClient');
+        }
+        this.#indexSuffix = indexSuffix;
+    }
+
+
+    get indexSuffix()       { return this.#indexSuffix; }
+    get deleteBatchSize()   { return 500; }
+    get numRetries()        { return 0; }
+    get retryBackoffDelay() { return 1000; }
+    get retryMaxDelay()     { return 8000; }
+
+
+    query() {
+        return new Query(this);
+    }
+
+
+    /***********************************************************************************************
+    * READ
+    ***********************************************************************************************/
+    async fetch(conditions, options = {}) {
+        const serializedConditions = this._serializeConditions(conditions);
+        const response = await this._withRetry(() => this._execute(serializedConditions, options));
+        return this._processResponse(response);
+    }
+
+
+    _processResponse(response) {
+        const hits = this._hitsFromResponse(response)
+
+        return {
+            total:   this._totalFromResponse(response),
+            results: hits.hits.map(hit => this._buildResult(hit))
+        };
+    }
+
+
+    _buildResult(hit) {
+        const id       = this._idFromHit(hit);
+        const score    = this._scoreFromHit(hit);
+        const document = this._documentFromHit(hit);
+        return { id, score, document };
+    }
+
+
+    _hitsFromResponse(response)  { throw new Error('Not Implemented') }
+    _totalFromResponse(response) { throw new Error('Not Implemented') }
+    _idFromHit(hit)              { throw new Error('Not Implemented') }
+    _scoreFromHit(hit)           { throw new Error('Not Implemented') }
+    _documentFromHit(hit)        { throw new Error('Not Implemented') }
+
+
+    /***********************************************************************************************
+    * WRITE
+    ***********************************************************************************************/
+    async upsert(id, document) {
+        return await this._withRetry(() => this._upsert(id, document));
+    }
+
+
+    async delete(id) {
+        return await this._withRetry(() => this._delete(id));
+    }
+
+
+    async append(document) {
+        return await this._append(document);
+    }
+
+
+    async bulkUpsert(documents) {
+        return await this._bulkUpsert(documents);
+    }
+
+
+    async deleteWhere(query) {
+        const deletedIds = new Set();
+        let   batch      = await this.#fetchDeletionBatch(query, deletedIds);
+
+        while (batch.length) {
+            await this.#deleteBatch(batch, deletedIds);
+            batch = await this.#fetchDeletionBatch(query, deletedIds);
+        }
+        return deletedIds.size;
+    }
+
+
+    async #fetchDeletionBatch(query, deletedIds) {
+        const { results } = await query.limit(this.deleteBatchSize).fetch();
+        return results.map(result => result.id).filter(id => !deletedIds.has(id));
+    }
+
+
+    async #deleteBatch(ids, deletedIds) {
+        for (const id of ids) {
+            await this.delete(id);
+            deletedIds.add(id);
+        }
+    }
+
+
+    /***********************************************************************************************
+    * ADMIN
+    ***********************************************************************************************/
+    static async syncIndex(indexSuffix) {
+        return await new this(indexSuffix).syncIndex();
+    }
+
+
+    async syncIndex() {
+        return await this._withRetry(() => this._syncIndex());
+    }
+
+
+    async _syncIndex() {
+        if (!await this._indexExists()) await this._createIndex();
+        return await this._putMapping();
+    }
+
+
+    /***********************************************************************************************
+    * RETRIES
+    ***********************************************************************************************/
+    shouldRetry(attemptCount, _error) {
+        return attemptCount - 1 < this.numRetries;
+    }
+
+
+    async _withRetry(operation) {
+        let attemptCount = 0;
+
+        while (true) {
+            attemptCount += 1;
+            try {
+                return await operation();
+            } catch (error) {
+                await this.#handleFailedAttempt(error, attemptCount);
+            }
+        }
+    }
+
+
+    async #handleFailedAttempt(error, attemptCount) {
+        if (!this.shouldRetry(attemptCount, error)) {
+            throw error;
+        }
+        await this.#waitBeforeRetry(attemptCount);
+    }
+
+
+    async #waitBeforeRetry(attemptCount) {
+        const backoff = 2 ** (attemptCount - 1) * this.retryBackoffDelay;
+        const delay   = Math.min(backoff, this.retryMaxDelay);
+        await new Promise(resolve => setTimeout(resolve, delay));
+    }
+
+
+    /***********************************************************************************************
+    * BACKEND HOOKS
+    ***********************************************************************************************/
+    get index()   { throw new Error('Not Implemented') }
+    get mapping() { throw new Error('Not Implemented') }
+
+    _serializeConditions(conditions)        { throw new Error('Not Implemented') }
+    _execute(serializedConditions, options) { throw new Error('Not Implemented') }
+    _upsert(id, document)                   { throw new Error('Not Implemented') }
+    _append(document)                       { throw new Error('Not Implemented') }
+    _bulkUpsert(documents)                  { throw new Error('Not Implemented') }
+    _delete(id)                             { throw new Error('Not Implemented') }
+    _indexExists()                          { throw new Error('Not Implemented') }
+    _createIndex()                          { throw new Error('Not Implemented') }
+    _putMapping()                           { throw new Error('Not Implemented') }
+}
+
+
+module.exports = {
+    AbstractSearchClient
+};

package/lib/search/errors.js

@@ -0,0 +1,31 @@
+class SearchError extends Error {
+
+    constructor(message, options) {
+        super(message, options);
+        this.name = this.constructor.name;
+    }
+}
+
+
+class UnsupportedOperationError extends SearchError {}
+
+
+class BulkWriteError extends SearchError {
+    #failures;
+
+
+    constructor(message, failures) {
+        super(message);
+        this.#failures = failures;
+    }
+
+
+    get failures() { return this.#failures; }
+}
+
+
+module.exports = {
+    BulkWriteError,
+    SearchError,
+    UnsupportedOperationError
+};

package/lib/search/index.js

@@ -0,0 +1,16 @@
+const { AbstractSearchClient } = require('./abstractSearchClient');
+const { OpenSearchClient }     = require('./openSearchClient');
+const { Query }                = require('./query');
+const { SearchError,
+        BulkWriteError,
+        UnsupportedOperationError } = require('./errors');
+
+
+module.exports = {
+    AbstractSearchClient,
+    SearchError,
+    BulkWriteError,
+    OpenSearchClient,
+    Query,
+    UnsupportedOperationError
+};

package/lib/search/openSearchClient.js

@@ -0,0 +1,319 @@
+const { AbstractSearchClient } = require('./abstractSearchClient');
+const { AwsSigv4Signer }       = require('@opensearch-project/opensearch/aws');
+const { BulkWriteError, UnsupportedOperationError } = require('./errors');
+const { Client }               = require('@opensearch-project/opensearch');
+
+
+class OpenSearchClient extends AbstractSearchClient {
+    #_client;
+
+
+    static get SERVICE() { return 'aoss'; }
+
+
+    get wildcardIndex() { return `${this.index}-*`; }
+    get endpoint()      { throw new Error('Not Implemented') }
+
+
+    get writeIndex() {
+        if (!this.indexSuffix) return this.index;
+        return `${this.index}-${this.indexSuffix}`;
+    }
+
+
+    /***********************************************************************************************
+    * READ
+    ***********************************************************************************************/
+    async _execute(serializedConditions, options) {
+        return await this.#client.search({
+            index: this.#readIndexForFetch(options),
+            body:  serializedConditions
+        });
+    }
+
+
+    _hitsFromResponse(response)  { return response.body.hits                   }
+    _totalFromResponse(response) { return response.body.hits.total?.value || 0 }
+    _idFromHit(hit)              { return hit._id                              }
+    _scoreFromHit(hit)           { return hit._score                           }
+    _documentFromHit(hit)        { return hit._source                          }
+
+
+    #readIndexForFetch({indexWildcard}) {
+        return indexWildcard ? this.wildcardIndex : this.writeIndex;
+    }
+
+
+    /***********************************************************************************************
+    * SERIALIZATION
+    ***********************************************************************************************/
+    _serializeConditions(conditions) {
+        return {
+            query:            this.#serializeTree(conditions),
+            track_total_hits: true,
+            ...this.#serializeOptions(conditions.options)
+        };
+    }
+
+
+    #serializeTree(conditions) {
+        const must    = this.#serializeAll(conditions.and);
+        const mustNot = this.#serializeAll(conditions.not);
+        if (conditions.or.length) must.push(this.#serializeShouldGroup(conditions.or));
+
+        if (must.length === 1 && !mustNot.length) return must[0];
+        return {bool: {must, must_not: mustNot}};
+    }
+
+
+    #serializeAll(conditions) {
+        return conditions.map(condition => this.#serializeCondition(condition));
+    }
+
+
+    // `minimum_should_match` is not optional here: alongside a populated `must`, OpenSearch treats
+    // `should` clauses as score boosters rather than filters, so the group would stop filtering.
+    #serializeShouldGroup(conditions) {
+        return {bool: {should: this.#serializeAll(conditions), minimum_should_match: 1}};
+    }
+
+
+    #serializeCondition(condition) {
+        if (condition.operator) return this.#serializeLeaf(condition);
+        return this.#serializeTree(condition);
+    }
+
+
+    #serializeLeaf(leaf) {
+        const operator = `${leaf.operator[0].toUpperCase()}${leaf.operator.slice(1)}`;
+        return this[`_serialize${operator}`](leaf);
+    }
+
+
+    #serializeOptions({limit, minScore, offset, sort}) {
+        const body = {};
+
+        if (limit    !== undefined) body.size      = limit;
+        if (offset   !== undefined) body.from      = offset;
+        if (minScore !== undefined) body.min_score = minScore;
+        if (sort)                   body.sort      = this.#serializeSort(sort);
+        return body;
+    }
+
+
+    #serializeSort(sortKeys) {
+        return sortKeys.map(({field, direction}) => ({[field]: {order: direction}}));
+    }
+
+
+    /***********************************************************************************************
+    * CONDITION SERIALIZERS
+    ***********************************************************************************************/
+    _serializeEquals({field, value}) {
+        return {term: {[field]: {value}}};
+    }
+
+
+    _serializeIn({field, value}) {
+        return {terms: {[field]: value}};
+    }
+
+
+    _serializeRange({field, value}) {
+        return {range: {[field]: value}};
+    }
+
+
+    _serializeExists({field}) {
+        return {exists: {field}};
+    }
+
+
+    _serializeMissing({field}) {
+        return {bool: {must_not: {exists: {field}}}};
+    }
+
+
+    _serializeMatches({field, value, options}) {
+        return {match: {[field]: {query: value, fuzziness: options.fuzziness}}};
+    }
+
+
+    _serializePhrase({field, value}) {
+        return {match_phrase: {[field]: {query: value}}};
+    }
+
+
+    _serializeFullText({field, value, options}) {
+        const query  = options.wildcard ? `*${value}*` : value;
+        const fields = Object.entries(field).map(([name, boost]) => `${name}^${boost}`);
+
+        return {query_string: {query, fields}};
+    }
+
+
+    _serializeNested({field, value}) {
+        return {nested: {path: field, query: this.#serializeTree(value)}};
+    }
+
+
+    /***********************************************************************************************
+    * WRITE
+    ***********************************************************************************************/
+    async _upsert(id, document) {
+        return await this.#rethrowUnsupported('upsert', () => this.#client.update({
+            id,
+            index: this.writeIndex,
+            body:  {doc: document, doc_as_upsert: true}
+        }));
+    }
+
+
+    async _append(document) {
+        return await this.#client.index({index: this.writeIndex, body: document});
+    }
+
+
+    async _delete(id) {
+        return await this.#client.delete({id, index: this.writeIndex});
+    }
+
+
+    async _bulkUpsert(documents) {
+        const body     = this.#buildBulkBody(documents);
+        const response = await this.#rethrowUnsupported('bulkUpsert', () => this.#client.bulk({body}));
+        this.#assertBulkSucceeded(response);
+        return response;
+    }
+
+
+    #buildBulkBody(documents) {
+        const operations = [];
+
+        for (const { id, document } of documents) {
+            operations.push({update: {_id: id, _index: this.writeIndex}});
+            operations.push({doc: document, doc_as_upsert: true});
+        }
+        return operations;
+    }
+
+
+    #assertBulkSucceeded(response) {
+        if (!response.body.errors) return;
+
+        const failures = this.#collectBulkFailures(response.body.items);
+        throw new BulkWriteError(`${failures.length} documents could not be written`, failures);
+    }
+
+
+    #collectBulkFailures(items) {
+        const failures = [];
+
+        for (const item of items) {
+            const outcome = item.update;
+            if (outcome.error) failures.push({id: outcome._id, reason: outcome.error.reason});
+        }
+        return failures;
+    }
+
+
+    async #rethrowUnsupported(operationName, operation) {
+        try {
+            return await operation();
+        } catch (error) {
+            if (!this._isUnsupportedOperation(error)) throw error;
+            throw new UnsupportedOperationError(
+                `${operationName} is not supported by the collection behind ${this.constructor.name}`,
+                {cause: error}
+            );
+        }
+    }
+
+
+    // Serverless rejects every write addressed to a document id unless the collection is of type
+    // SEARCH. Nothing in the response names the collection type, so the refusal is read off the
+    // message. The regex here is a guess and should be fixed if it doesn't properly catch the error
+    _isUnsupportedOperation(error) {
+        const statusCode = error.statusCode || error.meta?.statusCode;
+        if (statusCode !== 400) return false;
+        return /not supported/i.test(error.message || '');
+    }
+
+
+    /***********************************************************************************************
+    * ADMIN
+    ***********************************************************************************************/
+    async _indexExists() {
+        const response = await this.#client.indices.exists({index: this.writeIndex});
+        return Boolean(response.body);
+    }
+
+
+    async _createIndex() {
+        return await this.#client.indices.create({index: this.writeIndex});
+    }
+
+
+    async _putMapping() {
+        return await this.#client.indices.putMapping({index: this.writeIndex, body: this.mapping});
+    }
+
+
+    /***********************************************************************************************
+    * RETRIES
+    ***********************************************************************************************/
+    shouldRetry(attemptCount, error) {
+        if (!super.shouldRetry(attemptCount, error)) return false;
+        return this.#isServerError(error);
+    }
+
+
+    #isServerError(error) {
+        const statusCode = error.statusCode || error.meta?.statusCode;
+        return statusCode >= 500 && statusCode < 600;
+    }
+
+
+    /***********************************************************************************************
+    * CLIENT CONFIG
+    ***********************************************************************************************/
+    get openSearchClientClass() { return Client; }
+    get awsSignerFactory()      { return AwsSigv4Signer; }
+
+
+    get #client() {
+        return this.#_client ||= new this.openSearchClientClass(this.#config);
+    }
+
+
+    get #config() {
+        return {...this.awsSigner, node: this.endpoint};
+    }
+
+
+    get awsSigner() {
+        return this.awsSignerFactory({
+            region:         this.region,
+            service:        this.constructor.SERVICE,
+            getCredentials: this.getCredentials.bind(this)
+        });
+    }
+
+
+    get region() {
+        return process.env.OPENSEARCH_REGION || process.env.DEFAULT_AWS_REGION;
+    }
+
+
+    async getCredentials() {
+        return {
+            accessKeyId:     process.env.OPENSEARCH_AWS_KEY,
+            secretAccessKey: process.env.OPENSEARCH_AWS_SECRET
+        };
+    }
+}
+
+
+module.exports = {
+    OpenSearchClient
+};