@openstax/ts-utils 1.34.0 → 1.35.0

package/dist/cjs/services/searchProvider/memorySearchTheBadWay.js

@@ -0,0 +1,191 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.memorySearchTheBadWay = void 0;
+const __1 = require("../..");
+const errors_1 = require("../../errors");
+const guards_1 = require("../../guards");
+const MIN_MATCH = 0.5;
+const MAX_RESULTS = 10;
+var MatchType;
+(function (MatchType) {
+    MatchType[MatchType["Must"] = 0] = "Must";
+    MatchType[MatchType["MustNot"] = 1] = "MustNot";
+    MatchType[MatchType["Should"] = 2] = "Should";
+})(MatchType || (MatchType = {}));
+const resolveField = (context, field) => {
+    const path = field.startsWith(context.scope + '.')
+        ? field.slice(context.scope.length + 1)
+        : field;
+    return path.split('.').reduce((result, key) => {
+        if (result === undefined) {
+            throw new Error(`Key "${key}" not found in document resolving ${field}`);
+        }
+        return result[key];
+    }, context.document);
+};
+const getFieldDefinition = (context, field) => {
+    let current = context.fields;
+    const parts = field.split('.');
+    const head = parts.slice(0, -1);
+    const tail = parts[parts.length - 1];
+    for (const part of head) {
+        const currentDefinition = current[part];
+        if (!currentDefinition || !('properties' in currentDefinition)) {
+            throw new Error(`Field "${field}" not found in mappings`);
+        }
+        current = currentDefinition.properties;
+    }
+    if (!current[tail]) {
+        throw new Error(`Field "${field}" not found in mappings`);
+    }
+    return current[tail];
+};
+const getFieldType = (context, field) => {
+    const definition = getFieldDefinition(context, field);
+    return definition.type;
+};
+const matchNested = (context, nested) => {
+    const nestedPath = nested.path;
+    const nestedDocuments = resolveField(context, nestedPath);
+    if (!Array.isArray(nestedDocuments)) {
+        throw new errors_1.InvalidRequestError(`Nested path "${nestedPath}" is not an array`);
+    }
+    return nestedDocuments.some(nestedDocument => matchClause({
+        ...context,
+        scope: nestedPath,
+        document: nestedDocument
+    }, nested.query));
+};
+const matchExists = (context, exists) => {
+    const value = resolveField(context, exists.field);
+    return value !== undefined && value !== null;
+};
+const matchTerms = (context, terms) => {
+    for (const key in terms) {
+        const docValues = (0, __1.coerceArray)(resolveField(context, key));
+        const coerceValue = getFieldType(context, key) === 'boolean'
+            ? (input) => {
+                if ([true, 'true', '1', 1].includes(input)) {
+                    return true;
+                }
+                if ([false, 'false', '0', 0, ''].includes(input)) {
+                    return false;
+                }
+                throw new errors_1.InvalidRequestError('input is not a valid boolean filter');
+            }
+            : (x) => x;
+        const hasMatch = (0, __1.coerceArray)(terms[key]).map(coerceValue).some(v => docValues.includes(v));
+        if (!hasMatch) {
+            return false;
+        }
+    }
+    return true;
+};
+const matchClause = (context, clause) => {
+    const filter = ('key' in clause && 'value' in clause)
+        ? { terms: { [clause.key]: clause.value } }
+        : clause;
+    if ('terms' in filter)
+        return matchTerms(context, filter.terms);
+    else if ('term' in filter)
+        return matchTerms(context, filter.term);
+    else if ('exists' in filter)
+        return matchExists(context, filter.exists);
+    else if ('bool' in filter)
+        return matchBool(context, filter.bool);
+    else if ('nested' in filter)
+        return matchNested(context, filter.nested);
+    else
+        throw new errors_1.InvalidRequestError('invalid filter type');
+};
+const matchFilters = (context, filters, matchType) => {
+    for (const field of filters) {
+        const hasMatch = matchClause(context, field);
+        if ((matchType === MatchType.Must && !hasMatch) || (matchType === MatchType.MustNot && hasMatch)) {
+            return false;
+        }
+        else if (matchType === MatchType.Should && hasMatch) {
+            return true;
+        }
+    }
+    return matchType !== MatchType.Should;
+};
+const matchBool = (context, filters) => {
+    const { must_not, should, must, filter } = filters;
+    if ((must === null || must === void 0 ? void 0 : must.length) && !matchFilters(context, must, MatchType.Must)) {
+        return false;
+    }
+    if ((filter === null || filter === void 0 ? void 0 : filter.length) && !matchFilters(context, filter, MatchType.Must)) {
+        return false;
+    }
+    if ((must_not === null || must_not === void 0 ? void 0 : must_not.length) && !matchFilters(context, must_not, MatchType.MustNot)) {
+        return false;
+    }
+    if ((should === null || should === void 0 ? void 0 : should.length) && !matchFilters(context, should, MatchType.Should)) {
+        return false;
+    }
+    return true;
+};
+const memorySearchTheBadWay = () => ({ store, mappings }) => {
+    return {
+        // This method is intentionally stubbed because index deletion is not applicable for in-memory storage.
+        deleteIndexIfExists: async () => undefined,
+        ensureIndexCreated: async () => undefined,
+        index: async (_options) => undefined,
+        bulkIndex: async (_items) => undefined,
+        search: async (options) => {
+            const results = (await store.loadAllDocumentsTheBadWay())
+                .map(document => {
+                let weight = 0;
+                if (options.query !== undefined) {
+                    for (const field of options.fields) {
+                        if (field.type !== undefined && field.type !== 'text') {
+                            continue;
+                        }
+                        const value = resolveField({ document, fields: mappings, scope: '' }, field.key);
+                        if (value === undefined || value === null) {
+                            continue;
+                        }
+                        if (typeof value === 'string') {
+                            if (value.toUpperCase().indexOf(options.query.toUpperCase()) !== -1) {
+                                weight += (1 * field.weight);
+                            }
+                        }
+                        else {
+                            throw new Error(`invalid value type when searching in field ${field.key.toString()}`);
+                        }
+                    }
+                }
+                if (!matchBool({ document, fields: mappings, scope: '' }, options)) {
+                    return undefined;
+                }
+                return { document, weight };
+            })
+                .filter(guards_1.isDefined)
+                .filter(r => !options.query || r.weight >= MIN_MATCH);
+            results.sort((a, b) => {
+                for (const sort of (options.sort || [])) {
+                    const aValue = resolveField({ document: a.document, fields: mappings, scope: '' }, sort.key);
+                    const bValue = resolveField({ document: b.document, fields: mappings, scope: '' }, sort.key);
+                    if (aValue < bValue) {
+                        return sort.order === 'asc' ? -1 : 1;
+                    }
+                    if (aValue > bValue) {
+                        return sort.order === 'asc' ? 1 : -1;
+                    }
+                }
+                return b.weight - a.weight;
+            });
+            const page = options.page || 1;
+            const offset = (page - 1) * MAX_RESULTS;
+            return {
+                items: results.slice(offset, offset + MAX_RESULTS).map(r => r.document),
+                pageSize: MAX_RESULTS,
+                currentPage: page,
+                totalItems: results.length,
+                totalPages: Math.ceil(results.length / MAX_RESULTS)
+            };
+        },
+    };
+};
+exports.memorySearchTheBadWay = memorySearchTheBadWay;

package/dist/cjs/services/searchProvider/openSearch.d.ts

@@ -0,0 +1,28 @@
+import { ConfigProviderForConfig } from '../../config';
+import { FieldMappings, IndexOptions, SearchOptions } from '.';
+export type Config = {
+    node: string;
+    region: string;
+};
+export interface Initializer<C> {
+    configSpace?: C;
+}
+export type IndexConfig = {
+    name: string;
+    mappings: FieldMappings;
+    pageSize?: number;
+};
+export declare const openSearchService: <C extends string = "deployed">(initializer?: Initializer<C>) => (configProvider: { [_key in C]: ConfigProviderForConfig<Config>; }) => <T>(indexConfig: IndexConfig) => {
+    ensureIndexCreated: () => Promise<void>;
+    deleteIndexIfExists: () => Promise<void>;
+    index: (params: IndexOptions<T>) => Promise<void>;
+    bulkIndex: (items: IndexOptions<T>[]) => Promise<void>;
+    search: (options: SearchOptions) => Promise<{
+        items: Exclude<T, undefined>[];
+        pageSize: number;
+        currentPage: number;
+        totalItems: number;
+        totalPages: number;
+    }>;
+    updateMappings: () => Promise<void>;
+};

package/dist/cjs/services/searchProvider/openSearch.js

@@ -0,0 +1,162 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.openSearchService = void 0;
+// cspell:ignore opensearch, Sigv
+const credential_provider_node_1 = require("@aws-sdk/credential-provider-node");
+const opensearch_1 = require("@opensearch-project/opensearch");
+const aws_1 = require("@opensearch-project/opensearch/aws");
+const config_1 = require("../../config");
+const guards_1 = require("../../guards");
+const helpers_1 = require("../../misc/helpers");
+const mapFilter = (filter) => {
+    if ('key' in filter && 'value' in filter) {
+        const { key } = filter;
+        const values = filter.value instanceof Array ? filter.value : [filter.value];
+        return { terms: { [key]: values } };
+    }
+    else {
+        return filter;
+    }
+};
+const openSearchService = (initializer = {}) => (configProvider) => {
+    const config = configProvider[(0, guards_1.ifDefined)(initializer.configSpace, 'deployed')];
+    const client = (0, helpers_1.once)(async () => new opensearch_1.Client({
+        ...(0, aws_1.AwsSigv4Signer)({
+            getCredentials: () => (0, credential_provider_node_1.defaultProvider)()(),
+            region: await (0, config_1.resolveConfigValue)(config.region),
+            service: 'es',
+        }),
+        maxRetries: 4, // default is 3
+        requestTimeout: 5000, // default is 30000
+        pingTimeout: 2000, // default is 30000
+        node: await (0, config_1.resolveConfigValue)(config.node),
+    }));
+    return (indexConfig) => {
+        const pageSize = indexConfig.pageSize || 10;
+        const deleteIndexIfExists = async () => {
+            const { indices } = await client();
+            const index = indexConfig.name;
+            const { body } = await indices.exists({ index });
+            if (body) {
+                await indices.delete({ index });
+            }
+        };
+        const ensureIndexCreated = async () => {
+            const { indices } = await client();
+            const index = indexConfig.name;
+            const { body } = await indices.exists({ index });
+            if (!body) {
+                await indices.create({
+                    index,
+                    body: {
+                        mappings: {
+                            dynamic: false,
+                            properties: indexConfig.mappings
+                        }
+                    }
+                });
+            }
+        };
+        const index = async (params) => {
+            const openSearchClient = await client();
+            await openSearchClient.index({
+                index: indexConfig.name,
+                body: params.body,
+                id: params.id,
+                refresh: true,
+                version: params.version,
+                version_type: params.version_type
+            }, {
+                requestTimeout: 10000,
+                maxRetries: 1,
+            });
+        };
+        const bulkIndex = async (items) => {
+            const openSearchClient = await client();
+            await openSearchClient.bulk({
+                index: indexConfig.name,
+                body: items.flatMap((item) => [
+                    {
+                        index: {
+                            _id: item.id,
+                            ...(item.version !== undefined ? { version: item.version } : {}),
+                            ...(item.version_type !== undefined ? { version_type: item.version_type } : {})
+                        }
+                    },
+                    item.body
+                ]),
+                refresh: true
+            }, {
+                requestTimeout: 10000,
+                maxRetries: 1,
+            });
+        };
+        const search = async (options) => {
+            var _a;
+            const body = {
+                query: { bool: {} },
+                track_total_hits: true,
+                size: pageSize
+            };
+            if (options.query) {
+                body.query.bool.must = [{
+                        multi_match: {
+                            fields: options.fields.map((field) => 'weight' in field ? `${field.key}^${field.weight}` : field.key),
+                            query: options.query
+                        }
+                    }];
+            }
+            const { must_not, should, must, filter } = options;
+            if (filter && filter.length > 0) {
+                body.query.bool.filter = filter.map(mapFilter);
+            }
+            if (must && must.length > 0) {
+                (_a = body.query.bool).must || (_a.must = []);
+                body.query.bool.must = body.query.bool.must.concat(must.map(mapFilter));
+            }
+            if (must_not && must_not.length > 0) {
+                body.query.bool.must_not = must_not.map(mapFilter);
+            }
+            if (should && should.length > 0) {
+                body.query.bool.should = should.map(mapFilter);
+                body.query.bool.minimum_should_match = 1;
+            }
+            if (options.sort && options.sort.length > 0) {
+                body.sort = options.sort.map(sort => ({
+                    [sort.key]: { order: sort.order }
+                }));
+            }
+            if (options.page) {
+                body.from = (options.page - 1) * pageSize;
+            }
+            const response = await (await client()).search({
+                body,
+                index: indexConfig.name
+            });
+            if (response.statusCode !== 200) {
+                throw new Error(`Unexpected status code: ${response.statusCode} from OpenSearch`);
+            }
+            const hits = response.body.hits;
+            const items = hits.hits.map((hit) => hit._source).filter(guards_1.isDefined);
+            const currentPage = options.page || 1;
+            const { total } = hits;
+            const totalItems = typeof total === 'number' ? total : total.value;
+            const totalPages = Math.ceil(totalItems / pageSize) || 1;
+            return { items, pageSize, currentPage, totalItems, totalPages };
+        };
+        const updateMappings = async () => {
+            const { indices } = await client();
+            await indices.putMapping({
+                index: indexConfig.name,
+                body: {
+                    properties: indexConfig.mappings
+                }
+            }, {
+                requestTimeout: 10000,
+                maxRetries: 1,
+            });
+        };
+        return { ensureIndexCreated, deleteIndexIfExists, index, bulkIndex, search, updateMappings };
+    };
+};
+exports.openSearchService = openSearchService;

package/dist/cjs/services/searchProvider/streamIndexer.d.ts

@@ -0,0 +1,17 @@
+import { DynamoDBRecord } from 'aws-lambda';
+import { Logger } from '../logger';
+import { IndexOptions } from '../searchProvider';
+export interface SearchProvider<T> {
+    index: (options: IndexOptions<T>) => Promise<void>;
+    bulkIndex: (items: IndexOptions<T>[]) => Promise<void>;
+}
+export interface SearchIndexUpdaterConfig<T> {
+    searchProvider: SearchProvider<T>;
+    logger: Logger;
+    getDocumentId: (document: T) => string;
+    getDocumentVersion?: (document: T) => number | undefined;
+}
+/**
+ * Function to update search index from DynamoDB Stream records.
+ */
+export declare const updateSearchIndexFromStream: <T>(records: DynamoDBRecord[], config: SearchIndexUpdaterConfig<T>) => Promise<void>;

package/dist/cjs/services/searchProvider/streamIndexer.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.updateSearchIndexFromStream = void 0;
+const util_dynamodb_1 = require("@aws-sdk/util-dynamodb");
+const logger_1 = require("../logger");
+const VALID_EVENTS = new Set(['INSERT', 'MODIFY']);
+/**
+ * Function to update search index from DynamoDB Stream records.
+ */
+const updateSearchIndexFromStream = async (records, config) => {
+    var _a, _b;
+    const { searchProvider, logger, getDocumentId, getDocumentVersion } = config;
+    const itemsToIndex = [];
+    for (const record of records) {
+        if (!record.eventName || !VALID_EVENTS.has(record.eventName)) {
+            logger.log(`Skipping ${record.eventName || 'undefined'} event`, logger_1.Level.Error);
+            continue;
+        }
+        if (!((_a = record.dynamodb) === null || _a === void 0 ? void 0 : _a.NewImage)) {
+            logger.log('Missing NewImage in record, skipping', logger_1.Level.Error);
+            continue;
+        }
+        const document = (0, util_dynamodb_1.unmarshall)(record.dynamodb.NewImage);
+        const documentId = getDocumentId(document);
+        const indexOptions = {
+            id: documentId,
+            body: document,
+        };
+        const version = (_b = getDocumentVersion === null || getDocumentVersion === void 0 ? void 0 : getDocumentVersion(document)) !== null && _b !== void 0 ? _b : record.dynamodb.ApproximateCreationDateTime;
+        if (version !== undefined) {
+            indexOptions.version = version;
+            indexOptions.version_type = 'external';
+        }
+        itemsToIndex.push(indexOptions);
+    }
+    if (itemsToIndex.length > 0) {
+        await searchProvider.bulkIndex(itemsToIndex);
+        logger.log(`Successfully indexed ${itemsToIndex.length} documents`);
+    }
+};
+exports.updateSearchIndexFromStream = updateSearchIndexFromStream;