@adobe/spacecat-shared-data-access 1.60.1 → 1.60.2

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-data-access-v1.60.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.60.1...@adobe/spacecat-shared-data-access-v1.60.2) (2024-12-19)
+
+
+### Bug Fixes
+
+* reduce number of indexes ([#499](https://github.com/adobe/spacecat-shared/issues/499)) ([95601cc](https://github.com/adobe/spacecat-shared/commit/95601cca8ca37f989a650fa841e0dae678cebc25))
+
 # [@adobe/spacecat-shared-data-access-v1.60.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.60.0...@adobe/spacecat-shared-data-access-v1.60.1) (2024-12-18)
 
 

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "1.60.1",
+  "version": "1.60.2",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {
     "node": ">=20.0.0 <23.0.0",
-    "npm": ">=10.0.0 <11.0.0"
+    "npm": ">=10.0.0 <12.0.0"
   },
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/v2/models/api-key/api-key.schema.js

@@ -69,12 +69,10 @@ const schema = new SchemaBuilder(ApiKey, ApiKeyCollection)
     },
   })
   .addIndex(
-    'byHashedApiKey',
     { composite: ['hashedApiKey'] },
     { composite: ['updatedAt'] },
   )
   .addIndex(
-    'byImsOrgIdAndImsUserId',
     { composite: ['imsOrgId', 'imsUserId'] },
     { composite: ['updatedAt'] },
   );

package/src/v2/models/base/base.collection.js

@@ -24,7 +24,6 @@ import { guardId } from '../../util/guards.js';
 import {
   entityNameToAllPKValue,
   isNonEmptyArray,
-  keyNamesToIndexName,
   removeElectroProperties,
 } from '../../util/util.js';
 import { INDEX_TYPES } from './constants.js';
@@ -40,26 +39,24 @@ function isValidParent(parent, child) {
 }
 
 /**
- * Attempts to find an index name matching a generated name from the given keyNames.
- * If no exact match is found, it progressively shortens the keyNames by removing the last one
- * and tries again. If still no match, it tries the "all" index, and then "primary".
- *
- * @param {object} indexes - The available indexes, keyed by their names.
- * @param {object} keys - The keys to find an index name for.
- * @returns {object} The found index.
+ * Finds the index name by the keys provided. The index is searched
+ * keys to match the combination of partition and sort keys. If no
+ * index is found, we fall back to the "all" index, then the "primary".
+ * @param {Schema} schema - The schema to search for the index.
+ * @param {Object} keys - The keys to search for.
+ * @return {*|string} - The index name.
  */
-function findIndexNameByKeys(indexes, keys) {
+function findIndexNameByKeys(schema, keys) {
   const keyNames = Object.keys(keys);
-  for (let { length } = keyNames; length > 0; length -= 1) {
-    const subKeyNames = keyNames.slice(0, length);
-    const candidateName = keyNamesToIndexName(subKeyNames);
-    if (indexes[candidateName]) {
-      return candidateName;
-    }
+
+  const index = schema.findIndexBySortKeys(keyNames);
+  if (index) {
+    return index.index;
   }
 
-  if (indexes.all) {
-    return INDEX_TYPES.ALL;
+  const allIndex = schema.findIndexByType(INDEX_TYPES.ALL);
+  if (allIndex) {
+    return allIndex.index;
   }
 
   return INDEX_TYPES.PRIMARY;
@@ -176,11 +173,11 @@ class BaseCollection {
       throw new Error(message);
     }
 
-    const indexName = options.index || findIndexNameByKeys(this.entity.query, keys);
+    const indexName = options.index || findIndexNameByKeys(this.schema, keys);
     const index = this.entity.query[indexName];
 
     if (!index) {
-      const message = `Failed to query [${this.entityName}]: index [${indexName}] not found`;
+      const message = `Failed to query [${this.entityName}]: query proxy [${indexName}] not found`;
       this.log.error(message);
       throw new Error(message);
     }
@@ -243,7 +240,8 @@ class BaseCollection {
    * Finds a single entity from the "all" index. Requires an index named "all" with a partition key
    * named "pk" with a static value of "ALL_<ENTITYNAME>".
    * @param {Object} [sortKeys] - The sort keys to use for the query.
-   * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query.
+   * @param {{index?: string, attributes?: string[], order?: string}} [options] -
+   * Additional options for the query.
    * @return {Promise<BaseModel|Array<BaseModel>|null>}
    */
   async findByAll(sortKeys = {}, options = {}) {
@@ -254,7 +252,7 @@ class BaseCollection {
     }
 
     const keys = { pk: entityNameToAllPKValue(this.entityName), ...sortKeys };
-    return this.#queryByIndexKeys(keys, { ...options, index: INDEX_TYPES.ALL, limit: 1 });
+    return this.#queryByIndexKeys(keys, { ...options, limit: 1 });
   }
 
   /**

package/src/v2/models/base/index.d.ts

@@ -58,12 +58,20 @@ export interface Reference {
   isRemoveDependents(): boolean;
 }
 
+export interface IndexAccessor {
+  indexName: string;
+  keySets: string[][];
+}
+
 export interface Schema {
+  findIndexBySortKeys(sortKeys: string[]): object | null;
+  findIndexByType(type: string): object | null;
   getAttribute(name: string): object;
   getAttributes(): object;
   getCollectionName(): string;
   getEntityName(): string;
   getIdName(): string;
+  getIndexAccessors(): Array<IndexAccessor>;
   getIndexes(): object;
   getIndexKeys(indexName: string): string[];
   getModelClass(): object;
@@ -75,8 +83,7 @@ export interface Schema {
 
 export interface SchemaBuilder {
   addAttribute(name: string, data: object): SchemaBuilder;
-  addAllIndexWithComposite(...attributeNames: string[]): SchemaBuilder
-  addAllIndexWithTemplateField(fieldName: string, template: string): SchemaBuilder;
+  addAllIndex(sortKeys: string[]): SchemaBuilder;
   addIndex(name: string, partitionKey: object, sortKey: object): SchemaBuilder;
   addReference(referenceType: string, entityName: string, sortKeys?: string[]): SchemaBuilder;
   build(): Schema;

package/src/v2/models/base/schema.builder.js

@@ -15,10 +15,10 @@ import { hasText, isInteger, isNonEmptyObject } from '@adobe/spacecat-shared-uti
 import { v4 as uuid, validate as uuidValidate } from 'uuid';
 
 import {
-  capitalize,
   decapitalize,
   entityNameToAllPKValue,
-  entityNameToIdName, isNonEmptyArray,
+  entityNameToIdName,
+  isNonEmptyArray,
 } from '../../util/util.js';
 
 import { INDEX_TYPES } from './constants.js';
@@ -70,52 +70,6 @@ const UPDATED_AT_ATTRIBUTE_DATA = {
   set: () => new Date().toISOString(),
 };
 
-/** Certain index names (primary, all) are reserved and cannot be reused. */
-const RESERVED_INDEX_NAMES = [INDEX_TYPES.PRIMARY, INDEX_TYPES.ALL];
-
-/**
- * Constructs a fully qualified index name.
- * @param {string} service - The name of the service.
- * @param {string} entity - The name of the entity.
- * @param {string} name - The index name (e.g., 'all', 'byForeignKey').
- * @returns {string} The fully qualified index name.
- */
-const createdIndexName = (service, entity, name) => `${service.toLowerCase()}-data-${entity}-${name}`;
-
-/**
- * Sorts an indexes object by its keys alphabetically.
- * @param {object} indexes - An object whose keys are index names and values are index definitions.
- * @returns {object} A new object with the same entries, but keys sorted alphabetically.
- */
-const sortIndexes = (indexes) => Object.fromEntries(
-  Object.entries(indexes).sort((a, b) => a[0].localeCompare(b[0])),
-);
-
-/**
- * Assigns GSI field names to indexes that don't have them yet.
- * Ensures that if an "all" index exists, it uses gsi1 (already assigned)
- * and other indexes continue numbering from gsi2 onwards.
- *
- * @param {object} indexes - Object of indexes that require naming.
- * @param {object|null} all - The "all" index object if present, null otherwise.
- */
-const numberGSIsIndexes = (indexes, all) => {
-  // if there's an "all" index, we start indexing subsequent GSIs from 2,
-  // because "all" index already occupies gsi1.
-  // if no "all" index exists, start from 1.
-  let gsiCounter = isNonEmptyObject(all) ? 1 : 0;
-
-  Object.values(indexes).forEach((index) => { /* eslint-disable no-param-reassign */
-    // only assign new field names and number through if none are provided.
-    if (!index.pk.field || !index.sk.field) {
-      gsiCounter += 1;
-    }
-
-    index.pk.field = index.pk.field || `gsi${gsiCounter}pk`;
-    index.sk.field = index.sk.field || `gsi${gsiCounter}sk`;
-  });
-};
-
 /**
  * The SchemaBuilder class allows for constructing a schema definition
  * including attributes, indexes, and references to other entities.
@@ -159,9 +113,9 @@ class SchemaBuilder {
 
     this.rawIndexes = {
       primary: null,
-      all: null,
-      belongs_to: {},
-      other: {},
+      all: [],
+      belongs_to: [],
+      other: [],
     };
 
     this.attributes = {};
@@ -189,16 +143,14 @@ class SchemaBuilder {
     };
   }
 
-  #internalAddIndex(name, partitionKey, sortKey, type) {
-    const indexFullName = createdIndexName(this.serviceName, this.entityName, name);
-
+  #internalAddIndex(partitionKey, sortKey, type) {
     // store index config without assigning fields yet
     // the fields will be assigned in build phase based on sorting and presence of "all" index
-    this.rawIndexes[type][name] = {
-      ...(indexFullName && { index: indexFullName }),
+    this.rawIndexes[type].push({
+      type,
       pk: { ...partitionKey },
       sk: { ...sortKey },
-    };
+    });
   }
 
   /**
@@ -224,52 +176,24 @@ class SchemaBuilder {
   }
 
   /**
-   * Adds an "all" index based on composite attributes.
-   * The "all" index is a special index listing all entities, sorted by given attributes.
-   * Useful for global queries across all entities of this type.
-   * Will overwrite any existing "all" index.
+   * Adds an "all" index with composite partition and sort keys, or a template-based sort key.
+   * Useful for querying all entities of this type. Only one "all" index is allowed and a
+   * pre-existing "all" index will be overwritten.
    *
-   * @param {...string} attributeNames - The attribute names forming the composite sort key.
+   * @param {Array<string>} sortKeys - The attributes to form the sort key.
    * @returns {SchemaBuilder} Returns this builder for method chaining.
-   * @throws {Error} If no attribute names are provided.
+   * @throws {Error} If composite attribute names or template are not provided.
    */
-  addAllIndexWithComposite(...attributeNames) {
-    if (attributeNames.length === 0) {
-      throw new Error('At least one composite attribute name is required.');
+  addAllIndex(sortKeys) {
+    if (!isNonEmptyArray(sortKeys)) {
+      throw new Error('Sort keys are required and must be a non-empty array.');
     }
 
-    this.rawIndexes.all = {
-      index: createdIndexName(this.serviceName, this.entityName, INDEX_TYPES.ALL),
-      pk: { field: 'gsi1pk', template: entityNameToAllPKValue(this.entityName) },
-      sk: { field: 'gsi1sk', composite: attributeNames },
-    };
-
-    return this;
-  }
-
-  /**
-   * Adds an "all" index with a template-based sort key.
-   * Useful if a single value template defines how entries are sorted.
-   *
-   * @param {string} fieldName - The sort key field name.
-   * @param {string} template - A template string defining how to generate the sort key value.
-   * @returns {SchemaBuilder} Returns this builder for method chaining.
-   * @throws {Error} If fieldName or template are not valid strings.
-   */
-  addAllIndexWithTemplateField(fieldName, template) {
-    if (!hasText(fieldName)) {
-      throw new Error('fieldName is required and must be a non-empty string.');
-    }
-
-    if (!hasText(template)) {
-      throw new Error('template is required and must be a non-empty string.');
-    }
-
-    this.rawIndexes.all = {
-      index: createdIndexName(this.serviceName, this.entityName, 'all'),
-      pk: { field: 'gsi1pk', template: entityNameToAllPKValue(this.entityName) },
-      sk: { field: fieldName, template },
-    };
+    this.#internalAddIndex(
+      { template: entityNameToAllPKValue(this.entityName) },
+      { composite: sortKeys },
+      INDEX_TYPES.ALL,
+    );
 
     return this;
   }
@@ -277,22 +201,13 @@ class SchemaBuilder {
   /**
    * Adds a generic secondary index (GSI).
    *
-   * @param {string} name - The index name. Cannot be 'primary' or 'all'.
    * @param {object} partitionKey - The partition key definition
    * (e.g., { composite: [attributeName] }).
    * @param {object} sortKey - The sort key definition.
    * @returns {SchemaBuilder} Returns this builder for method chaining.
    * @throws {Error} If index name is reserved or pk/sk configs are invalid.
    */
-  addIndex(name, partitionKey, sortKey) {
-    if (!hasText(name)) {
-      throw new Error('Index name is required and must be a non-empty string.');
-    }
-
-    if (RESERVED_INDEX_NAMES.includes(name)) {
-      throw new Error(`Index name "${name}" is reserved.`);
-    }
-
+  addIndex(partitionKey, sortKey) {
     if (!isNonEmptyObject(partitionKey)) {
       throw new Error('Partition key configuration (pk) is required and must be a non-empty object.');
     }
@@ -301,7 +216,7 @@ class SchemaBuilder {
       throw new Error('Sort key configuration (sk) is required and must be a non-empty object.');
     }
 
-    this.#internalAddIndex(name, partitionKey, sortKey, INDEX_TYPES.OTHER);
+    this.#internalAddIndex(partitionKey, sortKey, INDEX_TYPES.OTHER);
 
     return this;
   }
@@ -357,7 +272,6 @@ class SchemaBuilder {
       });
 
       this.#internalAddIndex(
-        `by${capitalize(foreignKeyName)}`,
         { composite: [decapitalize(foreignKeyName)] },
         { composite: isNonEmptyArray(sortKeys) ? sortKeys : ['updatedAt'] },
         INDEX_TYPES.BELONGS_TO,
@@ -378,21 +292,37 @@ class SchemaBuilder {
    */
   #buildIndexes() {
     // eslint-disable-next-line camelcase
-    const { belongs_to, other } = this.rawIndexes;
+    const { all, belongs_to, other } = this.rawIndexes;
+
+    // set the order of indexes
+    const orderedIndexes = [
+      ...all,
+      // eslint-disable-next-line camelcase
+      ...belongs_to,
+      ...other,
+    ];
+
+    if (orderedIndexes.length > 5) {
+      throw new Error('Cannot have more than 5 indexes.');
+    }
 
-    // belongs_to indexes come before other indexes
-    const indexes = {
-      ...sortIndexes(belongs_to),
-      ...sortIndexes(other),
-    };
+    this.indexes = { primary: this.rawIndexes.primary };
 
-    numberGSIsIndexes(indexes, this.rawIndexes.all);
+    let indexCounter = 0;
+    Object.values(orderedIndexes).forEach((index) => {
+      indexCounter += 1;
 
-    this.indexes = {
-      primary: this.rawIndexes.primary,
-      ...(this.rawIndexes.all && { all: this.rawIndexes.all }),
-      ...indexes,
-    };
+      const pkFieldName = `gsi${indexCounter}pk`;
+      const skFieldName = `gsi${indexCounter}sk`;
+      const indexName = `${this.serviceName.toLowerCase()}-data-${pkFieldName}-${skFieldName}`;
+
+      this.indexes[indexName] = {
+        index: indexName,
+        indexType: index.type,
+        pk: { field: pkFieldName, ...index.pk },
+        sk: { field: skFieldName, ...index.sk },
+      };
+    });
   }
 
   /**

package/src/v2/models/base/schema.js

@@ -140,6 +140,31 @@ class Schema {
     return this.indexes[indexName];
   }
 
+  findIndexBySortKeys(sortKeys) {
+    // find index that has same sort keys, then remove the last sort key
+    // and find the index that has the remaining sort keys, etc.
+    for (let { length } = sortKeys; length > 0; length -= 1) {
+      const subKeyNames = sortKeys.slice(0, length);
+      const index = Object.values(this.indexes).find((candidate) => {
+        const { pk, sk } = candidate;
+        const allKeys = [...(pk?.facets || []), ...(sk?.facets || [])];
+
+        // check if all keys in the index are in the sort keys
+        return subKeyNames.every((key) => allKeys.includes(key));
+      });
+
+      if (isNonEmptyObject(index)) {
+        return index;
+      }
+    }
+
+    return null;
+  }
+
+  findIndexByType(type) {
+    return Object.values(this.indexes).find((index) => index.indexType === type) || null;
+  }
+
   /**
    * Returns the indexes for the schema. By default, this returns all indexes.
    * You can use the `exclude` parameter to exclude certain indexes.
@@ -170,7 +195,7 @@ class Schema {
     }
 
     const pkKeys = Array.isArray(index.pk?.facets) ? index.pk.facets : [];
-    const skKeys = Array.isArray(index.sk?.facets) ? index.sk.facets : [index.sk?.field];
+    const skKeys = Array.isArray(index.sk?.facets) ? index.sk.facets : [];
 
     return [...pkKeys, ...skKeys];
   }

package/src/v2/models/configuration/configuration.collection.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-import { incrementVersion, sanitizeIdAndAuditFields } from '../../util/util.js';
+import { incrementVersion, sanitizeIdAndAuditFields, zeroPad } from '../../util/util.js';
 import BaseCollection from '../base/base.collection.js';
 
 /**
@@ -31,6 +31,10 @@ class ConfigurationCollection extends BaseCollection {
     return super.create(sanitizedData);
   }
 
+  async findByVersion(version) {
+    return this.findByAll({ versionString: zeroPad(version, 10) });
+  }
+
   async findLatest() {
     return this.findByAll({}, { order: 'desc' });
   }

package/src/v2/models/configuration/configuration.schema.js

@@ -19,6 +19,7 @@ import Joi from 'joi';
 import SchemaBuilder from '../base/schema.builder.js';
 import Configuration from './configuration.model.js';
 import ConfigurationCollection from './configuration.collection.js';
+import { zeroPad } from '../../util/util.js';
 
 const handlerSchema = Joi.object().pattern(Joi.string(), Joi.object(
   {
@@ -97,7 +98,13 @@ const schema = new SchemaBuilder(Configuration, ConfigurationCollection)
     required: true,
     readOnly: true,
   })
-  // eslint-disable-next-line no-template-curly-in-string
-  .addAllIndexWithTemplateField('version', '${version}');
+  .addAttribute('versionString', { // used for indexing/sorting
+    type: 'string',
+    required: true,
+    readOnly: true,
+    default: '0', // setting the default forces set() to run, to transform the version number to a string
+    set: (value, all) => zeroPad(all.version, 10),
+  })
+  .addAllIndex(['versionString']);
 
 export default schema.build();

package/src/v2/models/import-job/import-job.schema.js

@@ -142,9 +142,8 @@ const schema = new SchemaBuilder(ImportJob, ImportJobCollection)
     default: 0,
     validate: (value) => !value || isInteger(value),
   })
-  .addAllIndexWithComposite('startedAt')
+  .addAllIndex(['startedAt'])
   .addIndex(
-    'byStatus',
     { composite: ['status'] },
     { composite: ['updatedAt'] },
   );

package/src/v2/models/organization/organization.schema.js

@@ -46,6 +46,6 @@ const schema = new SchemaBuilder(Organization, OrganizationCollection)
     type: 'any',
     validate: (value) => !value || isNonEmptyObject(value),
   })
-  .addAllIndexWithComposite('imsOrgId');
+  .addAllIndex(['imsOrgId']);
 
 export default schema.build();

package/src/v2/models/site/site.schema.js

@@ -81,9 +81,8 @@ const schema = new SchemaBuilder(Site, SiteCollection)
     set: () => new Date().toISOString(),
     validate: (value) => !value || isIsoDate(value),
   })
-  .addAllIndexWithComposite('baseURL')
+  .addAllIndex(['baseURL'])
   .addIndex(
-    'byDeliveryType',
     { composite: ['deliveryType'] },
     { composite: ['updatedAt'] },
   );

package/src/v2/models/site-candidate/site-candidate.schema.js

@@ -54,6 +54,6 @@ const schema = new SchemaBuilder(SiteCandidate, SiteCandidateCollection)
   .addAttribute('updatedBy', {
     type: 'string',
   })
-  .addAllIndexWithComposite('baseURL');
+  .addAllIndex(['baseURL']);
 
 export default schema.build();

package/src/v2/readme.md

@@ -135,7 +135,7 @@ const userSchema = new SchemaBuilder(User, UserCollection)
     validate: (value) => value.includes('@'),
   })
   .addAttribute('name', { type: 'string', required: true })
-  .addAllIndexWithComposite('email')
+  .addAllIndex(['email'])
   .addReference('belongs_to', 'Organization') // Adds organizationId and byOrganizationId index
   .build();
 

package/src/v2/util/util.js

@@ -82,6 +82,13 @@ const incrementVersion = (version) => (isInteger(version) ? parseInt(version, 10
 
 const isNonEmptyArray = (value) => Array.isArray(value) && value.length > 0;
 
+const zeroPad = (num, length) => {
+  const str = String(num);
+  return str.length >= length
+    ? str
+    : '0'.repeat(length - str.length) + str;
+};
+
 export {
   capitalize,
   classExtends,
@@ -101,4 +108,5 @@ export {
   removeElectroProperties,
   sanitizeIdAndAuditFields,
   sanitizeTimestamps,
+  zeroPad,
 };