@adobe/spacecat-shared-data-access 1.59.2 → 1.60.0

package/src/v2/models/audit/audit.model.js

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { isObject } from '@adobe/spacecat-shared-utils';
+
+import { ValidationError } from '../../errors/index.js';
+import BaseModel from '../base/base.model.js';
+
+const AUDIT_TYPES = {
+  404: '404',
+  BROKEN_BACKLINKS: 'broken-backlinks',
+  EXPERIMENTATION: 'experimentation',
+  ORGANIC_KEYWORDS: 'organic-keywords',
+  ORGANIC_TRAFFIC: 'organic-traffic',
+  CWV: 'cwv',
+  LHS_DESKTOP: 'lhs-desktop',
+  LHS_MOBILE: 'lhs-mobile',
+  EXPERIMENTATION_ESS_MONTHLY: 'experimentation-ess-monthly',
+  EXPERIMENTATION_ESS_DAILY: 'experimentation-ess-daily',
+};
+
+const AUDIT_TYPE_PROPERTIES = {
+  [AUDIT_TYPES.LHS_DESKTOP]: ['performance', 'seo', 'accessibility', 'best-practices'],
+  [AUDIT_TYPES.LHS_MOBILE]: ['performance', 'seo', 'accessibility', 'best-practices'],
+};
+
+export const AUDIT_CONFIG = {
+  TYPES: AUDIT_TYPES,
+  PROPERTIES: AUDIT_TYPE_PROPERTIES,
+};
+
+/**
+ * Validates if the auditResult contains the required properties for the given audit type.
+ * @param {object} auditResult - The audit result to validate.
+ * @param {string} auditType - The type of the audit.
+ * @returns {boolean} - True if valid, false otherwise.
+ */
+export const validateAuditResult = (auditResult, auditType) => {
+  if (!isObject(auditResult) && !Array.isArray(auditResult)) {
+    throw new ValidationError('Audit result must be an object or array');
+  }
+
+  if (isObject(auditResult.runtimeError)) {
+    return true;
+  }
+
+  if ((auditType === AUDIT_CONFIG.TYPES.LHS_MOBILE || auditType === AUDIT_CONFIG.TYPES.LHS_DESKTOP)
+    && !isObject(auditResult.scores)) {
+    throw new ValidationError(`Missing scores property for audit type '${auditType}'`);
+  }
+
+  const expectedProperties = AUDIT_CONFIG.PROPERTIES[auditType];
+
+  if (expectedProperties) {
+    for (const prop of expectedProperties) {
+      if (!(prop in auditResult.scores)) {
+        throw new ValidationError(`Missing expected property '${prop}' for audit type '${auditType}'`);
+      }
+    }
+  }
+
+  return true;
+};
+
+/**
+ * Audit - A class representing an Audit entity.
+ * Provides methods to access and manipulate Audit-specific data.
+ *
+ * @class Audit
+ * @extends BaseModel
+ */
+class Audit extends BaseModel {
+  // add your custom methods or overrides here
+
+  getScores() {
+    return this.getAuditResult()?.scores;
+  }
+}
+
+export default Audit;

package/src/v2/models/audit/audit.schema.js

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* c8 ignore start */
+
+import { isIsoDate, isNonEmptyObject } from '@adobe/spacecat-shared-utils';
+
+import SchemaBuilder from '../base/schema.builder.js';
+import Audit, { validateAuditResult } from './audit.model.js';
+import AuditCollection from './audit.collection.js';
+
+/*
+Schema Doc: https://electrodb.dev/en/modeling/schema/
+Attribute Doc: https://electrodb.dev/en/modeling/attributes/
+Indexes Doc: https://electrodb.dev/en/modeling/indexes/
+ */
+
+const schema = new SchemaBuilder(Audit, AuditCollection)
+  .addReference('belongs_to', 'Site', ['auditType', 'auditedAt'])
+  .addReference('has_many', 'Opportunities')
+  .addAttribute('auditResult', {
+    type: 'any',
+    required: true,
+    validate: (value) => isNonEmptyObject(value),
+    set: (value, attributes) => {
+      // as the electroDb validate function does not provide access to the model instance
+      // we need to call the validate function from the model on setting the value
+      validateAuditResult(value, attributes.auditType);
+      return value;
+    },
+  })
+  .addAttribute('auditType', {
+    type: 'string',
+    required: true,
+  })
+  .addAttribute('fullAuditRef', {
+    type: 'string',
+    required: true,
+  })
+  .addAttribute('isLive', {
+    type: 'boolean',
+    required: true,
+    default: false,
+  })
+  .addAttribute('isError', {
+    type: 'boolean',
+    required: true,
+    default: false,
+  })
+  .addAttribute('auditedAt', {
+    type: 'string',
+    required: true,
+    default: () => new Date().toISOString(),
+    validate: (value) => isIsoDate(value),
+  });
+
+export default schema.build();

package/src/v2/models/audit/index.d.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import type {
+  BaseCollection, BaseModel, Opportunity, Site,
+} from '../index';
+
+export interface Audit extends BaseModel {
+  getAuditResult(): object;
+  getAuditType(): string;
+  getAuditedAt(): number;
+  getFullAuditRef(): string;
+  getIsError(): boolean;
+  getIsLive(): boolean;
+  getOpportunities(): Promise<Opportunity[]>;
+  getSite(): Promise<Site>;
+  getSiteId(): string;
+  setAuditResult(auditResult: object): Audit;
+  setAuditType(auditType: string): Audit;
+  setAuditedAt(auditedAt: number): Audit;
+  setFullAuditRef(fullAuditRef: string): Audit;
+  setIsError(isError: boolean): Audit;
+  setIsLive(isLive: boolean): Audit;
+  setSiteId(siteId: string): Audit;
+  toggleLive(): Audit;
+}
+
+export interface AuditCollection extends BaseCollection<Audit> {
+  allBySiteId(siteId: string): Promise<Audit[]>;
+  allBySiteAndType(siteId: string, auditType: string): Promise<Audit[]>;
+}

package/src/v2/models/audit/index.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import Audit from './audit.model.js';
+import AuditCollection from './audit.collection.js';
+
+export {
+  Audit,
+  AuditCollection,
+};

package/src/v2/models/base/base.collection.js

@@ -0,0 +1,450 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import {
+  hasText,
+  isNonEmptyObject,
+  isObject,
+} from '@adobe/spacecat-shared-utils';
+
+import { ElectroValidationError } from 'electrodb';
+
+import { removeElectroProperties } from '../../../../test/it/util/util.js';
+import { createAccessors } from '../../util/accessor.utils.js';
+import ValidationError from '../../errors/validation.error.js';
+import { guardId } from '../../util/guards.js';
+import {
+  entityNameToAllPKValue,
+  isNonEmptyArray,
+  keyNamesToIndexName,
+} from '../../util/util.js';
+import { INDEX_TYPES } from './constants.js';
+
+function isValidParent(parent, child) {
+  if (!hasText(parent.entityName)) {
+    return false;
+  }
+
+  const foreignKey = `${parent.entityName}Id`;
+
+  return child.record?.[foreignKey] === parent.record?.[foreignKey];
+}
+
+/**
+ * Attempts to find an index name matching a generated name from the given keyNames.
+ * If no exact match is found, it progressively shortens the keyNames by removing the last one
+ * and tries again. If still no match, it tries the "all" index, and then "primary".
+ *
+ * @param {object} indexes - The available indexes, keyed by their names.
+ * @param {object} keys - The keys to find an index name for.
+ * @returns {object} The found index.
+ */
+function findIndexNameByKeys(indexes, keys) {
+  const keyNames = Object.keys(keys);
+  for (let { length } = keyNames; length > 0; length -= 1) {
+    const subKeyNames = keyNames.slice(0, length);
+    const candidateName = keyNamesToIndexName(subKeyNames);
+    if (indexes[candidateName]) {
+      return candidateName;
+    }
+  }
+
+  if (indexes.all) {
+    return INDEX_TYPES.ALL;
+  }
+
+  return INDEX_TYPES.PRIMARY;
+}
+
+/**
+ * BaseCollection - A base class for managing collections of entities in the application.
+ * This class uses ElectroDB to interact with entities and provides common functionality
+ * for data operations.
+ *
+ * @class BaseCollection
+ * @abstract
+ */
+class BaseCollection {
+  /**
+   * Constructs an instance of BaseCollection.
+   * @constructor
+   * @param {Object} electroService - The ElectroDB service used for managing entities.
+   * @param {Object} entityRegistry - The registry holding entities, their schema and collection.
+   * @param {Object} schema - The schema for the entity.
+   * @param {Object} log - A log for capturing logging information.
+   */
+  constructor(electroService, entityRegistry, schema, log) {
+    this.electroService = electroService;
+    this.entityRegistry = entityRegistry;
+    this.schema = schema;
+    this.log = log;
+
+    this.clazz = this.schema.getModelClass();
+    this.entityName = this.schema.getEntityName();
+    this.idName = this.schema.getIdName();
+    this.entity = electroService.entities[this.entityName];
+
+    this.#initializeCollectionMethods();
+  }
+
+  /**
+   * Initialize collection methods for each "by..." index defined in the entity schema.
+   * For each index that starts with "by", we:
+   *  1. Retrieve its composite pk and sk arrays from the schema.
+   *  2. Generate convenience methods for every prefix of the composite keys.
+   *     For example, if the index keys are ['opportunityId', 'status', 'createdAt'],
+   *     we create methods:
+   *       - allByOpportunityId(...) / findByOpportunityId(...)
+   *       - allByOpportunityIdAndStatus(...) / findByOpportunityIdAndStatus(...)
+   *       - allByOpportunityIdAndStatusAndCreatedAt(...) /
+   *            findByOpportunityIdAndStatusAndCreatedAt(...)
+   *
+   * Each generated method calls allByIndexKeys() or findByIndexKeys() with the appropriate keys.
+   *
+   * @private
+   */
+  #initializeCollectionMethods() {
+    const accessorConfigs = this.schema.toAccessorConfigs(this, this.log);
+    createAccessors(accessorConfigs, this.log);
+  }
+
+  /**
+   * Creates an instance of a model from a record.
+   * @private
+   * @param {Object} record - The record containing data to create the model instance.
+   * @returns {BaseModel|null} - Returns an instance of the model class if the data is valid,
+   * otherwise null.
+   */
+  #createInstance(record) {
+    if (!isNonEmptyObject(record)) {
+      this.log.warn(`Failed to create instance of [${this.entityName}]: record is empty`);
+      return null;
+    }
+    // eslint-disable-next-line new-cap
+    return new this.clazz(
+      this.electroService,
+      this.entityRegistry,
+      this.schema,
+      record,
+      this.log,
+    );
+  }
+
+  /**
+   * Creates instances of models from a set of records.
+   * @private
+   * @param {Object} records - The records containing data to create the model instances.
+   * @returns {Array<BaseModel>} - An array of instances of the model class.
+   */
+  #createInstances(records) {
+    return records.map((record) => this.#createInstance(record));
+  }
+
+  #invalidateCache() {
+    this._accessorCache = {};
+  }
+
+  /**
+   * General method to query entities by index keys. This method is used by other
+   * query methods to perform the actual query operation. It will use the index keys
+   * to find the appropriate index and query the entities. The query result will be
+   * transformed into model instances.
+   * @private
+   * @param {Object} keys - The index keys to use for the query.
+   * @param {Object} options - Additional options for the query.
+   * @returns {Promise<BaseModel|Array<BaseModel>|null>} - The query result.
+   */
+  async #queryByIndexKeys(keys, options = {}) {
+    if (!isNonEmptyObject(keys)) {
+      const message = `Failed to query [${this.entityName}]: keys are required`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    if (!isObject(options)) {
+      const message = `Failed to query [${this.entityName}]: options must be an object`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    const indexName = options.index || findIndexNameByKeys(this.entity.query, keys);
+    const index = this.entity.query[indexName];
+
+    if (!index) {
+      const message = `Failed to query [${this.entityName}]: index [${indexName}] not found`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    const queryOptions = {
+      order: options.order || 'desc',
+      ...options.limit && { limit: options.limit },
+      ...options.attributes && { attributes: options.attributes },
+    };
+
+    let query = index(keys);
+
+    if (isObject(options.between)) {
+      query = query.between(
+        { [options.between.attribute]: options.between.start },
+        { [options.between.attribute]: options.between.end },
+      );
+    }
+
+    const records = await query.go(queryOptions);
+
+    if (options.limit === 1) {
+      if (records.data?.length === 0) {
+        return null;
+      }
+      return this.#createInstance(records.data[0]);
+    } else {
+      return this.#createInstances(records.data);
+    }
+  }
+
+  /**
+   * Finds all entities in the collection. Requires an index named "all" with a partition key
+   * named "pk" with a static value of "ALL_<ENTITYNAME>".
+   * @param {Object} [sortKeys] - The sort keys to use for the query.
+   * @param {Object} [options] - Additional options for the query.
+   * @return {Promise<BaseModel|Array<BaseModel>|null>}
+   */
+  async all(sortKeys = {}, options = {}) {
+    const keys = { pk: entityNameToAllPKValue(this.entityName), ...sortKeys };
+    return this.#queryByIndexKeys(keys, options);
+  }
+
+  /**
+   * Finds entities by a set of index keys. Index keys are used to query entities by
+   * a specific index defined in the entity schema. The index keys must match the
+   * fields defined in the index.
+   * @param {Object} keys - The index keys to use for the query.
+   * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query.
+   * @return {Promise<Array<BaseModel>>} - A promise that resolves to an array of model instances.
+   * @throws {Error} - Throws an error if the index keys are not provided or if the index
+   * is not found.
+   * @async
+   */
+  async allByIndexKeys(keys, options = {}) {
+    return this.#queryByIndexKeys(keys, options);
+  }
+
+  /**
+   * Finds a single entity from the "all" index. Requires an index named "all" with a partition key
+   * named "pk" with a static value of "ALL_<ENTITYNAME>".
+   * @param {Object} [sortKeys] - The sort keys to use for the query.
+   * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query.
+   * @return {Promise<BaseModel|Array<BaseModel>|null>}
+   */
+  async findByAll(sortKeys = {}, options = {}) {
+    if (!isObject(sortKeys)) {
+      const message = `Failed to find by all [${this.entityName}]: sort keys must be an object`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    const keys = { pk: entityNameToAllPKValue(this.entityName), ...sortKeys };
+    return this.#queryByIndexKeys(keys, { ...options, index: INDEX_TYPES.ALL, limit: 1 });
+  }
+
+  /**
+   * Finds an entity by its ID.
+   * @async
+   * @param {string} id - The unique identifier of the entity to be found.
+   * @returns {Promise<BaseModel|null>} - A promise that resolves to an instance of
+   * the model if found, otherwise null.
+   * @throws {Error} - Throws an error if the ID is not provided.
+   */
+  async findById(id) {
+    guardId(this.idName, id, this.entityName);
+
+    const record = await this.entity.get({ [this.idName]: id }).go();
+
+    return this.#createInstance(record?.data);
+  }
+
+  /**
+   * Finds a single entity by index keys.
+   * @param {Object} keys - The index keys to use for the query.
+   * @param {{index?: string, attributes?: string[]}} [options] - Additional options for the query.
+   * @returns {Promise<BaseModel|null>} - A promise that resolves to the model instance or null.
+   * @async
+   */
+  async findByIndexKeys(keys, options = {}) {
+    return this.#queryByIndexKeys(keys, { ...options, limit: 1 });
+  }
+
+  /**
+   * Creates a new entity in the collection and directly persists it to the database.
+   * There is no need to call the save method (which is for updates only) after creating
+   * the entity.
+   * @async
+   * @param {Object} item - The data for the entity to be created.
+   * @returns {Promise<BaseModel>} - A promise that resolves to the created model instance.
+   * @throws {Error} - Throws an error if the data is invalid or if the creation process fails.
+   */
+  async create(item) {
+    if (!isNonEmptyObject(item)) {
+      const message = `Failed to create [${this.entityName}]: data is required`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    try {
+      const record = await this.entity.create(item).go();
+      const instance = this.#createInstance(record.data);
+
+      this.#invalidateCache();
+
+      return instance;
+    } catch (error) {
+      this.log.error(`Failed to create [${this.entityName}]`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Validates and batches items for batch operations.
+   * @private
+   * @param {Array<Object>} items - Items to be validated.
+   * @returns {Object} - An object containing validated items and error items.
+   */
+  #validateItems(items) {
+    const validatedItems = [];
+    const errorItems = [];
+
+    items.forEach((item) => {
+      try {
+        const { Item } = this.entity.put(item).params();
+        validatedItems.push({ ...removeElectroProperties(Item), ...item });
+      } catch (error) {
+        if (error instanceof ElectroValidationError) {
+          errorItems.push({ item, error: new ValidationError(error) });
+        }
+      }
+    });
+
+    return { validatedItems, errorItems };
+  }
+
+  /**
+   * Creates multiple entities in the collection and directly persists them to the database in
+   * a batch write operation. Batches are written in parallel and are limited to 25 items per batch.
+   *
+   * @async
+   * @param {Array<Object>} newItems - An array of data for the entities to be created.
+   * @param {BaseModel} [parent] - Optional parent entity that these items are associated with.
+   * @return {Promise<{ createdItems: BaseModel[],
+   * errorItems: { item: Object, error: ValidationError }[] }>} - A promise that resolves to
+   * an object containing the created items and any items that failed validation.
+   * @throws {ValidationError} - Throws a validation error if any of the items has validation
+   * failures.
+   */
+  async createMany(newItems, parent = null) {
+    if (!isNonEmptyArray(newItems)) {
+      const message = `Failed to create many [${this.entityName}]: items must be a non-empty array`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    try {
+      const { validatedItems, errorItems } = this.#validateItems(newItems);
+
+      if (validatedItems.length > 0) {
+        const response = await this.entity.put(validatedItems).go();
+
+        if (isNonEmptyArray(response?.unprocessed)) {
+          this.log.error(`Failed to process all items in batch write for [${this.entityName}]: ${JSON.stringify(response.unprocessed)}`);
+        }
+      }
+
+      const createdItems = this.#createInstances(validatedItems);
+
+      if (isNonEmptyObject(parent)) {
+        createdItems.forEach((record) => {
+          if (!isValidParent(parent, record)) {
+            this.log.warn(`Failed to associate parent with child [${this.entityName}]: parent is invalid`);
+            return;
+          }
+          // eslint-disable-next-line no-underscore-dangle,no-param-reassign
+          record._accessorCache[`get${parent.schema.getModelName()}`] = parent;
+        });
+      }
+
+      this.#invalidateCache();
+
+      this.log.info(`Created ${createdItems.length} items for [${this.entityName}]`);
+
+      return { createdItems, errorItems };
+    } catch (error) {
+      this.log.error(`Failed to create many [${this.entityName}]`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Updates a collection of entities in the database using a batch write (put) operation.
+   *
+   * @async
+   * @param {Array<BaseModel>} items - An array of model instances to be updated.
+   * @return {Promise<void>} - A promise that resolves when the update operation is complete.
+   * @throws {Error} - Throws an error if the update operation fails.
+   * @protected
+   */
+  async _saveMany(items) {
+    if (!isNonEmptyArray(items)) {
+      const message = `Failed to save many [${this.entityName}]: items must be a non-empty array`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    try {
+      const updates = items.map((item) => item.record);
+      const response = await this.entity.put(updates).go();
+
+      this.#invalidateCache();
+
+      if (response.unprocessed) {
+        this.log.error(`Failed to process all items in batch write for [${this.entityName}]: ${JSON.stringify(response.unprocessed)}`);
+      }
+    } catch (error) {
+      this.log.error(`Failed to save many [${this.entityName}]`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Removes all records of this entity based on the provided IDs. This will perform a batch
+   * delete operation. This operation does not remove dependent records.
+   * @param {Array<string>} ids - An array of IDs to remove.
+   * @return {Promise<void>} - A promise that resolves when the removal operation is complete.
+   * @throws {Error} - Throws an error if the IDs are not provided or if the
+   * removal operation fails.
+   */
+  async removeByIds(ids) {
+    if (!isNonEmptyArray(ids)) {
+      const message = `Failed to remove [${this.entityName}]: ids must be a non-empty array`;
+      this.log.error(message);
+      throw new Error(message);
+    }
+
+    this.log.info(`Removing ${ids.length} items for [${this.entityName}]`);
+    // todo: consider removing dependent records
+
+    await this.entity.delete(ids.map((id) => ({ [this.idName]: id }))).go();
+
+    this.#invalidateCache();
+  }
+}
+
+export default BaseCollection;