@adobe/spacecat-shared-data-access 2.108.0 → 3.0.0

package/src/models/latest-audit/latest-audit.collection.js

@@ -10,8 +10,10 @@
  * governing permissions and limitations under the License.
  */
 
-import BaseCollection from '../base/base.collection.js';
+import { isNonEmptyArray } from '@adobe/spacecat-shared-utils';
+import DataAccessError from '../../errors/data-access.error.js';
 import { guardId, guardString } from '../../util/index.js';
+import BaseCollection from '../base/base.collection.js';
 
 /**
  * LatestAuditCollection - A collection class responsible for managing LatestAudit entities.
@@ -23,8 +25,80 @@ import { guardId, guardString } from '../../util/index.js';
 class LatestAuditCollection extends BaseCollection {
   static COLLECTION_NAME = 'LatestAuditCollection';
 
-  async create(item) {
-    return super.create(item, { upsert: true });
+  // LatestAudit is a virtual view in v3; writes are not supported.
+  // eslint-disable-next-line class-methods-use-this
+  async create() {
+    throw new DataAccessError('LatestAudit is derived from Audit in v3 and cannot be created directly', this);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async createMany() {
+    throw new DataAccessError('LatestAudit is derived from Audit in v3 and cannot be created directly', this);
+  }
+
+  static #groupLatest(items, groupFields) {
+    const grouped = new Map();
+    items.forEach((item) => {
+      const key = groupFields.map((field) => item.record[field]).join('#');
+      const existing = grouped.get(key);
+      if (!existing || existing.getAuditedAt() < item.getAuditedAt()) {
+        grouped.set(key, item);
+      }
+    });
+    return [...grouped.values()];
+  }
+
+  async #allAuditsByKeys(keys, options = {}) {
+    const auditCollection = this.entityRegistry.getCollection('AuditCollection');
+    return auditCollection.allByIndexKeys(keys, {
+      ...options,
+      fetchAllPages: true,
+      order: 'desc',
+      returnCursor: false,
+    });
+  }
+
+  async all(sortKeys = {}, options = {}) {
+    return this.allByIndexKeys(sortKeys, options);
+  }
+
+  async findByAll(sortKeys = {}, options = {}) {
+    return this.findByIndexKeys(sortKeys, options);
+  }
+
+  async findByIndexKeys(keys, options = {}) {
+    const auditCollection = this.entityRegistry.getCollection('AuditCollection');
+
+    if (keys.siteId && keys.auditType) {
+      return auditCollection.findByIndexKeys(keys, { ...options, order: 'desc' });
+    }
+
+    // For single-key lookups, we only need the latest row, not all pages.
+    return auditCollection.findByIndexKeys(keys, { ...options, order: 'desc' });
+  }
+
+  async allByIndexKeys(keys, options = {}) {
+    const audits = await this.#allAuditsByKeys(keys, options);
+    if (!isNonEmptyArray(audits)) {
+      return options.returnCursor ? { data: [], cursor: null } : [];
+    }
+
+    let groupFields = ['siteId', 'auditType'];
+    if (keys.siteId && !keys.auditType) {
+      groupFields = ['auditType'];
+    } else if (!keys.siteId && keys.auditType) {
+      groupFields = ['siteId'];
+    }
+
+    const latest = LatestAuditCollection.#groupLatest(audits, groupFields);
+    // Preserve v2 behavior: default order is descending (most recent first).
+    const ascending = options.order === 'asc';
+    latest.sort((a, b) => (ascending
+      ? a.getAuditedAt().localeCompare(b.getAuditedAt())
+      : b.getAuditedAt().localeCompare(a.getAuditedAt())));
+    const limited = Number.isInteger(options.limit) ? latest.slice(0, options.limit) : latest;
+
+    return options.returnCursor ? { data: limited, cursor: null } : limited;
   }
 
   async allByAuditType(auditType) {

package/src/models/scrape-job/scrape-job.collection.js

@@ -42,6 +42,44 @@ class ScrapeJobCollection extends BaseCollection {
       },
     });
   }
+
+  async allByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL,
+    processingType,
+    optEnableJavascript,
+    optHideConsentBanner,
+    options = {},
+  ) {
+    const keys = {
+      baseURL,
+      processingType,
+      optEnableJavascript,
+      optHideConsentBanner,
+    };
+
+    return this.allByIndexKeys(keys, options);
+  }
+
+  async findByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL,
+    processingType,
+    optEnableJavascript,
+    optHideConsentBanner,
+    options = {},
+  ) {
+    const jobs = await this
+      .allByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+        baseURL,
+        processingType,
+        optEnableJavascript,
+        optHideConsentBanner,
+        { ...options, limit: options.limit || 1 },
+      );
+    if (Array.isArray(jobs)) {
+      return jobs[0] || null;
+    }
+    return jobs || null;
+  }
 }
 
 export default ScrapeJobCollection;

package/src/models/scrape-job/scrape-job.schema.js

@@ -130,17 +130,19 @@ const schema = new SchemaBuilder(ScrapeJob, ScrapeJobCollection)
   })
   .addAttribute('optEnableJavascript', {
     type: 'string',
+    postgrestIgnore: true,
     hidden: true,
     readOnly: true,
     watch: ['options'],
-    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.ENABLE_JAVASCRIPT] ? 'T' : 'F'),
+    set: (_, { options }) => (options?.[ScrapeJob.ScrapeOptions.ENABLE_JAVASCRIPT] ? 'T' : 'F'),
   })
   .addAttribute('optHideConsentBanner', {
     type: 'string',
+    postgrestIgnore: true,
     hidden: true,
     readOnly: true,
     watch: ['options'],
-    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.HIDE_CONSENT_BANNER] ? 'T' : 'F'),
+    set: (_, { options }) => (options?.[ScrapeJob.ScrapeOptions.HIDE_CONSENT_BANNER] ? 'T' : 'F'),
   })
   // access pattern: get all jobs sorted by startedAt
   .addAllIndex(['startedAt'])

package/src/models/site/site.schema.js

@@ -102,6 +102,7 @@ const schema = new SchemaBuilder(Site, SiteCollection)
   })
   .addAttribute('gitHubURL', {
     type: 'string',
+    postgrestField: 'github_url',
     validate: (value) => !value || isValidUrl(value),
   })
   .addAttribute('deliveryConfig', {

package/src/models/site-enrollment/site-enrollment.collection.js

@@ -22,7 +22,17 @@ import BaseCollection from '../base/base.collection.js';
 class SiteEnrollmentCollection extends BaseCollection {
   static COLLECTION_NAME = 'SiteEnrollmentCollection';
 
-  // add your custom collection methods here
+  async create(item, options = {}) {
+    if (item?.siteId && item?.entitlementId) {
+      const existing = await this.findByIndexKeys({
+        siteId: item.siteId,
+        entitlementId: item.entitlementId,
+      });
+      if (existing) return existing;
+    }
+
+    return super.create(item, options);
+  }
 }
 
 export default SiteEnrollmentCollection;

package/src/models/trial-user/trial-user.collection.js

@@ -21,8 +21,6 @@ import BaseCollection from '../base/base.collection.js';
  */
 class TrialUserCollection extends BaseCollection {
   static COLLECTION_NAME = 'TrialUserCollection';
-
-  // add custom methods here
 }
 
 export default TrialUserCollection;

package/src/models/trial-user/trial-user.schema.js

@@ -54,6 +54,10 @@ const schema = new SchemaBuilder(TrialUser, TrialUserCollection)
     type: 'any',
     validate: (value) => !value || isObject(value),
   })
-  .addAllIndex(['emailId']);
+  .addAllIndex(['emailId'])
+  .addIndex(
+    { composite: ['organizationId'] },
+    { composite: ['updatedAt'] },
+  );
 
 export default schema.build();

package/src/service/index.d.ts

@@ -11,10 +11,16 @@
  */
 
 interface DataAccessConfig {
-  tableNameData: string;
+  postgrestUrl: string;
+  postgrestSchema?: string;
+  postgrestApiKey?: string;
+  postgrestHeaders?: object;
+  s3Bucket?: string;
+  region?: string;
 }
 
 export function createDataAccess(
   config: DataAccessConfig,
   logger: object,
+  client?: object,
 ): object;

package/src/service/index.js

@@ -10,10 +10,8 @@
  * governing permissions and limitations under the License.
  */
 
-import { DynamoDB } from '@aws-sdk/client-dynamodb';
-import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
 import { S3Client } from '@aws-sdk/client-s3';
-import { Service } from 'electrodb';
+import { PostgrestClient } from '@supabase/postgrest-js';
 
 import { instrumentAWSClient } from '@adobe/spacecat-shared-utils';
 import { EntityRegistry } from '../models/index.js';
@@ -23,47 +21,31 @@ export * from '../errors/index.js';
 export * from '../models/index.js';
 export * from '../util/index.js';
 
-let defaultDynamoDBClient;
-const documentClientCache = new WeakMap();
-
-const createRawClient = (client = undefined) => {
-  const rawClient = client || (() => {
-    if (!defaultDynamoDBClient) {
-      defaultDynamoDBClient = new DynamoDB();
-    }
-    return defaultDynamoDBClient;
-  })();
-
-  let documentClient = documentClientCache.get(rawClient);
-  if (!documentClient) {
-    documentClient = DynamoDBDocument.from(instrumentAWSClient(rawClient), {
-      marshallOptions: {
-        convertEmptyValues: true,
-        removeUndefinedValues: true,
-      },
-    });
-    documentClientCache.set(rawClient, documentClient);
+const createPostgrestService = (config, client = undefined) => {
+  if (client) {
+    return client;
   }
 
-  return documentClient;
-};
+  const {
+    postgrestUrl,
+    postgrestSchema = 'public',
+    postgrestApiKey,
+    postgrestHeaders = {},
+  } = config;
+
+  if (!postgrestUrl) {
+    throw new Error('postgrestUrl is required to create data access');
+  }
 
-const createElectroService = (client, config, log) => {
-  const { tableNameData: table } = config;
-  /* c8 ignore start */
-  const logger = (event) => {
-    log.debug(JSON.stringify(event, null, 4));
+  const headers = {
+    ...postgrestHeaders,
+    ...postgrestApiKey ? { apikey: postgrestApiKey, Authorization: `Bearer ${postgrestApiKey}` } : {},
   };
-  /* c8 ignore end */
-
-  return new Service(
-    EntityRegistry.getEntities(),
-    {
-      client,
-      table,
-      logger,
-    },
-  );
+
+  return new PostgrestClient(postgrestUrl, {
+    schema: postgrestSchema,
+    headers,
+  });
 };
 
 /**
@@ -91,31 +73,30 @@ const createS3Service = (config) => {
  * Creates a services dictionary containing all datastore services.
  * Each collection can declare which service it needs via its DATASTORE_TYPE.
  *
- * @param {object} electroService - The ElectroDB service for DynamoDB operations
+ * @param {PostgrestClient} postgrestService - PostgREST client
  * @param {object} config - Configuration object
- * @returns {object} Services dictionary with dynamo and s3 services
+ * @returns {object} Services dictionary with postgrest and s3 services
  */
-const createServices = (electroService, config) => ({
-  dynamo: electroService,
+const createServices = (postgrestService, config) => ({
+  postgrest: postgrestService,
   s3: createS3Service(config),
 });
 
 /**
- * Creates a data access layer for interacting with DynamoDB using ElectroDB.
+ * Creates a data access layer for interacting with Postgres via PostgREST.
  *
- * @param {{tableNameData: string, s3Bucket?: string, region?: string}} config - Configuration
- *   object containing table name and optional S3 configuration
+ * @param {{postgrestUrl: string, postgrestSchema?: string, postgrestApiKey?: string,
+ * postgrestHeaders?: object, s3Bucket?: string, region?: string}} config - Configuration object
  * @param {object} log - Logger instance, defaults to console
- * @param {DynamoDB} [client] - Optional custom DynamoDB client instance
+ * @param {PostgrestClient} [client] - Optional custom Postgrest client instance
  * @returns {object} Data access collections for interacting with entities
  */
 export const createDataAccess = (config, log = console, client = undefined) => {
   registerLogger(log);
 
-  const rawClient = createRawClient(client);
-  const electroService = createElectroService(rawClient, config, log);
-  const services = createServices(electroService, config);
-  const entityRegistry = new EntityRegistry(services, log);
+  const postgrestService = createPostgrestService(config, client);
+  const services = createServices(postgrestService, config);
+  const entityRegistry = new EntityRegistry(services, config, log);
 
   return entityRegistry.getCollections();
 };

package/src/util/index.js

@@ -25,6 +25,7 @@ export {
 export {
   registerLogger,
   getLogger,
+  resetLoggerRegistry,
 } from './logger-registry.js';
 
 /**
@@ -33,6 +34,6 @@ export {
  * @enum {string}
  */
 export const DATASTORE_TYPE = Object.freeze({
-  DYNAMO: 'dynamo',
+  POSTGREST: 'postgrest',
   S3: 's3',
 });

package/src/util/logger-registry.js

@@ -29,6 +29,10 @@ class LoggerRegistry {
   getLogger() {
     return this.#logger || console;
   }
+
+  reset() {
+    this.#logger = null;
+  }
 }
 
 /**
@@ -48,3 +52,11 @@ export function registerLogger(logger) {
 export function getLogger() {
   return LoggerRegistry.getInstance().getLogger();
 }
+
+/**
+ * Resets the registered logger to default console logger.
+ * Primarily intended for test isolation.
+ */
+export function resetLoggerRegistry() {
+  LoggerRegistry.getInstance().reset();
+}

package/src/util/patcher.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-import { isNonEmptyArray, isObject } from '@adobe/spacecat-shared-utils';
+import { isObject } from '@adobe/spacecat-shared-utils';
 
 import ValidationError from '../errors/validation.error.js';
 
@@ -26,13 +26,6 @@ import {
   guardString,
 } from './index.js';
 
-/**
- * Checks if a property is read-only and throws an error if it is.
- * @param {string} propertyName - The name of the property to check.
- * @param {Object} attribute - The attribute to check.
- * @throws {Error} - Throws an error if the property is read-only.
- * @private
- */
 const checkReadOnly = (propertyName, attribute) => {
   if (attribute.readOnly) {
     throw new ValidationError(`The property ${propertyName} is read-only and cannot be updated.`);
@@ -48,137 +41,68 @@ const checkUpdatesAllowed = (schema) => {
 class Patcher {
   /**
    * Creates a new Patcher instance for an entity.
-   * @param {object} entity - The entity backing the record.
+   * @param {object} collection - The backing collection instance.
    * @param {Schema} schema - The schema for the entity.
    * @param {object} record - The record to patch.
    */
-  constructor(entity, schema, record) {
-    this.entity = entity;
+  constructor(collection, schema, record) {
+    this.collection = collection;
     this.schema = schema;
     this.record = record;
 
     this.entityName = schema.getEntityName();
-    this.model = entity.model;
     this.idName = schema.getIdName();
 
-    // holds the previous value of updated attributes
     this.previous = {};
-
-    // holds the updates to the attributes
     this.updates = {};
 
+    this.legacyEntity = collection && typeof collection.patch === 'function'
+      ? collection
+      : null;
     this.patchRecord = null;
   }
 
-  /**
-   * Checks if a property is nullable.
-   * @param {string} propertyName - The name of the property to check.
-   * @return {boolean} True if the property is nullable, false otherwise.
-   * @private
-   */
   #isAttributeNullable(propertyName) {
-    return !this.model.schema.attributes[propertyName]?.required;
-  }
-
-  /**
-   * Composite keys have to be provided to ElectroDB in order to update a record across
-   * multiple indexes. This method retrieves the composite values for the entity from
-   * the schema indexes and filters out any values that are being updated.
-   * @return {{}} - An object containing the composite values for the entity.
-   * @private
-   */
-  #getCompositeValues() {
-    const { indexes } = this.model;
-    const result = {};
-
-    const processComposite = (index, compositeType) => {
-      const compositeArray = index[compositeType]?.facets;
-      if (isNonEmptyArray(compositeArray)) {
-        compositeArray.forEach((compositeKey) => {
-          if (
-            !Object.keys(this.updates).includes(compositeKey)
-            && this.record[compositeKey] !== undefined
-          ) {
-            result[compositeKey] = this.record[compositeKey];
-          }
-        });
-      }
-    };
-
-    Object.values(indexes).forEach((index) => {
-      processComposite(index, 'pk');
-      processComposite(index, 'sk');
-    });
-
-    return result;
+    return !this.schema.getAttribute(propertyName)?.required;
   }
 
-  /**
-   * Sets a property on the record and updates the patch record.
-   * @param {string} attribute - The attribute to set.
-   * @param {any} value - The value to set for the property.
-   * @private
-   */
-  #set(attribute, value) {
-    this.patchRecord = this.#getPatchRecord().set({ [attribute.name]: value });
-
-    const transmutedValue = attribute.get(value, () => {});
+  #set(propertyName, attribute, value) {
     const update = {
-      [attribute.name]: {
-        previous: this.record[attribute.name],
-        current: transmutedValue,
+      [propertyName]: {
+        previous: this.record[propertyName],
+        current: value,
       },
     };
 
-    // update the record with the update value for later save
-    this.record[attribute.name] = transmutedValue;
-
-    // remember the update operation with the previous and current value
+    const hydratedValue = typeof attribute.get === 'function'
+      ? attribute.get(value)
+      : value;
+    this.record[propertyName] = hydratedValue;
     this.updates = { ...this.updates, ...update };
+
+    if (this.legacyEntity) {
+      if (!this.patchRecord) {
+        this.patchRecord = this.legacyEntity.patch(this.#getPrimaryKeyValues());
+      }
+      this.patchRecord = this.patchRecord.set({ [propertyName]: value });
+    }
   }
 
-  /**
-   * Gets the primary key values for the entity from the schema's primary index.
-   * This supports composite primary keys (e.g., siteId + url).
-   * @return {Object} - An object containing the primary key values.
-   * @private
-   */
   #getPrimaryKeyValues() {
     const primaryKeys = this.schema.getIndexKeys('primary');
-    if (isNonEmptyArray(primaryKeys)) {
+    if (Array.isArray(primaryKeys) && primaryKeys.length > 0) {
       return primaryKeys.reduce((acc, key) => {
         acc[key] = this.record[key];
         return acc;
       }, {});
     }
-    // Fallback to default id name
     return { [this.idName]: this.record[this.idName] };
   }
 
-  /**
-   * Gets the patch record for the entity. If it does not exist, it will be created.
-   * @return {Object} - The patch record for the entity.
-   * @private
-   */
-  #getPatchRecord() {
-    if (!this.patchRecord) {
-      this.patchRecord = this.entity.patch(this.#getPrimaryKeyValues());
-    }
-    return this.patchRecord;
-  }
-
-  /**
-   * Patches a value for a given property on the entity. This method will validate the value
-   * against the schema and throw an error if the value is invalid. If the value is declared as
-   * a reference, it will validate the ID format.
-   * @param {string} propertyName - The name of the property to patch.
-   * @param {any} value - The value to patch.
-   * @param {boolean} [isReference=false] - Whether the value is a reference to another entity.
-   */
   patchValue(propertyName, value, isReference = false) {
     checkUpdatesAllowed(this.schema);
 
-    const attribute = this.model.schema?.attributes[propertyName];
+    const attribute = this.schema.getAttribute(propertyName);
     if (!isObject(attribute)) {
       throw new ValidationError(`Property ${propertyName} does not exist on entity ${this.entityName}.`);
     }
@@ -189,6 +113,8 @@ class Patcher {
 
     if (isReference) {
       guardId(propertyName, value, this.entityName, nullable);
+    } else if (Array.isArray(attribute.type)) {
+      guardEnum(propertyName, value, attribute.type, this.entityName, nullable);
     } else {
       switch (attribute.type) {
         case 'any':
@@ -220,14 +146,9 @@ class Patcher {
       }
     }
 
-    this.#set(attribute, value);
+    this.#set(propertyName, attribute, value);
   }
 
-  /**
-   * Saves the current state of the entity to the database.
-   * @return {Promise<void>}
-   * @throws {Error} - Throws an error if the save operation fails.
-   */
   async save() {
     checkUpdatesAllowed(this.schema);
 
@@ -235,11 +156,42 @@ class Patcher {
       return;
     }
 
-    const compositeValues = this.#getCompositeValues();
-    await this.#getPatchRecord()
-      .composite(compositeValues)
-      .go();
-    this.record.updatedAt = new Date().toISOString();
+    const previousUpdatedAt = this.record.updatedAt;
+    let nextUpdatedAt = new Date().toISOString();
+    if (typeof previousUpdatedAt === 'string' && previousUpdatedAt === nextUpdatedAt) {
+      const previousDate = new Date(previousUpdatedAt);
+      if (!Number.isNaN(previousDate.getTime())) {
+        nextUpdatedAt = new Date(previousDate.getTime() + 1000).toISOString();
+      }
+    }
+    this.record.updatedAt = nextUpdatedAt;
+    this.updates.updatedAt = {
+      previous: previousUpdatedAt,
+      current: nextUpdatedAt,
+    };
+
+    const keys = this.#getPrimaryKeyValues();
+    const updates = Object.keys(this.updates).reduce((acc, key) => {
+      acc[key] = this.updates[key].current;
+      return acc;
+    }, {});
+
+    if (this.collection
+      && typeof this.collection.applyUpdateWatchers === 'function'
+      && typeof this.collection.updateByKeys === 'function') {
+      const watched = this.collection.applyUpdateWatchers(this.record, updates);
+      this.record = watched.record;
+      await this.collection.updateByKeys(keys, watched.updates);
+      return;
+    }
+
+    if (this.patchRecord && typeof this.patchRecord.go === 'function') {
+      this.patchRecord = this.patchRecord.set({ updatedAt: nextUpdatedAt });
+      await this.patchRecord.go();
+      return;
+    }
+
+    throw new ValidationError(`No persistence strategy available for ${this.entityName}`);
   }
 
   getUpdates() {