@adobe/spacecat-shared-data-access 2.109.0 → 3.0.0

package/src/models/base/base.model.js

@@ -50,15 +50,15 @@ class BaseModel {
   /**
    * Constructs an instance of BaseModel.
    * @constructor
-   * @param {Object} electroService - The ElectroDB service used for managing entities.
+   * @param {Object} postgrestService - The PostgREST client used for managing entities.
    * @param {EntityRegistry} entityRegistry - The registry holding entities, their schema
    * and collection.
    * @param {Schema} schema - The schema for the entity.
    * @param {Object} record - The initial data for the entity instance.
    * @param {Object} log - A log for capturing logging information.
    */
-  constructor(electroService, entityRegistry, schema, record, log) {
-    this.electroService = electroService;
+  constructor(postgrestService, entityRegistry, schema, record, log) {
+    this.postgrestService = postgrestService;
     this.entityRegistry = entityRegistry;
     this.schema = schema;
     this.record = record;
@@ -68,9 +68,8 @@ class BaseModel {
     this.idName = entityNameToIdName(this.entityName);
 
     this.collection = entityRegistry.getCollection(schema.getCollectionName());
-    this.entity = electroService.entities[this.entityName];
 
-    this.patcher = new Patcher(this.entity, this.schema, this.record);
+    this.patcher = new Patcher(this.collection, this.schema, this.record);
 
     this._accessorCache = {};
 
@@ -283,7 +282,15 @@ class BaseModel {
 
       await Promise.all(removePromises);
 
-      await this.entity.remove(this.generateCompositeKeys()).go();
+      if (this.collection && typeof this.collection.removeByIndexKeys === 'function') {
+        await this.collection.removeByIndexKeys([this.generateCompositeKeys()]);
+      } else if (this.postgrestService?.entities?.[this.entityName]?.remove) {
+        await this.postgrestService.entities[this.entityName]
+          .remove(this.generateCompositeKeys())
+          .go();
+      } else {
+        throw new DataAccessError(`No remove strategy available for ${this.entityName}`, this);
+      }
 
       this.#invalidateCache();
 

package/src/models/base/entity.registry.js

@@ -86,11 +86,13 @@ import SentimentTopicSchema from '../sentiment-topic/sentiment-topic.schema.js';
 class EntityRegistry {
   static entities = {};
 
+  static defaultEntities = {};
+
   /**
    * Constructs an instance of EntityRegistry.
    * @constructor
    * @param {Object} services - Dictionary of services keyed by datastore type.
-   * @param {Object} services.dynamo - The ElectroDB service instance for DynamoDB operations.
+   * @param {Object} services.postgrest - The PostgREST client instance for Postgres operations.
    * @param {{s3Client: S3Client, s3Bucket: string}|null} [services.s3] - S3 service configuration.
    * @param {Object} config - Configuration object containing environment-derived settings.
    * @param {Object} log - A logger for capturing and logging information.
@@ -107,14 +109,14 @@ class EntityRegistry {
   /**
    * Initializes the collections managed by the EntityRegistry.
    * This method creates instances of each collection and stores them in an internal map.
-   * ElectroDB-based collections are initialized with the dynamo service.
+   * PostgREST-based collections are initialized with the postgrest service.
    * Configuration is handled specially as it's a standalone S3-based collection.
    * @private
    */
   #initialize() {
-    // Initialize ElectroDB-based collections
+    // Initialize PostgREST-based collections
     Object.values(EntityRegistry.entities).forEach(({ collection: Collection, schema }) => {
-      const collection = new Collection(this.services.dynamo, this, schema, this.log);
+      const collection = new Collection(this.services.postgrest, this, schema, this.log);
       this.collections.set(Collection.COLLECTION_NAME, collection);
     });
 
@@ -164,6 +166,10 @@ class EntityRegistry {
   static registerEntity(schema, collection) {
     this.entities[decapitalize(schema.getEntityName())] = { schema, collection };
   }
+
+  static resetEntities() {
+    this.entities = { ...this.defaultEntities };
+  }
 }
 
 // Register ElectroDB-based entities only (Configuration is handled separately)
@@ -198,5 +204,6 @@ EntityRegistry.registerEntity(TrialUserActivitySchema, TrialUserActivityCollecti
 EntityRegistry.registerEntity(PageCitabilitySchema, PageCitabilityCollection);
 EntityRegistry.registerEntity(SentimentGuidelineSchema, SentimentGuidelineCollection);
 EntityRegistry.registerEntity(SentimentTopicSchema, SentimentTopicCollection);
+EntityRegistry.defaultEntities = { ...EntityRegistry.entities };
 
 export default EntityRegistry;

package/src/models/base/schema.builder.js

@@ -36,6 +36,7 @@ const DEFAULT_SERVICE_NAME = 'SpaceCat';
  */
 const ID_ATTRIBUTE_DATA = {
   type: 'string',
+  postgrestField: 'id',
   required: true,
   readOnly: true,
   // https://electrodb.dev/en/modeling/attributes/#default
@@ -215,6 +216,7 @@ class SchemaBuilder {
 
     this.addAttribute('recordExpiresAt', {
       type: 'number',
+      postgrestIgnore: true,
       required: true,
       readOnly: true,
       default: () => Math.floor(Date.now() / 1000) + ttlInDays * 24 * 60 * 60,

package/src/models/base/schema.js

@@ -156,6 +156,17 @@ class Schema {
     return result;
   }
 
+  // Internal helper: v3 schemas use `composite` keys (legacy Electro used `facets`).
+  static #getKeyParts(indexPart) {
+    if (Array.isArray(indexPart?.composite)) {
+      return indexPart.composite;
+    }
+    if (Array.isArray(indexPart?.facets)) {
+      return indexPart.facets;
+    }
+    return [];
+  }
+
   getIndexByName(indexName) {
     return this.indexes[indexName];
   }
@@ -167,10 +178,10 @@ class Schema {
       const subKeyNames = sortKeys.slice(0, length);
       const index = Object.values(this.indexes).find((candidate) => {
         const { pk, sk } = candidate;
-        const allKeys = [...(pk?.facets || []), ...(sk?.facets || [])];
+        const allKeys = [...Schema.#getKeyParts(pk), ...Schema.#getKeyParts(sk)];
 
         // check if all keys in the index are in the sort keys
-        const pkKeys = Array.isArray(pk?.facets) ? pk.facets : [];
+        const pkKeys = Schema.#getKeyParts(pk);
         return pkKeys.every((key) => subKeyNames.includes(key))
           && subKeyNames.every((key) => allKeys.includes(key));
       });
@@ -246,8 +257,8 @@ class Schema {
       return [];
     }
 
-    const pkKeys = Array.isArray(index.pk?.facets) ? index.pk.facets : [];
-    const skKeys = Array.isArray(index.sk?.facets) ? index.sk.facets : [];
+    const pkKeys = Schema.#getKeyParts(index.pk);
+    const skKeys = Schema.#getKeyParts(index.sk);
 
     return [...pkKeys, ...skKeys];
   }

package/src/models/fix-entity-suggestion/fix-entity-suggestion.model.js

@@ -25,6 +25,21 @@ class FixEntitySuggestion extends BaseModel {
 
   static DEFAULT_UPDATED_BY = 'spacecat';
 
+  getId() {
+    const id = super.getId();
+    if (id) {
+      return id;
+    }
+
+    const suggestionId = this.getSuggestionId();
+    const fixEntityId = this.getFixEntityId();
+    if (suggestionId && fixEntityId) {
+      return `${suggestionId}#${fixEntityId}`;
+    }
+
+    return undefined;
+  }
+
   /**
    * Generates the composite keys for the FixEntitySuggestion model.
    * @returns {Object} - The composite keys.

package/src/models/fix-entity-suggestion/fix-entity-suggestion.schema.js

@@ -23,6 +23,17 @@ Indexes Doc: https://electrodb.dev/en/modeling/indexes/
 const schema = new SchemaBuilder(FixEntitySuggestion, FixEntitySuggestionCollection)
   .withPrimaryPartitionKeys(['suggestionId'])
   .withPrimarySortKeys(['fixEntityId'])
+  .addAttribute('fixEntitySuggestionId', {
+    type: 'string',
+    required: false,
+    readOnly: true,
+    postgrestIgnore: true,
+  })
+  .addAttribute('updatedBy', {
+    type: 'string',
+    required: false,
+    postgrestIgnore: true,
+  })
   .addReference('belongs_to', 'FixEntity')
   .addReference('belongs_to', 'Suggestion')
   .addAttribute('opportunityId', {
@@ -38,6 +49,7 @@ const schema = new SchemaBuilder(FixEntitySuggestion, FixEntitySuggestionCollect
   .addAttribute('fixEntityCreatedDate', {
     type: 'string',
     readOnly: true,
+    postgrestIgnore: true,
     watch: ['fixEntityCreatedAt'],
     set: (_, { fixEntityCreatedAt }) => (fixEntityCreatedAt ? fixEntityCreatedAt.split('T')[0] : undefined),
   })

package/src/models/key-event/key-event.collection.js

@@ -11,6 +11,7 @@
  */
 
 import BaseCollection from '../base/base.collection.js';
+import DataAccessError from '../../errors/data-access.error.js';
 
 /**
  * KeyEventCollection - A collection class responsible for managing KeyEvent entities.
@@ -22,7 +23,28 @@ import BaseCollection from '../base/base.collection.js';
 class KeyEventCollection extends BaseCollection {
   static COLLECTION_NAME = 'KeyEventCollection';
 
-  // add custom methods here
+  #throwDeprecated() {
+    throw new DataAccessError(
+      'KeyEvent is deprecated in data-access v3. Use Audit/LatestAudit and related Postgres-backed entities instead.',
+      this,
+    );
+  }
+
+  async all() { return this.#throwDeprecated(); }
+
+  async allByIndexKeys() { return this.#throwDeprecated(); }
+
+  async findById() { return this.#throwDeprecated(); }
+
+  async findByIndexKeys() { return this.#throwDeprecated(); }
+
+  async create() { return this.#throwDeprecated(); }
+
+  async createMany() { return this.#throwDeprecated(); }
+
+  async removeByIds() { return this.#throwDeprecated(); }
+
+  async removeByIndexKeys() { return this.#throwDeprecated(); }
 }
 
 export default KeyEventCollection;

package/src/models/latest-audit/latest-audit.collection.js

@@ -10,8 +10,10 @@
  * governing permissions and limitations under the License.
  */
 
-import BaseCollection from '../base/base.collection.js';
+import { isNonEmptyArray } from '@adobe/spacecat-shared-utils';
+import DataAccessError from '../../errors/data-access.error.js';
 import { guardId, guardString } from '../../util/index.js';
+import BaseCollection from '../base/base.collection.js';
 
 /**
  * LatestAuditCollection - A collection class responsible for managing LatestAudit entities.
@@ -23,8 +25,80 @@ import { guardId, guardString } from '../../util/index.js';
 class LatestAuditCollection extends BaseCollection {
   static COLLECTION_NAME = 'LatestAuditCollection';
 
-  async create(item) {
-    return super.create(item, { upsert: true });
+  // LatestAudit is a virtual view in v3; writes are not supported.
+  // eslint-disable-next-line class-methods-use-this
+  async create() {
+    throw new DataAccessError('LatestAudit is derived from Audit in v3 and cannot be created directly', this);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async createMany() {
+    throw new DataAccessError('LatestAudit is derived from Audit in v3 and cannot be created directly', this);
+  }
+
+  static #groupLatest(items, groupFields) {
+    const grouped = new Map();
+    items.forEach((item) => {
+      const key = groupFields.map((field) => item.record[field]).join('#');
+      const existing = grouped.get(key);
+      if (!existing || existing.getAuditedAt() < item.getAuditedAt()) {
+        grouped.set(key, item);
+      }
+    });
+    return [...grouped.values()];
+  }
+
+  async #allAuditsByKeys(keys, options = {}) {
+    const auditCollection = this.entityRegistry.getCollection('AuditCollection');
+    return auditCollection.allByIndexKeys(keys, {
+      ...options,
+      fetchAllPages: true,
+      order: 'desc',
+      returnCursor: false,
+    });
+  }
+
+  async all(sortKeys = {}, options = {}) {
+    return this.allByIndexKeys(sortKeys, options);
+  }
+
+  async findByAll(sortKeys = {}, options = {}) {
+    return this.findByIndexKeys(sortKeys, options);
+  }
+
+  async findByIndexKeys(keys, options = {}) {
+    const auditCollection = this.entityRegistry.getCollection('AuditCollection');
+
+    if (keys.siteId && keys.auditType) {
+      return auditCollection.findByIndexKeys(keys, { ...options, order: 'desc' });
+    }
+
+    // For single-key lookups, we only need the latest row, not all pages.
+    return auditCollection.findByIndexKeys(keys, { ...options, order: 'desc' });
+  }
+
+  async allByIndexKeys(keys, options = {}) {
+    const audits = await this.#allAuditsByKeys(keys, options);
+    if (!isNonEmptyArray(audits)) {
+      return options.returnCursor ? { data: [], cursor: null } : [];
+    }
+
+    let groupFields = ['siteId', 'auditType'];
+    if (keys.siteId && !keys.auditType) {
+      groupFields = ['auditType'];
+    } else if (!keys.siteId && keys.auditType) {
+      groupFields = ['siteId'];
+    }
+
+    const latest = LatestAuditCollection.#groupLatest(audits, groupFields);
+    // Preserve v2 behavior: default order is descending (most recent first).
+    const ascending = options.order === 'asc';
+    latest.sort((a, b) => (ascending
+      ? a.getAuditedAt().localeCompare(b.getAuditedAt())
+      : b.getAuditedAt().localeCompare(a.getAuditedAt())));
+    const limited = Number.isInteger(options.limit) ? latest.slice(0, options.limit) : latest;
+
+    return options.returnCursor ? { data: limited, cursor: null } : limited;
   }
 
   async allByAuditType(auditType) {

package/src/models/scrape-job/scrape-job.collection.js

@@ -42,6 +42,44 @@ class ScrapeJobCollection extends BaseCollection {
       },
     });
   }
+
+  async allByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL,
+    processingType,
+    optEnableJavascript,
+    optHideConsentBanner,
+    options = {},
+  ) {
+    const keys = {
+      baseURL,
+      processingType,
+      optEnableJavascript,
+      optHideConsentBanner,
+    };
+
+    return this.allByIndexKeys(keys, options);
+  }
+
+  async findByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL,
+    processingType,
+    optEnableJavascript,
+    optHideConsentBanner,
+    options = {},
+  ) {
+    const jobs = await this
+      .allByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+        baseURL,
+        processingType,
+        optEnableJavascript,
+        optHideConsentBanner,
+        { ...options, limit: options.limit || 1 },
+      );
+    if (Array.isArray(jobs)) {
+      return jobs[0] || null;
+    }
+    return jobs || null;
+  }
 }
 
 export default ScrapeJobCollection;

package/src/models/scrape-job/scrape-job.schema.js

@@ -130,17 +130,19 @@ const schema = new SchemaBuilder(ScrapeJob, ScrapeJobCollection)
   })
   .addAttribute('optEnableJavascript', {
     type: 'string',
+    postgrestIgnore: true,
     hidden: true,
     readOnly: true,
     watch: ['options'],
-    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.ENABLE_JAVASCRIPT] ? 'T' : 'F'),
+    set: (_, { options }) => (options?.[ScrapeJob.ScrapeOptions.ENABLE_JAVASCRIPT] ? 'T' : 'F'),
   })
   .addAttribute('optHideConsentBanner', {
     type: 'string',
+    postgrestIgnore: true,
     hidden: true,
     readOnly: true,
     watch: ['options'],
-    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.HIDE_CONSENT_BANNER] ? 'T' : 'F'),
+    set: (_, { options }) => (options?.[ScrapeJob.ScrapeOptions.HIDE_CONSENT_BANNER] ? 'T' : 'F'),
   })
   // access pattern: get all jobs sorted by startedAt
   .addAllIndex(['startedAt'])

package/src/models/site/site.schema.js

@@ -102,6 +102,7 @@ const schema = new SchemaBuilder(Site, SiteCollection)
   })
   .addAttribute('gitHubURL', {
     type: 'string',
+    postgrestField: 'github_url',
     validate: (value) => !value || isValidUrl(value),
   })
   .addAttribute('deliveryConfig', {

package/src/models/site-enrollment/site-enrollment.collection.js

@@ -22,7 +22,17 @@ import BaseCollection from '../base/base.collection.js';
 class SiteEnrollmentCollection extends BaseCollection {
   static COLLECTION_NAME = 'SiteEnrollmentCollection';
 
-  // add your custom collection methods here
+  async create(item, options = {}) {
+    if (item?.siteId && item?.entitlementId) {
+      const existing = await this.findByIndexKeys({
+        siteId: item.siteId,
+        entitlementId: item.entitlementId,
+      });
+      if (existing) return existing;
+    }
+
+    return super.create(item, options);
+  }
 }
 
 export default SiteEnrollmentCollection;

package/src/models/trial-user/trial-user.collection.js

@@ -21,8 +21,6 @@ import BaseCollection from '../base/base.collection.js';
  */
 class TrialUserCollection extends BaseCollection {
   static COLLECTION_NAME = 'TrialUserCollection';
-
-  // add custom methods here
 }
 
 export default TrialUserCollection;

package/src/models/trial-user/trial-user.schema.js

@@ -54,6 +54,10 @@ const schema = new SchemaBuilder(TrialUser, TrialUserCollection)
     type: 'any',
     validate: (value) => !value || isObject(value),
   })
-  .addAllIndex(['emailId']);
+  .addAllIndex(['emailId'])
+  .addIndex(
+    { composite: ['organizationId'] },
+    { composite: ['updatedAt'] },
+  );
 
 export default schema.build();

package/src/service/index.d.ts

@@ -11,10 +11,16 @@
  */
 
 interface DataAccessConfig {
-  tableNameData: string;
+  postgrestUrl: string;
+  postgrestSchema?: string;
+  postgrestApiKey?: string;
+  postgrestHeaders?: object;
+  s3Bucket?: string;
+  region?: string;
 }
 
 export function createDataAccess(
   config: DataAccessConfig,
   logger: object,
+  client?: object,
 ): object;

package/src/service/index.js

@@ -10,10 +10,8 @@
  * governing permissions and limitations under the License.
  */
 
-import { DynamoDB } from '@aws-sdk/client-dynamodb';
-import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
 import { S3Client } from '@aws-sdk/client-s3';
-import { Service } from 'electrodb';
+import { PostgrestClient } from '@supabase/postgrest-js';
 
 import { instrumentAWSClient } from '@adobe/spacecat-shared-utils';
 import { EntityRegistry } from '../models/index.js';
@@ -23,47 +21,31 @@ export * from '../errors/index.js';
 export * from '../models/index.js';
 export * from '../util/index.js';
 
-let defaultDynamoDBClient;
-const documentClientCache = new WeakMap();
-
-const createRawClient = (client = undefined) => {
-  const rawClient = client || (() => {
-    if (!defaultDynamoDBClient) {
-      defaultDynamoDBClient = new DynamoDB();
-    }
-    return defaultDynamoDBClient;
-  })();
-
-  let documentClient = documentClientCache.get(rawClient);
-  if (!documentClient) {
-    documentClient = DynamoDBDocument.from(instrumentAWSClient(rawClient), {
-      marshallOptions: {
-        convertEmptyValues: true,
-        removeUndefinedValues: true,
-      },
-    });
-    documentClientCache.set(rawClient, documentClient);
+const createPostgrestService = (config, client = undefined) => {
+  if (client) {
+    return client;
   }
 
-  return documentClient;
-};
+  const {
+    postgrestUrl,
+    postgrestSchema = 'public',
+    postgrestApiKey,
+    postgrestHeaders = {},
+  } = config;
+
+  if (!postgrestUrl) {
+    throw new Error('postgrestUrl is required to create data access');
+  }
 
-const createElectroService = (client, config, log) => {
-  const { tableNameData: table } = config;
-  /* c8 ignore start */
-  const logger = (event) => {
-    log.debug(JSON.stringify(event, null, 4));
+  const headers = {
+    ...postgrestHeaders,
+    ...postgrestApiKey ? { apikey: postgrestApiKey, Authorization: `Bearer ${postgrestApiKey}` } : {},
   };
-  /* c8 ignore end */
-
-  return new Service(
-    EntityRegistry.getEntities(),
-    {
-      client,
-      table,
-      logger,
-    },
-  );
+
+  return new PostgrestClient(postgrestUrl, {
+    schema: postgrestSchema,
+    headers,
+  });
 };
 
 /**
@@ -91,30 +73,29 @@ const createS3Service = (config) => {
  * Creates a services dictionary containing all datastore services.
  * Each collection can declare which service it needs via its DATASTORE_TYPE.
  *
- * @param {object} electroService - The ElectroDB service for DynamoDB operations
+ * @param {PostgrestClient} postgrestService - PostgREST client
  * @param {object} config - Configuration object
- * @returns {object} Services dictionary with dynamo and s3 services
+ * @returns {object} Services dictionary with postgrest and s3 services
  */
-const createServices = (electroService, config) => ({
-  dynamo: electroService,
+const createServices = (postgrestService, config) => ({
+  postgrest: postgrestService,
   s3: createS3Service(config),
 });
 
 /**
- * Creates a data access layer for interacting with DynamoDB using ElectroDB.
+ * Creates a data access layer for interacting with Postgres via PostgREST.
  *
- * @param {{tableNameData: string, s3Bucket?: string, region?: string}} config - Configuration
- *   object containing table name and optional S3 configuration
+ * @param {{postgrestUrl: string, postgrestSchema?: string, postgrestApiKey?: string,
+ * postgrestHeaders?: object, s3Bucket?: string, region?: string}} config - Configuration object
  * @param {object} log - Logger instance, defaults to console
- * @param {DynamoDB} [client] - Optional custom DynamoDB client instance
+ * @param {PostgrestClient} [client] - Optional custom Postgrest client instance
  * @returns {object} Data access collections for interacting with entities
  */
 export const createDataAccess = (config, log = console, client = undefined) => {
   registerLogger(log);
 
-  const rawClient = createRawClient(client);
-  const electroService = createElectroService(rawClient, config, log);
-  const services = createServices(electroService, config);
+  const postgrestService = createPostgrestService(config, client);
+  const services = createServices(postgrestService, config);
   const entityRegistry = new EntityRegistry(services, config, log);
 
   return entityRegistry.getCollections();

package/src/util/index.js

@@ -25,6 +25,7 @@ export {
 export {
   registerLogger,
   getLogger,
+  resetLoggerRegistry,
 } from './logger-registry.js';
 
 /**
@@ -33,6 +34,6 @@ export {
  * @enum {string}
  */
 export const DATASTORE_TYPE = Object.freeze({
-  DYNAMO: 'dynamo',
+  POSTGREST: 'postgrest',
   S3: 's3',
 });

package/src/util/logger-registry.js

@@ -29,6 +29,10 @@ class LoggerRegistry {
   getLogger() {
     return this.#logger || console;
   }
+
+  reset() {
+    this.#logger = null;
+  }
 }
 
 /**
@@ -48,3 +52,11 @@ export function registerLogger(logger) {
 export function getLogger() {
   return LoggerRegistry.getInstance().getLogger();
 }
+
+/**
+ * Resets the registered logger to default console logger.
+ * Primarily intended for test isolation.
+ */
+export function resetLoggerRegistry() {
+  LoggerRegistry.getInstance().reset();
+}