@adobe/spacecat-shared-data-access 2.74.2 → 2.75.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-data-access-v2.75.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.74.2...@adobe/spacecat-shared-data-access-v2.75.0) (2025-10-29)
+
+
+### Features
+
+* scrape url access pattern ([#1059](https://github.com/adobe/spacecat-shared/issues/1059)) ([26facf6](https://github.com/adobe/spacecat-shared/commit/26facf63dfae4908ed88221f16266910205e2473))
+
 # [@adobe/spacecat-shared-data-access-v2.74.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.74.1...@adobe/spacecat-shared-data-access-v2.74.2) (2025-10-28)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "2.74.2",
+  "version": "2.75.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {

package/src/models/audit/audit.model.js

@@ -67,6 +67,7 @@ class Audit extends BaseModel {
     READABILITY: 'readability',
     PRERENDER: 'prerender',
     PRODUCT_METATAGS: 'product-metatags',
+    PRODUCT_METATAGS_AUTO_SUGGEST: 'product-metatags-auto-suggest',
     SUMMARIZATION: 'summarization',
     PAGE_TYPE_DETECTION: 'page-type-detection',
     FAQS: 'faqs',

package/src/models/scrape-url/index.d.ts

@@ -20,6 +20,7 @@ export interface ScrapeUrl extends BaseModel {
   getReason(): string,
   getStatus(): string,
   getUrl(): string,
+  getProcessingType(): string,
   getIsOriginal(): boolean,
   setFile(file: string): void,
   setScrapeJobId(ScrapeJobId: string): void,
@@ -27,12 +28,15 @@ export interface ScrapeUrl extends BaseModel {
   setReason(reason: string): void,
   setStatus(status: string): void,
   setUrl(url: string): void,
+  setProcessingType(processingType: string): void,
   setIsOriginal(isOriginal: boolean): void,
 }
 
 export interface ScrapeUrlCollection extends BaseCollection<ScrapeUrl> {
   allByScrapeJobId(ScrapeJobId: string): Promise<ScrapeUrl[]>;
   allByScrapeUrlsByJobIdAndStatus(ScrapeJobId: string, status: string): Promise<ScrapeUrl[]>;
+  allByUrlAndIsOriginalAndProcessingType(url: string, isOriginal: boolean, processingType: string): Promise<ScrapeUrl[]>;
+  allRecentByUrlAndProcessingType(url: string, processingType: string, maxAgeInHours?: number): Promise<ScrapeUrl[]>;
   findByScrapeJobId(ScrapeJobId: string): Promise<ScrapeUrl | null>;
   findByScrapeJobIdAndUrl(ScrapeJobId: string, url: string): Promise<ScrapeUrl | null>;
 }

package/src/models/scrape-url/scrape-url.collection.js

@@ -20,7 +20,20 @@ import BaseCollection from '../base/base.collection.js';
  * @extends BaseCollection
  */
 class ScrapeUrlCollection extends BaseCollection {
-  // add custom methods here
+  async allRecentByUrlAndProcessingType(url, processingType, maxAgeInHours = 168) {
+    const now = new Date();
+    const pastDate = new Date(now.getTime() - maxAgeInHours * 60 * 60 * 1000);
+    const pastDateIso = pastDate.toISOString();
+    const nowIso = now.toISOString();
+
+    return this.allByIndexKeys({ url, isOriginal: true, processingType }, {
+      between: {
+        attribute: 'createdAt',
+        start: pastDateIso,
+        end: nowIso,
+      },
+    });
+  }
 }
 
 export default ScrapeUrlCollection;

package/src/models/scrape-url/scrape-url.schema.js

@@ -12,7 +12,7 @@
 
 /* c8 ignore start */
 
-import { isValidUrl } from '@adobe/spacecat-shared-utils';
+import { isObject, isString, isValidUrl } from '@adobe/spacecat-shared-utils';
 
 import SchemaBuilder from '../base/schema.builder.js';
 import ScrapeUrl from './scrape-url.model.js';
@@ -46,9 +46,22 @@ const schema = new SchemaBuilder(ScrapeUrl, ScrapeUrlCollection)
     required: true,
     validate: (value) => isValidUrl(value),
   })
+  .addAttribute('processingType', {
+    type: 'string',
+    required: true,
+    validate: (value) => isString(value),
+  })
+  .addAttribute('options', {
+    type: 'any',
+    validate: (value) => !value || isObject(value),
+  })
   .addAttribute('isOriginal', {
     type: 'boolean',
     default: true,
-  });
+  })
+  .addIndex(
+    { composite: ['url'] },
+    { composite: ['isOriginal', 'processingType', 'createdAt'] },
+  );
 
 export default schema.build();