@adobe/spacecat-shared-data-access 2.24.1 → 2.26.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-data-access-v2.26.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.25.0...@adobe/spacecat-shared-data-access-v2.26.0) (2025-06-19)
+
+
+### Features
+
+* added scrape client ([#814](https://github.com/adobe/spacecat-shared/issues/814)) ([fad6614](https://github.com/adobe/spacecat-shared/commit/fad6614672a046da5319e493cc7c26bfdc3993d2))
+
+# [@adobe/spacecat-shared-data-access-v2.25.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.24.1...@adobe/spacecat-shared-data-access-v2.25.0) (2025-06-18)
+
+
+### Features
+
+* introduce ScrapeJob and ScrapeUrl entities ([#803](https://github.com/adobe/spacecat-shared/issues/803)) ([d295f65](https://github.com/adobe/spacecat-shared/commit/d295f65a89a986f08d5d3b28fe60b45d4c65ee36))
+
 # [@adobe/spacecat-shared-data-access-v2.24.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.24.0...@adobe/spacecat-shared-data-access-v2.24.1) (2025-06-14)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "2.24.1",
+  "version": "2.26.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {

package/src/models/base/entity.registry.js

@@ -25,6 +25,8 @@ import KeyEventCollection from '../key-event/key-event.collection.js';
 import LatestAuditCollection from '../latest-audit/latest-audit.collection.js';
 import OpportunityCollection from '../opportunity/opportunity.collection.js';
 import OrganizationCollection from '../organization/organization.collection.js';
+import ScrapeJobCollection from '../scrape-job/scrape-job.collection.js';
+import ScrapeUrlCollection from '../scrape-url/scrape-url.collection.js';
 import SiteCandidateCollection from '../site-candidate/site-candidate.collection.js';
 import SiteCollection from '../site/site.collection.js';
 import SiteTopPageCollection from '../site-top-page/site-top-page.collection.js';
@@ -42,6 +44,8 @@ import KeyEventSchema from '../key-event/key-event.schema.js';
 import LatestAuditSchema from '../latest-audit/latest-audit.schema.js';
 import OpportunitySchema from '../opportunity/opportunity.schema.js';
 import OrganizationSchema from '../organization/organization.schema.js';
+import ScrapeJobSchema from '../scrape-job/scrape-job.schema.js';
+import ScrapeUrlSchema from '../scrape-url/scrape-url.schema.js';
 import SiteSchema from '../site/site.schema.js';
 import SiteCandidateSchema from '../site-candidate/site-candidate.schema.js';
 import SiteTopPageSchema from '../site-top-page/site-top-page.schema.js';
@@ -140,6 +144,8 @@ EntityRegistry.registerEntity(KeyEventSchema, KeyEventCollection);
 EntityRegistry.registerEntity(LatestAuditSchema, LatestAuditCollection);
 EntityRegistry.registerEntity(OpportunitySchema, OpportunityCollection);
 EntityRegistry.registerEntity(OrganizationSchema, OrganizationCollection);
+EntityRegistry.registerEntity(ScrapeJobSchema, ScrapeJobCollection);
+EntityRegistry.registerEntity(ScrapeUrlSchema, ScrapeUrlCollection);
 EntityRegistry.registerEntity(SiteSchema, SiteCollection);
 EntityRegistry.registerEntity(SiteCandidateSchema, SiteCandidateCollection);
 EntityRegistry.registerEntity(SiteTopPageSchema, SiteTopPageCollection);

package/src/models/index.d.ts

@@ -22,6 +22,8 @@ export type * from './key-event';
 export type * from './latest-audit';
 export type * from './opportunity';
 export type * from './organization';
+export type * from './scrape-job';
+export type * from './scrape-url';
 export type * from './site';
 export type * from './site-candidate';
 export type * from './site-top-page';

package/src/models/index.js

@@ -23,6 +23,8 @@ export * from './key-event/index.js';
 export * from './latest-audit/index.js';
 export * from './opportunity/index.js';
 export * from './organization/index.js';
+export * from './scrape-job/index.js';
+export * from './scrape-url/index.js';
 export * from './site-candidate/index.js';
 export * from './site-top-page/index.js';
 export * from './site/index.js';

package/src/models/scrape-job/index.d.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import type { BaseCollection, BaseModel } from '../base';
+import type { ScrapeUrl } from '../scrape-url';
+
+export interface ScrapeJob extends BaseModel {
+  getBaseURL(): string,
+  getCustomHeaders(): IOptions,
+  getDuration(): number,
+  getEndedAt(): string,
+  getFailedCount(): number,
+  getOptions(): string,
+  getProcessingType(): string,
+  getRedirectCount(): number,
+  getResults(): string,
+  getScrapeQueueId(): string,
+  getScrapeUrls(): Promise<ScrapeUrl[]>,
+  getScrapeUrlsByStatus(status: string): Promise<ScrapeUrl[]>,
+  getStartedAt(): string,
+  getStatus(): string,
+  getSuccessCount(): number,
+  getUrlCount(): number,
+  setBaseURL(baseURL: string): void,
+  setCustomHeaders(customHeaders: IOptions): void,
+  setDuration(duration: number): void,
+  setEndedAt(endTime: string): void,
+  setFailedCount(failedCount: number): void,
+  setOptions(options: string): void,
+  setProcessingType(processingType: string): void,
+  setRedirectCount(redirectCount: number): void,
+  setResults(results: string): void,
+  setScrapeQueueId(ScrapeQueueId: string): void,
+  setStatus(status: string): void,
+  setSuccessCount(successCount: number): void,
+  setUrlCount(urlCount: number): void,
+}
+
+export interface ScrapeJobCollection extends BaseCollection<ScrapeJob> {
+  allByBaseURL(baseURL: string): Promise<ScrapeJob[]>;
+  allByBaseURLAndProcessingType(baseURL: string, processingType: string): Promise<ScrapeJob[]>;
+  allByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL: string,
+    processingType: string,
+    optEnableJavascript: string,
+    optHideConsentBanner: string): Promise<ScrapeJob[]>;
+  allByDateRange(startDate: string, endDate: string): Promise<ScrapeJob[]>;
+  allByStartedAt(startDate: string): Promise<ScrapeJob[]>;
+  allByStatus(status: string): Promise<ScrapeJob[]>;
+  allByStatusAndUpdatedAt(status: string, updatedAt: string): Promise<ScrapeJob[]>;
+  findByBaseURL(baseURL: string): Promise<ScrapeJob[]>;
+  findByBaseURLAndProcessingType(baseURL: string, processingType: string): Promise<ScrapeJob[]>;
+  findByBaseURLAndProcessingTypeAndOptEnableJavascriptAndOptHideConsentBanner(
+    baseURL: string,
+    processingType: string,
+    optEnableJavascript: string,
+    optHideConsentBanner: string): Promise<ScrapeJob[]>;
+  findByStartedAt(startDate: string): Promise<ScrapeJob | null>;
+  findByStatus(status: string): Promise<ScrapeJob | null>;
+  findByStatusAndUpdatedAt(status: string, updatedAt: string): Promise<ScrapeJob | null>;
+}

package/src/models/scrape-job/index.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import ScrapeJob from './scrape-job.model.js';
+import ScrapeJobCollection from './scrape-job.collection.js';
+
+export {
+  ScrapeJob,
+  ScrapeJobCollection,
+};

package/src/models/scrape-job/scrape-job.collection.js

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { isIsoDate } from '@adobe/spacecat-shared-utils';
+
+import { ValidationError } from '../../errors/index.js';
+import BaseCollection from '../base/base.collection.js';
+
+/**
+ * ScrapeJobCollection - A collection class responsible for managing ScrapeJob entities.
+ * Extends the BaseCollection to provide specific methods for interacting with ScrapeJob records.
+ *
+ * @class ScrapeJobCollection
+ * @extends BaseCollection
+ */
+class ScrapeJobCollection extends BaseCollection {
+  async allByDateRange(startDate, endDate) {
+    if (!isIsoDate(startDate)) {
+      throw new ValidationError(`Invalid start date: ${startDate}`);
+    }
+
+    if (!isIsoDate(endDate)) {
+      throw new ValidationError(`Invalid end date: ${endDate}`);
+    }
+
+    return this.all({}, {
+      between: {
+        attribute: 'startedAt',
+        start: startDate,
+        end: endDate,
+      },
+    });
+  }
+}
+
+export default ScrapeJobCollection;

package/src/models/scrape-job/scrape-job.model.js

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import BaseModel from '../base/base.model.js';
+
+/**
+ * ScrapeJob - A class representing an ScrapeJob entity.
+ * Provides methods to access and manipulate ScrapeJob-specific data.
+ *
+ * @class ScrapeJob
+ * @extends BaseModel
+ */
+class ScrapeJob extends BaseModel {
+  static SCRAPE_JOB_EXPIRES_IN_DAYS = 14;
+
+  /**
+   * Scrape Job Status types.
+   * Any changes to this object needs to be reflected in the index.d.ts file as well.
+   */
+  static ScrapeJobStatus = {
+    RUNNING: 'RUNNING',
+    COMPLETE: 'COMPLETE',
+    FAILED: 'FAILED',
+    STOPPED: 'STOPPED',
+  };
+
+  /**
+   * ScrapeURL Status types.
+   * Any changes to this object needs to be reflected in the index.d.ts file as well.
+   */
+  static ScrapeUrlStatus = {
+    PENDING: 'PENDING',
+    REDIRECT: 'REDIRECT',
+    ...ScrapeJob.ScrapeJobStatus,
+  };
+
+  /**
+   * Supported Scrape Options.
+   */
+  static ScrapeOptions = {
+    ENABLE_JAVASCRIPT: 'enableJavascript',
+    HIDE_CONSENT_BANNER: 'hideConsentBanners',
+    PAGE_LOAD_TIMEOUT: 'pageLoadTimeout',
+    WAIT_FOR_SELECTOR: 'waitForSelector',
+    SECTION_LOAD_WAIT_TIME: 'sectionLoadWaitTime',
+    SCREENSHOT_TYPES: 'screenshotTypes',
+  };
+
+  static ScrapeProcessingType = {
+    DEFAULT: 'default',
+    ACCESSIBILITY: 'accessibility',
+    FORM_ACCESSIBILITY: 'form-accessibility',
+    FORM: 'form',
+    TEXT_CONTENT: 'text-content',
+  };
+
+  static ScrapeScreenshotType = {
+    FULL_PAGE: 'fullPage',
+    THUMBNAIL: 'thumbnail',
+    SECTION: 'section',
+    BLOCK: 'block',
+    SCROLL: 'scroll',
+  };
+
+  // add your custom methods or overrides here
+}
+
+export default ScrapeJob;

package/src/models/scrape-job/scrape-job.schema.js

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* c8 ignore start */
+
+import {
+  isInteger,
+  isIsoDate,
+  isNumber,
+  isObject,
+  isValidUrl,
+  isString,
+} from '@adobe/spacecat-shared-utils';
+
+import SchemaBuilder from '../base/schema.builder.js';
+import ScrapeJob from './scrape-job.model.js';
+import ScrapeJobCollection from './scrape-job.collection.js';
+
+/*
+Schema Doc: https://electrodb.dev/en/modeling/schema/
+Attribute Doc: https://electrodb.dev/en/modeling/attributes/
+Indexes Doc: https://electrodb.dev/en/modeling/indexes/
+ */
+
+const schema = new SchemaBuilder(ScrapeJob, ScrapeJobCollection)
+  .withRecordExpiry(ScrapeJob.SCRAPE_JOB_EXPIRES_IN_DAYS)
+  .addReference('has_many', 'ScrapeUrls')
+  .addAttribute('baseURL', {
+    type: 'string',
+    required: true,
+    validate: (value) => isValidUrl(value),
+  })
+  .addAttribute('processingType', {
+    type: 'string',
+    required: true,
+    validate: (value) => isString(value),
+  })
+  .addAttribute('duration', {
+    type: 'number',
+    default: 0,
+    validate: (value) => !value || isNumber(value),
+  })
+  .addAttribute('endedAt', {
+    type: 'string',
+    validate: (value) => !value || isIsoDate(value),
+  })
+  .addAttribute('failedCount', {
+    type: 'number',
+    default: 0,
+    validate: (value) => !value || isInteger(value),
+  })
+  .addAttribute('scrapeQueueId', {
+    type: 'string',
+  })
+  .addAttribute('options', {
+    type: 'any',
+    validate: (value) => !value || isObject(value),
+  })
+  .addAttribute('customHeaders', {
+    type: 'any',
+  })
+  .addAttribute('redirectCount', {
+    type: 'number',
+    default: 0,
+    validate: (value) => !value || isInteger(value),
+  })
+  .addAttribute('status', {
+    type: Object.values(ScrapeJob.ScrapeJobStatus),
+    required: true,
+  })
+  .addAttribute('startedAt', {
+    type: 'string',
+    required: true,
+    readOnly: true,
+    default: () => new Date().toISOString(),
+    validate: (value) => isIsoDate(value),
+  })
+  .addAttribute('successCount', {
+    type: 'number',
+    default: 0,
+    validate: (value) => !value || isInteger(value),
+  })
+  .addAttribute('urlCount', {
+    type: 'number',
+    default: 0,
+    validate: (value) => !value || isInteger(value),
+  })
+  .addAttribute('results', {
+    type: 'any',
+  })
+  .addAttribute('optEnableJavascript', {
+    type: 'string',
+    hidden: true,
+    readOnly: true,
+    watch: ['options'],
+    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.ENABLE_JAVASCRIPT] ? 'T' : 'F'),
+  })
+  .addAttribute('optHideConsentBanner', {
+    type: 'string',
+    hidden: true,
+    readOnly: true,
+    watch: ['options'],
+    set: (_, { options }) => (options[ScrapeJob.ScrapeOptions.HIDE_CONSENT_BANNER] ? 'T' : 'F'),
+  })
+  // access pattern: get all jobs sorted by startedAt
+  .addAllIndex(['startedAt'])
+  .addIndex(
+    { composite: ['baseURL'] },
+    { composite: ['processingType', 'startedAt'] },
+  )
+  // access pattern: get all jobs for a given baseURL and processingType,
+  // can be filtered by optEnableJavascript and optHideConsentBanner
+  // are solrted by startedAt
+  .addIndex(
+    { composite: ['baseURL', 'processingType'] },
+    { composite: ['optEnableJavascript', 'optHideConsentBanner', 'startedAt'] },
+  )
+  // access pattern: get all jobs for a given status, sorted by updatedAt
+  .addIndex(
+    { composite: ['status'] },
+    { composite: ['updatedAt'] },
+  );
+
+export default schema.build();

package/src/models/scrape-url/index.d.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import type { BaseCollection, BaseModel, ScrapeJob } from '../index';
+
+export interface ScrapeUrl extends BaseModel {
+  getFile(): string,
+  getScrapeJob(): Promise<ScrapeJob>,
+  getScrapeJobId(): string,
+  getPath(): string,
+  getReason(): string,
+  getStatus(): string,
+  getUrl(): string,
+  setFile(file: string): void,
+  setScrapeJobId(ScrapeJobId: string): void,
+  setPath(path: string): void,
+  setReason(reason: string): void,
+  setStatus(status: string): void,
+  setUrl(url: string): void,
+}
+
+export interface ScrapeUrlCollection extends BaseCollection<ScrapeUrl> {
+  allByScrapeJobId(ScrapeJobId: string): Promise<ScrapeUrl[]>;
+  allByScrapeUrlsByJobIdAndStatus(ScrapeJobId: string, status: string): Promise<ScrapeUrl[]>;
+  findByScrapeJobId(ScrapeJobId: string): Promise<ScrapeUrl | null>;
+  findByScrapeJobIdAndUrl(ScrapeJobId: string, url: string): Promise<ScrapeUrl | null>;
+}

package/src/models/scrape-url/index.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import ScrapeUrl from './scrape-url.model.js';
+import ScrapeUrlCollection from './scrape-url.collection.js';
+
+export {
+  ScrapeUrl,
+  ScrapeUrlCollection,
+};

package/src/models/scrape-url/scrape-url.collection.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import BaseCollection from '../base/base.collection.js';
+
+/**
+ * ScraperUrlCollection - A collection class responsible for managing ScraperUrl entities.
+ * Extends the BaseCollection to provide specific methods for interacting with ScraperUrl records.
+ *
+ * @class ScraperUrlCollection
+ * @extends BaseCollection
+ */
+class ScrapeUrlCollection extends BaseCollection {
+  // add custom methods here
+}
+
+export default ScrapeUrlCollection;

package/src/models/scrape-url/scrape-url.model.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import BaseModel from '../base/base.model.js';
+
+/**
+ * ScraperUrl - A class representing an ScraperUrl entity.
+ * Provides methods to access and manipulate ScraperUrl-specific data.
+ *
+ * @class ScraperUrl
+ * @extends BaseModel
+ */
+class ScrapeUrl extends BaseModel {
+  static SCRAPE_URL_EXPIRES_IN_DAYS = 14;
+
+  // add your custom methods or overrides here
+}
+
+export default ScrapeUrl;

package/src/models/scrape-url/scrape-url.schema.js

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* c8 ignore start */
+
+import { isValidUrl } from '@adobe/spacecat-shared-utils';
+
+import SchemaBuilder from '../base/schema.builder.js';
+import ScrapeUrl from './scrape-url.model.js';
+import ScrapeUrlCollection from './scrape-url.collection.js';
+import { ScrapeJob } from '../scrape-job/index.js';
+
+/*
+Schema Doc: https://electrodb.dev/en/modeling/schema/
+Attribute Doc: https://electrodb.dev/en/modeling/attributes/
+Indexes Doc: https://electrodb.dev/en/modeling/indexes/
+ */
+
+const schema = new SchemaBuilder(ScrapeUrl, ScrapeUrlCollection)
+  .withRecordExpiry(ScrapeUrl.SCRAPE_URL_EXPIRES_IN_DAYS)
+  .addReference('belongs_to', 'ScrapeJob', ['status'])
+  .addAttribute('file', {
+    type: 'string',
+  })
+  .addAttribute('path', {
+    type: 'string',
+  })
+  .addAttribute('reason', {
+    type: 'string',
+  })
+  .addAttribute('status', {
+    type: Object.values(ScrapeJob.ScrapeUrlStatus),
+    required: true,
+  })
+  .addAttribute('url', {
+    type: 'string',
+    required: true,
+    validate: (value) => isValidUrl(value),
+  });
+
+export default schema.build();