@adobe/spacecat-shared-data-access 1.21.2 → 1.22.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-data-access-v1.22.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.21.3...@adobe/spacecat-shared-data-access-v1.22.0) (2024-04-29)
+
+
+### Features
+
+* add site top pages data access (SITES-21274) ([#213](https://github.com/adobe/spacecat-shared/issues/213)) ([b892e7c](https://github.com/adobe/spacecat-shared/commit/b892e7cb056824a4f5917fa2cf8a89c7dd077f7f))
+
+# [@adobe/spacecat-shared-data-access-v1.21.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.21.2...@adobe/spacecat-shared-data-access-v1.21.3) (2024-04-25)
+
+
+### Bug Fixes
+
+* **deps:** update external fixes ([#215](https://github.com/adobe/spacecat-shared/issues/215)) ([4227263](https://github.com/adobe/spacecat-shared/commit/4227263b6bc917982d361cd621c7001c4ee1fa56))
+
 # [@adobe/spacecat-shared-data-access-v1.21.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.21.1...@adobe/spacecat-shared-data-access-v1.21.2) (2024-04-16)
 
 

package/README.md

@@ -34,6 +34,14 @@ npm install @adobe/spacecat-shared-data-access
 - **expiresAt** (Number): Expiry timestamp of the audit.
 - **fullAuditRef** (String): Reference to the full audit details.
 
+### SiteTopPages
+- **siteId** (String): Identifier of the site.
+- **url** (String): URL of the top page.
+- **traffic** (Number): Traffic of the top page.
+- **source** (String): Source of the data.
+- **geo** (String): Geo of the top page.
+- **importedAt** (String): Timestamp of the import.
+
 ## DynamoDB Data Model
 
 The module is designed to work with the following DynamoDB tables:
@@ -41,6 +49,8 @@ The module is designed to work with the following DynamoDB tables:
 1. **Sites Table**: Manages site records.
 2. **Audits Table**: Stores audit information for each site.
 3. **Latest Audits Table**: Holds only the latest audit for each site for quick access.
+4. **Site Candidates Table**: Manages site candidates.
+5. **Site Top Pages Table**: Stores top pages for each site.
 
 Each table is designed with scalability and efficient querying in mind, utilizing both key and non-key attributes effectively.
 
@@ -58,7 +68,7 @@ These tests create the schema, generate sample data, and test the data access pa
 
 ## Data Access API
 
-The module provides two main DAOs:
+The module provides the following DAOs:
 
 ### Site Functions
 - `getSites`
@@ -85,10 +95,14 @@ The module provides two main DAOs:
 - `getLatestAuditForSite`
 - `addAudit`
 
+### Site Top Pages Functions
+- `getTopPagesForSite`
+- `addSiteTopPage`
 
 ## Integrating Data Access in AWS Lambda Functions
 
-Our `spacecat-shared-data-access` module includes a wrapper that can be easily integrated into AWS Lambda functions using `@adobe/helix-shared-wrap`. This integration allows your Lambda functions to access and manipulate data seamlessly.
+Our `spacecat-shared-data-access` module includes a wrapper that can be easily integrated into AWS Lambda functions using `@adobe/helix-shared-wrap`.
+This integration allows your Lambda functions to access and manipulate data seamlessly.
 
 ### Steps for Integration
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "1.21.2",
+  "version": "1.22.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "main": "src/index.js",
@@ -31,10 +31,10 @@
   "dependencies": {
     "@adobe/spacecat-shared-dynamo": "1.2.5",
     "@adobe/spacecat-shared-utils": "1.2.0",
-    "@aws-sdk/client-dynamodb": "3.554.0",
-    "@aws-sdk/lib-dynamodb": "3.554.0",
+    "@aws-sdk/client-dynamodb": "3.556.0",
+    "@aws-sdk/lib-dynamodb": "3.556.0",
     "@types/joi": "17.2.3",
-    "joi": "17.12.3",
+    "joi": "17.13.0",
     "uuid": "9.0.1"
   },
   "devDependencies": {

package/src/dto/site-top-page.js

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { createSiteTopPage } from '../models/site-top-page.js';
+
+export function padWithZeros(number) {
+  return String(number).padStart(12, '0');
+}
+
+export const SiteTopPageDto = {
+/**
+   * Converts a SiteTopPage object into a DynamoDB item.
+   * @param {Readonly<SiteTopPage>} siteTopPage - SiteTopPage object.
+   * @returns {{siteId, url, traffic, source, geo, importedAt, SK: string}}
+   */
+  toDynamoItem: (siteTopPage) => ({
+    siteId: siteTopPage.getSiteId(),
+    url: siteTopPage.getURL(),
+    traffic: siteTopPage.getTraffic(),
+    source: siteTopPage.getSource(),
+    geo: siteTopPage.getGeo(),
+    importedAt: siteTopPage.getImportedAt(),
+    SK: `${siteTopPage.getSource()}#${siteTopPage.getGeo()}#${padWithZeros(siteTopPage.getTraffic())}`,
+  }),
+
+  /**
+   * Converts a DynamoDB item into a SiteTopPage object.
+   * @param {{siteId, url, traffic, source, geo, importedAt, SK: string}} item - DynamoDB item.
+   * @returns {SiteTopPage}
+   */
+  fromDynamoItem: (item) => createSiteTopPage({
+    siteId: item.siteId,
+    url: item.url,
+    traffic: item.traffic,
+    source: item.source,
+    geo: item.geo,
+    importedAt: item.importedAt,
+  }),
+};

package/src/index.d.ts

@@ -281,6 +281,44 @@ export interface SiteCandidate {
   getUpdatedBy: () => string;
 }
 
+export interface SiteTopPage {
+  /**
+   * Retrieves the site ID of the site top page.
+   * @returns {string} The site ID.
+   */
+  getSiteId: () => string;
+
+  /**
+   * Retrieves the URL of the site top page.
+   * @returns {string} The URL.
+   */
+  getURL: () => string;
+
+  /**
+   * Retrieves the traffic of the site top page.
+   * @returns {number} The traffic.
+   */
+  getTraffic: () => number;
+
+  /**
+   * Retrieves the source of the site top page.
+   * @returns {string} The source.
+   */
+  getSource: () => string;
+
+  /**
+   * Retrieves the geo of the site top page.
+   * @returns {string} The geo.
+   */
+  getGeo: () => string;
+
+  /**
+   * Retrieves the timestamp when the import was performed.
+   * @returns {string} The import timestamp.
+   */
+  getImportedAt: () => string;
+}
+
 export interface Organization {
   /**
    * Retrieves the ID of the site.
@@ -442,6 +480,12 @@ export interface DataAccess {
   siteCandidateExists: (baseURL: string) => Promise<boolean>;
   updateSiteCandidate: (siteCandidate: SiteCandidate) => Promise<SiteCandidate>;
 
+  // site top pages functions
+  getTopPagesForSite: (siteId: string, source: string, geo: string)
+      => Promise<Readonly<SiteTopPage>[]>;
+  addSiteTopPage: (siteTopPageData: object) => Promise<SiteTopPage>;
+  removeSiteTopPages: (siteId: string, source: string, geo: string) => Promise<void>;
+
   // configuration functions
   getConfiguration: () => Promise<Readonly<Configuration>>
   getConfigurations: () => Promise<Readonly<Configuration>[]>
@@ -456,6 +500,7 @@ interface DataAccessConfig {
   tableNameSites: string;
   tableNameSiteCandidates: string;
   tableNameConfigurations: string;
+  tableNameSiteTopPages: string;
   indexNameAllSites: string;
   indexNameAllSitesOrganizations: string,
   indexNameAllSitesByDeliveryType: string;

package/src/index.js

@@ -18,6 +18,7 @@ const TABLE_NAME_SITES = 'spacecat-services-sites-dev';
 const TABLE_NAME_SITE_CANDIDATES = 'spacecat-services-site-candidates-dev';
 const TABLE_NAME_ORGANIZATIONS = 'spacecat-services-organizations-dev';
 const TABLE_NAME_CONFIGURATIONS = 'spacecat-services-configurations-dev';
+const TABLE_NAME_SITE_TOP_PAGES = 'spacecat-services-site-top-pages-dev';
 
 const INDEX_NAME_ALL_SITES = 'spacecat-services-all-sites-dev';
 const INDEX_NAME_ALL_ORGANIZATIONS = 'spacecat-services-all-organizations-dev';
@@ -43,6 +44,7 @@ export default function dataAccessWrapper(fn) {
         DYNAMO_TABLE_NAME_SITE_CANDIDATES = TABLE_NAME_SITE_CANDIDATES,
         DYNAMO_TABLE_NAME_ORGANIZATIONS = TABLE_NAME_ORGANIZATIONS,
         DYNAMO_TABLE_NAME_CONFIGURATIONS = TABLE_NAME_CONFIGURATIONS,
+        DYNAMO_TABLE_NAME_SITE_TOP_PAGES = TABLE_NAME_SITE_TOP_PAGES,
         DYNAMO_INDEX_NAME_ALL_SITES = INDEX_NAME_ALL_SITES,
         DYNAMO_INDEX_NAME_ALL_SITES_BY_DELIVERY_TYPE = INDEX_NAME_ALL_SITES_BY_DELIVERY_TYPE,
         DYNAMO_INDEX_NAME_ALL_LATEST_AUDIT_SCORES = INDEX_NAME_ALL_LATEST_AUDIT_SCORES,
@@ -59,6 +61,7 @@ export default function dataAccessWrapper(fn) {
         tableNameSites: DYNAMO_TABLE_NAME_SITES,
         tableNameSiteCandidates: DYNAMO_TABLE_NAME_SITE_CANDIDATES,
         tableNameConfigurations: DYNAMO_TABLE_NAME_CONFIGURATIONS,
+        tableNameSiteTopPages: DYNAMO_TABLE_NAME_SITE_TOP_PAGES,
         indexNameAllSites: DYNAMO_INDEX_NAME_ALL_SITES,
         indexNameAllOrganizations: DYNAMO_INDEX_NAME_ALL_ORGANIZATIONS,
         indexNameAllOrganizationsByImsOrgId: DYNAMO_INDEX_NAME_ALL_ORGANIZATIONS_BY_IMS_ORG_ID,

package/src/models/site-top-page.js

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import {
+  hasText, isInteger, isIsoDate, isValidUrl,
+} from '@adobe/spacecat-shared-utils';
+import { Base } from './base.js';
+
+export const DEFAULT_GEO = 'global';
+
+const SiteTopPage = (data = {}) => {
+  const self = Base(data);
+
+  self.getSiteId = () => self.state.siteId;
+  self.getURL = () => self.state.url;
+  self.getTraffic = () => self.state.traffic;
+  self.getSource = () => self.state.source.toLowerCase();
+  self.getGeo = () => self.state.geo;
+  self.getImportedAt = () => self.state.importedAt;
+
+  return Object.freeze(self);
+};
+
+export const createSiteTopPage = (data) => {
+  const newState = { ...data };
+
+  if (!hasText(newState.siteId)) {
+    throw new Error('Site ID must be provided');
+  }
+
+  if (!isValidUrl(newState.url)) {
+    throw new Error('Valid Url must be provided');
+  }
+
+  if (!isInteger(newState.traffic)) {
+    throw new Error('Traffic must be provided');
+  }
+
+  if (!hasText(newState.source)) {
+    throw new Error('Source must be provided');
+  }
+
+  if (!hasText(newState.geo)) {
+    newState.geo = DEFAULT_GEO;
+  }
+
+  if (!isIsoDate(newState.importedAt)) {
+    throw new Error('Imported at must be a valid ISO date');
+  }
+
+  return SiteTopPage(newState);
+};

package/src/service/index.js

@@ -16,6 +16,7 @@ import { siteFunctions } from './sites/index.js';
 import { siteCandidateFunctions } from './site-candidates/index.js';
 import { organizationFunctions } from './organizations/index.js';
 import { configurationFunctions } from './configurations/index.js';
+import { siteTopPagesFunctions } from './site-top-pages/index.js';
 
 /**
  * Creates a data access object.
@@ -23,8 +24,8 @@ import { configurationFunctions } from './configurations/index.js';
  * @param {{pkAllSites: string, pkAllLatestAudits: string, indexNameAllLatestAuditScores: string,
  * tableNameAudits: string,tableNameLatestAudits: string, indexNameAllSitesOrganizations: string,
  * tableNameSites: string, tableNameOrganizations: string, indexNameAllSites: string,
- * indexNameAllOrganizations: string, indexNameAllOrganizationsByImsOrgId: string,
- * pkAllOrganizations: string}} config configuration
+ * tableNameSiteTopPages: string, indexNameAllOrganizations: string,
+ * indexNameAllOrganizationsByImsOrgId: string, pkAllOrganizations: string}} config configuration
  * @param {Logger} log logger
  * @returns {object} data access object
  */
@@ -36,6 +37,7 @@ export const createDataAccess = (config, log = console) => {
   const siteCandidateFuncs = siteCandidateFunctions(dynamoClient, config, log);
   const organizationFuncs = organizationFunctions(dynamoClient, config, log);
   const configurationFuncs = configurationFunctions(dynamoClient, config);
+  const siteTopPagesFuncs = siteTopPagesFunctions(dynamoClient, config);
 
   return {
     ...auditFuncs,
@@ -43,5 +45,6 @@ export const createDataAccess = (config, log = console) => {
     ...siteCandidateFuncs,
     ...organizationFuncs,
     ...configurationFuncs,
+    ...siteTopPagesFuncs,
   };
 };

package/src/service/site-top-pages/accessPatterns.js

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { hasText } from '@adobe/spacecat-shared-utils';
+
+import { createSiteTopPage } from '../../models/site-top-page.js';
+import { padWithZeros, SiteTopPageDto } from '../../dto/site-top-page.js';
+
+export const getTopPagesForSite = async (
+  dynamoClient,
+  config,
+  log,
+  siteId,
+  source,
+  geo,
+) => {
+  const queryParams = {
+    TableName: config.tableNameSiteTopPages,
+    KeyConditionExpression: 'siteId = :siteId',
+    ExpressionAttributeValues: { ':siteId': siteId },
+    ScanIndexForward: false,
+  };
+
+  if (hasText(source)) {
+    if (hasText(geo)) {
+      queryParams.KeyConditionExpression += ' AND begins_with(SK, :sourceGeo)';
+      queryParams.ExpressionAttributeValues[':sourceGeo'] = `${source}#${geo}#`;
+    } else {
+      queryParams.KeyConditionExpression += ' AND begins_with(SK, :source)';
+      queryParams.ExpressionAttributeValues[':source'] = `${source}#`;
+    }
+  }
+
+  const dynamoItems = await dynamoClient.query(queryParams);
+
+  return dynamoItems.map((item) => SiteTopPageDto.fromDynamoItem(item));
+};
+
+export const addSiteTopPage = async (
+  dynamoClient,
+  config,
+  log,
+  siteTopPageData,
+) => {
+  const newSiteTopPage = createSiteTopPage(siteTopPageData);
+
+  await dynamoClient.putItem(
+    config.tableNameSiteTopPages,
+    SiteTopPageDto.toDynamoItem(newSiteTopPage),
+  );
+
+  return newSiteTopPage;
+};
+
+export const removeSiteTopPages = async (
+  dynamoClient,
+  config,
+  log,
+  siteId,
+  source,
+  geo,
+) => {
+  const siteTopPages = await getTopPagesForSite(dynamoClient, config, log, siteId, source, geo);
+
+  const removePromises = siteTopPages.map((siteTopPage) => dynamoClient.removeItem(
+    config.tableNameSiteTopPages,
+    {
+      siteId: siteTopPage.getSiteId(),
+      SK: `${siteTopPage.getSource()}#${siteTopPage.getGeo()}#${padWithZeros(siteTopPage.getTraffic())}`,
+    },
+  ));
+
+  await Promise.all(removePromises);
+};

package/src/service/site-top-pages/index.js

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import {
+  addSiteTopPage,
+  getTopPagesForSite,
+  removeSiteTopPages,
+} from './accessPatterns.js';
+
+export const siteTopPagesFunctions = (dynamoClient, config, log) => ({
+  getTopPagesForSite: (siteId, source, geo) => getTopPagesForSite(
+    dynamoClient,
+    config,
+    log,
+    siteId,
+    source,
+    geo,
+  ),
+  addSiteTopPage: (siteTopPageData) => addSiteTopPage(
+    dynamoClient,
+    config,
+    log,
+    siteTopPageData,
+  ),
+  removeSiteTopPages: (siteId, source, geo) => removeSiteTopPages(
+    dynamoClient,
+    config,
+    log,
+    siteId,
+    source,
+    geo,
+  ),
+});