@adobe/spacecat-shared-data-access 2.9.4 → 2.9.5

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-data-access-v2.9.5](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.9.4...@adobe/spacecat-shared-data-access-v2.9.5) (2025-02-28)
+
+
+### Bug Fixes
+
+* **override-base-url:** add overrideBaseURL in site.config.fetchConfig ([#633](https://github.com/adobe/spacecat-shared/issues/633)) ([526874f](https://github.com/adobe/spacecat-shared/commit/526874f8665e7bae94ca8968f1cb834cd424ae9f))
+
 # [@adobe/spacecat-shared-data-access-v2.9.4](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.9.3...@adobe/spacecat-shared-data-access-v2.9.4) (2025-02-27)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "2.9.4",
+  "version": "2.9.5",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "engines": {
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@adobe/spacecat-shared-utils": "1.26.4",
+    "@adobe/spacecat-shared-utils": "1.33.1",
     "@aws-sdk/client-dynamodb": "3.751.0",
     "@aws-sdk/lib-dynamodb": "3.751.0",
     "@types/joi": "17.2.3",
@@ -48,6 +48,7 @@
     "chai": "5.2.0",
     "chai-as-promised": "8.0.1",
     "dynamo-db-local": "9.4.0",
+    "nock": "14.0.1",
     "sinon": "19.0.2",
     "sinon-chai": "4.0.0"
   }

package/src/models/site/config.js

@@ -39,8 +39,10 @@ export const IMPORT_TYPE_SCHEMAS = {
   [IMPORT_TYPES.ORGANIC_KEYWORDS]: Joi.object({
     type: Joi.string().valid(IMPORT_TYPES.ORGANIC_KEYWORDS).required(),
     ...IMPORT_BASE_KEYS,
-    limit: Joi.number().integer().min(1).max(100),
-    pageUrl: Joi.string().uri(),
+    geo: Joi.string().optional(),
+    limit: Joi.number().integer().min(1).max(100)
+      .optional(),
+    pageUrl: Joi.string().uri().optional(),
   }),
   [IMPORT_TYPES.ORGANIC_TRAFFIC]: Joi.object({
     type: Joi.string().valid(IMPORT_TYPES.ORGANIC_TRAFFIC).required(),
@@ -49,7 +51,9 @@ export const IMPORT_TYPE_SCHEMAS = {
   [IMPORT_TYPES.TOP_PAGES]: Joi.object({
     type: Joi.string().valid(IMPORT_TYPES.TOP_PAGES).required(),
     ...IMPORT_BASE_KEYS,
-    geo: Joi.string(),
+    geo: Joi.string().optional(),
+    limit: Joi.number().integer().min(1).max(2000)
+      .optional(),
   }),
 };
 
@@ -86,6 +90,7 @@ export const configSchema = Joi.object({
   ),
   fetchConfig: Joi.object({
     headers: Joi.object().pattern(Joi.string(), Joi.string()),
+    overrideBaseURL: Joi.string().uri().optional(),
   }).optional(),
   handlers: Joi.object().pattern(Joi.string(), Joi.object({
     mentions: Joi.object().pattern(Joi.string(), Joi.array().items(Joi.string())),

package/src/models/site/index.d.ts

@@ -49,6 +49,7 @@ export interface ImportConfig {
   enabled: boolean;
   pageUrl?: string;
   geo?: string;
+  limit?: number;
 }
 
 export interface SiteConfig {
@@ -83,6 +84,7 @@ export interface SiteConfig {
     }>;
     fetchConfig?: {
       headers?: Record<string, string>;
+      overrideBaseURL?: string;
     };
   };
   getSlackConfig(): { workspace?: string; channel?: string; invitedUserCount?: number };
@@ -102,7 +104,7 @@ export interface SiteConfig {
   getGroupedURLs(type: string): Array<{ name: string; pattern: string }> | undefined;
   getLatestMetrics(type: string):
     { pageViewsChange: number; ctrChange: number; projectedTrafficValue: number } | undefined;
-  getFetchConfig(): { headers?: Record<string, string> } | undefined;
+  getFetchConfig(): { headers?: Record<string, string>, overrideBaseURL?: string } | undefined;
 }
 
 export interface Site extends BaseModel {

package/src/models/site/site.model.js

@@ -10,6 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
+import { composeAuditURL, hasText, isValidUrl } from '@adobe/spacecat-shared-utils';
 import BaseModel from '../base/base.model.js';
 
 /**
@@ -32,6 +33,32 @@ class Site extends BaseModel {
     this.setIsLive(newIsLive);
     return this;
   }
+
+  /**
+   * Resolves the site's base URL to a  final URL by fetching the URL,
+   * following the redirects and returning the final URL.
+   *
+   * If the site has a configured overrideBaseURL, that one will be returned.
+   * Otherwise, the site's base URL will be used.
+   *
+   * If the site has a configured User-Agent, it will be used to resolve the URL.
+   *
+   * @returns a promise that resolves the final URL.
+   * @throws {Error} if the final URL cannot be resolved.
+   */
+  async resolveFinalURL() {
+    const overrideBaseURL = this.getConfig()?.getFetchConfig()?.overrideBaseURL;
+    if (isValidUrl(overrideBaseURL)) {
+      return overrideBaseURL.replace(/^https?:\/\//, '');
+    }
+
+    const userAgentConfigured = this.getConfig()?.getFetchConfig()?.headers?.['User-Agent'];
+    if (hasText(userAgentConfigured)) {
+      return composeAuditURL(this.getBaseURL(), userAgentConfigured);
+    }
+
+    return composeAuditURL(this.getBaseURL());
+  }
 }
 
 export default Site;