@adobe/spacecat-shared-tokowaka-client 1.1.1 → 1.2.0

package/src/utils/custom-html-utils.js

@@ -0,0 +1,195 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { hasText } from '@adobe/spacecat-shared-utils';
+
+/**
+ * Helper function to wait for a specified duration
+ * @param {number} ms - Milliseconds to wait
+ * @returns {Promise<void>}
+ */
+function sleep(ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+/**
+ * Makes an HTTP request with retry logic
+ * Retries until max retries are exhausted or x-tokowaka-cache header is present
+ * @param {string} url - URL to fetch
+ * @param {Object} options - Fetch options
+ * @param {number} maxRetries - Maximum number of retries
+ * @param {number} retryDelayMs - Delay between retries in milliseconds
+ * @param {Object} log - Logger instance
+ * @param {string} fetchType - Context for logging (e.g., "optimized" or "original")
+ * @returns {Promise<Response>} - Fetch response
+ */
+async function fetchWithRetry(url, options, maxRetries, retryDelayMs, log, fetchType) {
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt += 1) {
+    try {
+      log.debug(`Retry attempt ${attempt}/${maxRetries} for ${fetchType} HTML`);
+
+      // eslint-disable-next-line no-await-in-loop
+      const response = await fetch(url, options);
+
+      log.debug(`Response status (attempt ${attempt}): ${response.status} ${response.statusText}`);
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+
+      // Check for x-tokowaka-cache header - if present, stop retrying
+      const cacheHeader = response.headers.get('x-tokowaka-cache');
+      if (cacheHeader) {
+        log.debug(`Cache header found (x-tokowaka-cache: ${cacheHeader}), stopping retry logic`);
+        return response;
+      }
+
+      // If no cache header and we haven't exhausted retries, continue
+      if (attempt < maxRetries + 1) {
+        log.debug(`No cache header found on attempt ${attempt}, will retry...`);
+        // Wait before retrying
+        log.debug(`Waiting ${retryDelayMs}ms before retry...`);
+        // eslint-disable-next-line no-await-in-loop
+        await sleep(retryDelayMs);
+      } else {
+        // Last attempt without cache header - throw error
+        log.error(`Max retries (${maxRetries}) exhausted without cache header`);
+        throw new Error(`Cache header (x-tokowaka-cache) not found after ${maxRetries} retries`);
+      }
+    } catch (error) {
+      log.warn(`Attempt ${attempt} failed for ${fetchType} HTML, error: ${error.message}`);
+
+      // If this was the last attempt, throw the error
+      if (attempt === maxRetries + 1) {
+        throw error;
+      }
+
+      // Wait before retrying
+      log.debug(`Waiting ${retryDelayMs}ms before retry...`);
+      // eslint-disable-next-line no-await-in-loop
+      await sleep(retryDelayMs);
+    }
+  }
+  /* c8 ignore next */
+  throw new Error(`Failed to fetch ${fetchType} HTML after ${maxRetries} retries`);
+}
+
+/**
+ * Fetches HTML content from Tokowaka edge with warmup call and retry logic
+ * Makes an initial warmup call, waits, then makes the actual call with retries
+ * @param {string} url - Full URL to fetch
+ * @param {string} apiKey - Tokowaka API key
+ * @param {string} forwardedHost - Host to forward in x-forwarded-host header
+ * @param {string} tokowakaEdgeUrl - Tokowaka edge URL
+ * @param {boolean} isOptimized - Whether to fetch optimized HTML (with preview param)
+ * @param {Object} log - Logger instance
+ * @param {Object} options - Additional options
+ * @param {number} options.warmupDelayMs - Delay after warmup call (default: 2000ms)
+ * @param {number} options.maxRetries - Maximum number of retries for actual call (default: 2)
+ * @param {number} options.retryDelayMs - Delay between retries (default: 1000ms)
+ * @returns {Promise<string>} - HTML content
+ * @throws {Error} - If validation fails or fetch fails after retries
+ */
+export async function fetchHtmlWithWarmup(
+  url,
+  apiKey,
+  forwardedHost,
+  tokowakaEdgeUrl,
+  log,
+  isOptimized = false,
+  options = {},
+) {
+  // Validate required parameters
+  if (!hasText(url)) {
+    throw new Error('URL is required for fetching HTML');
+  }
+
+  if (!hasText(apiKey)) {
+    throw new Error('Tokowaka API key is required for fetching HTML');
+  }
+
+  if (!hasText(forwardedHost)) {
+    throw new Error('Forwarded host is required for fetching HTML');
+  }
+
+  if (!hasText(tokowakaEdgeUrl)) {
+    throw new Error('TOKOWAKA_EDGE_URL is not configured');
+  }
+
+  // Default options
+  const {
+    warmupDelayMs = 2000,
+    maxRetries = 3,
+    retryDelayMs = 1000,
+  } = options;
+
+  const fetchType = isOptimized ? 'optimized' : 'original';
+
+  // Parse the URL to extract path and construct full URL
+  const urlObj = new URL(url);
+  const urlPath = urlObj.pathname + urlObj.search;
+
+  // Add tokowakaPreview param for optimized HTML
+  let fullUrl = `${tokowakaEdgeUrl}${urlPath}`;
+  if (isOptimized) {
+    const separator = urlPath.includes('?') ? '&' : '?';
+    fullUrl = `${fullUrl}${separator}tokowakaPreview=true`;
+  }
+
+  const headers = {
+    'x-forwarded-host': forwardedHost,
+    'x-tokowaka-api-key': apiKey,
+    'x-tokowaka-url': urlPath,
+  };
+
+  const fetchOptions = {
+    method: 'GET',
+    headers,
+  };
+
+  try {
+    // Warmup call (no retry logic for warmup)
+    log.debug(`Making warmup call for ${fetchType} HTML with URL: ${fullUrl}`);
+
+    const warmupResponse = await fetch(fullUrl, fetchOptions);
+
+    log.debug(`Warmup response status: ${warmupResponse.status} ${warmupResponse.statusText}`);
+    // Consume the response body to free up the connection
+    await warmupResponse.text();
+    log.debug(`Warmup call completed, waiting ${warmupDelayMs}ms...`);
+
+    // Wait before actual call
+    await sleep(warmupDelayMs);
+
+    // Actual call with retry logic
+    log.debug(`Making actual call for ${fetchType} HTML (max ${maxRetries} retries) with URL: ${fullUrl}`);
+
+    const response = await fetchWithRetry(
+      fullUrl,
+      fetchOptions,
+      maxRetries,
+      retryDelayMs,
+      log,
+      fetchType,
+    );
+
+    const html = await response.text();
+    log.debug(`Successfully fetched ${fetchType} HTML (${html.length} bytes)`);
+    return html;
+  } catch (error) {
+    const errorMsg = `Failed to fetch ${fetchType} HTML after ${maxRetries} retries: ${error.message}`;
+    log.error(errorMsg);
+    throw new Error(errorMsg);
+  }
+}

package/src/utils/markdown-utils.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { toHast } from 'mdast-util-to-hast';
+import { fromMarkdown } from 'mdast-util-from-markdown';
+
+/**
+ * Converts markdown text to HAST (Hypertext Abstract Syntax Tree) format
+ * @param {string} markdown - Markdown text
+ * @returns {Object} - HAST object
+ */
+export function markdownToHast(markdown) {
+  const mdast = fromMarkdown(markdown);
+  return toHast(mdast);
+}

package/src/utils/patch-utils.js

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Generates a unique key for a patch based on its structure
+ * Individual patches (one suggestion per patch):
+ *    → Key: opportunityId:suggestionId
+ * Patches with no suggestionId:
+ *    → Key: opportunityId
+ */
+export function getPatchKey(patch) {
+  // Heading patch (no suggestionId): use special key
+  if (!patch.suggestionId) {
+    return `${patch.opportunityId}`;
+  }
+
+  // Individual patches include suggestionId in key
+  // This ensures each suggestion gets its own separate patch
+  return `${patch.opportunityId}:${patch.suggestionId}`;
+}
+
+/**
+ * Merges new patches into existing patches based on patch keys
+ * - If a patch with the same key exists, it's updated
+ * - If a patch with a new key is found, it's added
+ * @param {Array} existingPatches - Array of existing patches
+ * @param {Array} newPatches - Array of new patches to merge
+ * @returns {Object} - { patches: Array, updateCount: number, addCount: number }
+ */
+export function mergePatches(existingPatches, newPatches) {
+  // Create a map of existing patches by their key
+  const patchMap = new Map();
+  existingPatches.forEach((patch, index) => {
+    const key = getPatchKey(patch);
+    patchMap.set(key, { patch, index });
+  });
+
+  // Process new patches
+  const mergedPatches = [...existingPatches];
+  let updateCount = 0;
+  let addCount = 0;
+
+  newPatches.forEach((newPatch) => {
+    const key = getPatchKey(newPatch);
+    const existing = patchMap.get(key);
+
+    if (existing) {
+      mergedPatches[existing.index] = newPatch;
+      updateCount += 1;
+    } else {
+      mergedPatches.push(newPatch);
+      addCount += 1;
+    }
+  });
+
+  return { patches: mergedPatches, updateCount, addCount };
+}
+
+/**
+ * Removes patches matching the given suggestion IDs from a config
+ * Works with flat config structure
+ * @param {Object} config - Tokowaka configuration object
+ * @param {Array<string>} suggestionIds - Array of suggestion IDs to remove
+ * @param {Array<string>} additionalPatchKeys - Optional array of additional patch keys to remove
+ * @returns {Object} - Updated configuration with patches removed
+ */
+export function removePatchesBySuggestionIds(config, suggestionIds, additionalPatchKeys = []) {
+  if (!config || !config.patches) {
+    return config;
+  }
+
+  const suggestionIdSet = new Set(suggestionIds);
+  const patchKeysToRemove = new Set(additionalPatchKeys);
+  let removedCount = 0;
+
+  // Filter out patches with matching suggestionIds or additional patch keys
+  const filteredPatches = config.patches.filter((patch) => {
+    const shouldRemoveBySuggestionId = suggestionIdSet.has(patch.suggestionId);
+    const patchKey = getPatchKey(patch);
+    const shouldRemoveByPatchKey = patchKeysToRemove.has(patchKey);
+
+    if (shouldRemoveBySuggestionId || shouldRemoveByPatchKey) {
+      removedCount += 1;
+      return false;
+    }
+    return true;
+  });
+
+  return {
+    ...config,
+    patches: filteredPatches,
+    removedCount,
+  };
+}

package/src/utils/s3-utils.js

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Normalizes a URL pathname for S3 storage
+ * - Removes trailing slash (except for root '/')
+ * - Ensures starts with '/'
+ * @param {string} pathname - URL pathname
+ * @returns {string} - Normalized pathname
+ */
+export function normalizePath(pathname) {
+  let normalized = pathname.endsWith('/') && pathname !== '/' ? pathname.slice(0, -1) : pathname;
+
+  if (!normalized.startsWith('/')) {
+    normalized = `/${normalized}`;
+  }
+
+  return normalized;
+}
+
+/**
+ * Extracts and normalizes hostname from URL
+ * - Strips 'www.' prefix
+ * @param {URL} url - URL object
+ * @param {Object} logger - Logger instance
+ * @returns {string} - Normalized hostname
+ * @throws {Error} - If hostname extraction fails
+ */
+export function getHostName(url, logger) {
+  try {
+    const finalHostname = url.hostname.replace(/^www\./, '');
+    return finalHostname;
+  } catch (error) {
+    logger.error(`Error extracting host name: ${error.message}`);
+    throw new Error(`Error extracting host name: ${url.toString()}`);
+  }
+}
+
+/**
+ * Base64 URL encodes a string (RFC 4648)
+ * - Uses URL-safe characters (- instead of +, _ instead of /)
+ * - Removes padding (=)
+ * @param {string} input - String to encode
+ * @returns {string} - Base64 URL encoded string
+ */
+export function base64UrlEncode(input) {
+  // Encode to UTF-8 bytes
+  const bytes = new TextEncoder().encode(input);
+  // Convert bytes → binary string
+  let binary = '';
+  for (let i = 0; i < bytes.length; i += 1) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  // Standard base64
+  const base64 = btoa(binary);
+  // Convert to base64url (RFC 4648)
+  return base64
+    .replace(/\+/g, '-') // + → -
+    .replace(/\//g, '_') // / → _
+    .replace(/=+$/, ''); // remove padding
+}
+
+/**
+ * Generates S3 path for Tokowaka configuration based on URL
+ * @param {string} url - Full URL (e.g., 'https://www.example.com/products/item')
+ * @param {Object} logger - Logger instance
+ * @param {boolean} isPreview - Whether this is a preview path
+ * @returns {string} - S3 path (e.g., 'opportunities/example.com/L3Byb2R1Y3RzL2l0ZW0')
+ * @throws {Error} - If URL parsing fails
+ */
+export function getTokowakaConfigS3Path(url, logger, isPreview = false) {
+  try {
+    const urlObj = new URL(url);
+    let path = urlObj.pathname;
+
+    path = normalizePath(path);
+    path = base64UrlEncode(path);
+
+    const normalizedHostName = getHostName(urlObj, logger);
+    const prefix = isPreview ? 'preview/opportunities' : 'opportunities';
+
+    return `${prefix}/${normalizedHostName}/${path}`;
+  } catch (error) {
+    logger.error(`Error generating S3 path for URL ${url}: ${error.message}`);
+    throw new Error(`Failed to generate S3 path: ${error.message}`);
+  }
+}
+
+/**
+ * Generates S3 path for domain-level metaconfig
+ * @param {string} url - Full URL (used to extract domain)
+ * @param {Object} logger - Logger instance
+ * @param {boolean} isPreview - Whether this is a preview path
+ * @returns {string} - S3 path for metaconfig (e.g., 'opportunities/example.com/config')
+ * @throws {Error} - If URL parsing fails
+ */
+export function getTokowakaMetaconfigS3Path(url, logger, isPreview = false) {
+  try {
+    const urlObj = new URL(url);
+    const normalizedHostName = getHostName(urlObj, logger);
+    const prefix = isPreview ? 'preview/opportunities' : 'opportunities';
+
+    return `${prefix}/${normalizedHostName}/config`;
+  } catch (error) {
+    logger.error(`Error generating metaconfig S3 path for URL ${url}: ${error.message}`);
+    throw new Error(`Failed to generate metaconfig S3 path: ${error.message}`);
+  }
+}

package/src/utils/site-utils.js

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { isValidUrl } from '@adobe/spacecat-shared-utils';
+
+/**
+ * Gets the effective base URL for a site, respecting overrideBaseURL if configured
+ * @param {Object} site - Site entity
+ * @returns {string} - Base URL to use
+ */
+export function getEffectiveBaseURL(site) {
+  const overrideBaseURL = site.getConfig()?.getFetchConfig?.()?.overrideBaseURL;
+  return (overrideBaseURL && isValidUrl(overrideBaseURL))
+    ? overrideBaseURL
+    : site.getBaseURL();
+}

package/src/utils/suggestion-utils.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Groups suggestions by URL pathname
+ * @param {Array} suggestions - Array of suggestion entities
+ * @param {string} baseURL - Base URL for pathname extraction
+ * @param {Object} log - Logger instance
+ * @returns {Object} - Object with URL paths as keys and arrays of suggestions as values
+ */
+export function groupSuggestionsByUrlPath(suggestions, baseURL, log) {
+  return suggestions.reduce((acc, suggestion) => {
+    const data = suggestion.getData();
+    const url = data?.url;
+
+    if (!url) {
+      log.warn(`Suggestion ${suggestion.getId()} does not have a URL, skipping`);
+      return acc;
+    }
+
+    let urlPath;
+    try {
+      urlPath = new URL(url, baseURL).pathname;
+    } catch (e) {
+      log.warn(`Failed to extract pathname from URL for suggestion ${suggestion.getId()}: ${url}`);
+      return acc;
+    }
+
+    if (!acc[urlPath]) {
+      acc[urlPath] = [];
+    }
+    acc[urlPath].push(suggestion);
+    return acc;
+  }, {});
+}
+
+/**
+ * Filters suggestions into eligible and ineligible based on mapper's canDeploy method
+ * @param {Array} suggestions - Array of suggestion entities
+ * @param {Object} mapper - Mapper instance with canDeploy method
+ * @returns {Object} - { eligible: Array, ineligible: Array<{suggestion, reason}> }
+ */
+export function filterEligibleSuggestions(suggestions, mapper) {
+  const eligible = [];
+  const ineligible = [];
+
+  suggestions.forEach((suggestion) => {
+    const eligibility = mapper.canDeploy(suggestion);
+    if (eligibility.eligible) {
+      eligible.push(suggestion);
+    } else {
+      ineligible.push({
+        suggestion,
+        reason: eligibility.reason || 'Suggestion cannot be deployed',
+      });
+    }
+  });
+
+  return { eligible, ineligible };
+}