@adobe/spacecat-shared-rum-api-client 2.34.1 → 2.36.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.36.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.35.0...@adobe/spacecat-shared-rum-api-client-v2.36.0) (2025-07-25)
+
+
+### Features
+
+* classify organic llm referral traffic ([#877](https://github.com/adobe/spacecat-shared/issues/877)) ([63e029e](https://github.com/adobe/spacecat-shared/commit/63e029ee3db343d487ba1cbe2a006b11dce683bf))
+
+# [@adobe/spacecat-shared-rum-api-client-v2.35.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.34.1...@adobe/spacecat-shared-rum-api-client-v2.35.0) (2025-07-24)
+
+
+### Features
+
+* **rum-api-client:** add query stream ([#872](https://github.com/adobe/spacecat-shared/issues/872)) ([7702138](https://github.com/adobe/spacecat-shared/commit/77021388a298843f3ec2571c9a6f148184fa2814))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.34.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.34.0...@adobe/spacecat-shared-rum-api-client-v2.34.1) (2025-07-24)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.34.1",
+  "version": "2.36.0",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "engines": {

package/src/common/rum-bundler-client.js

@@ -207,22 +207,7 @@ async function mergeBundlesWithSameId(bundles) {
 }
 /* c8 ignore end */
 
-async function fetchBundles(opts, log) {
-  const {
-    domain,
-    domainkey,
-    interval = 7,
-    granularity = GRANULARITY.DAILY,
-    checkpoints = [],
-    filterBotTraffic = true,
-    startTime,
-    endTime,
-  } = opts;
-
-  if (!hasText(domain) || !hasText(domainkey)) {
-    throw new Error('Missing required parameters');
-  }
-
+function validateDateRange(startTime, endTime) {
   // Validate startTime and endTime if provided
   if (startTime && endTime) {
     const start = parseDate(startTime);
@@ -236,27 +221,48 @@ async function fetchBundles(opts, log) {
       throw new Error('startTime must be before endTime');
     }
   }
+}
 
-  let urls = [];
-
+function generateURLs(domain, granularity, domainkey, startTime, endTime, interval) {
   if (startTime && endTime) {
+    validateDateRange(startTime, endTime);
     // Use custom date range
-    urls = generateUrlsForDateRange(startTime, endTime, domain, granularity, domainkey);
-  } else {
-    // Use existing interval-based logic
-    const multiplier = granularity.toUpperCase() === GRANULARITY.HOURLY ? ONE_HOUR : ONE_DAY;
-    const range = granularity.toUpperCase() === GRANULARITY.HOURLY
-      ? interval * HOURS_IN_DAY
-      : interval + 1;
-
-    const currentDate = new Date();
-
-    for (let i = 0; i < range; i += 1) {
-      const date = new Date(currentDate.getTime() - i * multiplier);
-      urls.push(constructUrl(domain, date, granularity, domainkey));
-    }
+    return generateUrlsForDateRange(startTime, endTime, domain, granularity, domainkey);
+  }
+
+  // Use existing interval-based logic
+  const multiplier = granularity.toUpperCase() === GRANULARITY.HOURLY ? ONE_HOUR : ONE_DAY;
+  const range = granularity.toUpperCase() === GRANULARITY.HOURLY
+    ? interval * HOURS_IN_DAY
+    : interval + 1;
+
+  const currentDate = new Date();
+  const urls = [];
+
+  for (let i = 0; i < range; i += 1) {
+    const date = new Date(currentDate.getTime() - i * multiplier);
+    urls.push(constructUrl(domain, date, granularity, domainkey));
+  }
+  return urls;
+}
+
+async function fetchBundles(opts, log) {
+  const {
+    domain,
+    domainkey,
+    interval = 7,
+    granularity = GRANULARITY.DAILY,
+    checkpoints = [],
+    filterBotTraffic = true,
+    startTime,
+    endTime,
+  } = opts;
+
+  if (!hasText(domain) || !hasText(domainkey)) {
+    throw new Error('Missing required parameters');
   }
 
+  const urls = generateURLs(domain, granularity, domainkey, startTime, endTime, interval);
   const chunks = getUrlChunks(urls, CHUNK_SIZE);
 
   let totalTransferSize = 0;
@@ -311,6 +317,88 @@ async function fetchBundles(opts, log) {
   return mergeBundlesWithSameId(result);
 }
 
+function createBundleStream(opts, log) {
+  const {
+    domain,
+    domainkey,
+    interval = 7,
+    granularity = GRANULARITY.DAILY,
+    checkpoints = [],
+    filterBotTraffic = true,
+    startTime,
+    endTime,
+    handler,
+  } = opts;
+
+  if (!hasText(domain) || !hasText(domainkey)) {
+    throw new Error('Missing required parameters');
+  }
+
+  const urls = generateURLs(domain, granularity, domainkey, startTime, endTime, interval);
+
+  return new ReadableStream({
+    async start(controller) {
+      const failedUrls = [];
+      let totalTransferSize = 0;
+      let bundlesCount = 0;
+      let lastCheckpoint = 0;
+
+      async function streamBundle(url) {
+        const response = await fetch(url);
+        totalTransferSize += parseInt(response.headers.get('content-length'), 10);
+
+        if (!response.ok) {
+          log.warn(`Failed to fetch URL: ${sanitizeURL(url)} - status: ${response.status}`);
+          failedUrls.push(url);
+          return;
+        }
+
+        const bundles = await response.json();
+
+        const filtered = bundles?.rumBundles?.filter(
+          (bundle) => !filterBotTraffic || !isBotTraffic(bundle),
+        ).map(filterEvents(checkpoints));
+
+        bundlesCount += filtered.length;
+        const currentCheckpoint = Math.floor(bundlesCount / 50000);
+
+        if (currentCheckpoint > lastCheckpoint) {
+          log.info(`Checkpoint: Fetched ${bundlesCount} bundles; resuming...`);
+          lastCheckpoint = currentCheckpoint;
+        }
+
+        const crunchedBundle = handler(filtered || []);
+        controller.enqueue(crunchedBundle);
+      }
+
+      async function worker() {
+        while (urls.length > 0) {
+          const url = urls.shift();
+          // eslint-disable-next-line no-await-in-loop
+          await streamBundle(url);
+        }
+      }
+
+      const workers = Array(CHUNK_SIZE)
+        .fill()
+        .map(() => worker());
+
+      await Promise.all(workers);
+
+      log.info(`Retrieved all RUM bundles. Total transfer size (in KB): ${(totalTransferSize / 1024).toFixed(2)}`);
+
+      // Add failedUrls to opts object for access by callers
+      if (failedUrls.length > 0) {
+        // eslint-disable-next-line no-param-reassign
+        opts.failedUrls = failedUrls;
+      }
+
+      controller.close();
+    },
+  });
+}
+
 export {
   fetchBundles,
+  createBundleStream,
 };

package/src/common/traffic.js

@@ -27,9 +27,8 @@ import URI from 'urijs';
 export function getSecondLevelDomain(url) {
   if (!hasText(url)) return url;
   const uri = new URI(url);
-  const domain = uri.domain();
   const tld = uri.tld();
-  return domain.split(`.${tld}`)[0];
+  return uri.hostname().split(`.${tld}`)[0];
 }
 
 /*
@@ -39,9 +38,10 @@ export function getSecondLevelDomain(url) {
 // Referrer related
 const referrers = {
   search: /google|yahoo|bing|yandex|baidu|duckduckgo|brave|ecosia|aol|startpage|ask/,
-  social: /^\b(x)\b|(.*(facebook|tiktok|snapchat|twitter|pinterest|reddit|linkedin|threads|quora|discord|tumblr|mastodon|bluesky|instagram).*)$/,
+  social: /^\b((www\.)?x)\b|(.*(facebook|tiktok|snapchat|twitter|pinterest|reddit|linkedin|threads|quora|discord|tumblr|mastodon|bluesky|instagram).*)$/,
   ad: /googlesyndication|2mdn|doubleclick|syndicatedsearch/,
   video: /youtube|vimeo|twitch|dailymotion|wistia/,
+  llm: /chatgpt|openai|perplexity|claude|gemini\.google|copilot\.microsoft/,
 };
 
 const mediums = {
@@ -66,6 +66,7 @@ const sources = {
   display: /optumib2b|jun|googleads|dv360|dv36|microsoft|flipboard|programmatic|yext|gdn|banner|newsshowcase/,
   affiliate: /brandreward|yieldkit|fashionistatop|partner|linkbux|stylesblog|linkinbio|affiliate/,
   email: /sfmc|email/,
+  llm: /chatgpt/,
 };
 
 /**
@@ -74,7 +75,7 @@ const sources = {
  * Using full word match for social media shorts like ig, fb, x
  */
 const vendorClassifications = [
-  { regex: /google|googleads|google-ads|google_search|google_deman|adwords|dv360|gdn|doubleclick|dbm|gmb/i, result: 'google' },
+  { regex: /google|googleads|google-ads|google_search|google_deman|adwords|dv360|gdn|doubleclick|dbm|gmb|gemini/i, result: 'google' },
   { regex: /instagram|\b(ig)\b/i, result: 'instagram' },
   { regex: /facebook|\b(fb)\b|meta/i, result: 'facebook' },
   { regex: /bing/i, result: 'bing' },
@@ -83,7 +84,7 @@ const vendorClassifications = [
   { regex: /linkedin/i, result: 'linkedin' },
   { regex: /twitter|^\b(x)\b/i, result: 'x' },
   { regex: /snapchat/i, result: 'snapchat' },
-  { regex: /microsoft/i, result: 'microsoft' },
+  { regex: /microsoft|copilot/i, result: 'microsoft' },
   { regex: /pinterest/i, result: 'pinterest' },
   { regex: /reddit/i, result: 'reddit' },
   { regex: /spotify/i, result: 'spotify' },
@@ -102,6 +103,9 @@ const vendorClassifications = [
   { regex: /amazon|ctv/i, result: 'amazon' },
   { regex: /dailymotion/i, result: 'dailymotion' },
   { regex: /twitch/i, result: 'twitch' },
+  { regex: /chatgpt|openai/i, result: 'openai' },
+  { regex: /perplexity/i, result: 'perplexity' },
+  { regex: /claude/i, result: 'claude' },
   { regex: /direct/i, result: 'direct' },
 ];
 
@@ -176,6 +180,8 @@ const RULES = (domain) => ([
   { type: 'paid', category: 'uncategorized', referrer: not(domain), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
 
   // EARNED
+  { type: 'earned', category: 'llm', referrer: anyOf(referrers.llm), utmSource: any, utmMedium: any, tracking: none },
+  { type: 'earned', category: 'llm', referrer: any, utmSource: anyOf(sources.llm), utmMedium: any, tracking: none },
   { type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: none, utmMedium: none, tracking: none },
   { type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: not(mediums.paidall), tracking: not(paidTrackingParams) },
   { type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: anyOf(mediums.organic), tracking: none },

package/src/index.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { hasText, fetch } from '@adobe/spacecat-shared-utils';
-import { fetchBundles } from './common/rum-bundler-client.js';
+import { fetchBundles, createBundleStream } from './common/rum-bundler-client.js';
 import notfound from './functions/404.js';
 import notfoundInternalLinks from './functions/404-internal-links.js';
 import cwv from './functions/cwv.js';
@@ -177,4 +177,21 @@ export default class RUMAPIClient {
       throw new Error(`Multi query failed. Queries: ${JSON.stringify(queries)}, Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
     }
   }
+
+  async queryStream(query, opts) {
+    const { handler, checkpoints } = HANDLERS[query] || {};
+    if (!handler) throw new Error(`Unknown query ${query}`);
+
+    try {
+      const domainkey = await this._getDomainkey(opts);
+      return createBundleStream({
+        ...opts,
+        domainkey,
+        checkpoints,
+        handler,
+      }, this.log);
+    } catch (e) {
+      throw new Error(`Query stream '${query}' failed. Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
+    }
+  }
 }