@beesolve/analytics-processor 0.4.0 → 0.5.0

package/dist/index.js

@@ -328,7 +328,7 @@ async function waitForQuery({
     delayMs = Math.min(delayMs * 2, maxDelayMs);
   }
 }
-async function fetchRows({
+async function fetchRawRows({
   athena,
   queryExecutionId
 }) {
@@ -340,25 +340,20 @@ async function fetchRows({
     const resultRows = results.ResultSet?.Rows ?? [];
     const dataRows = isFirstPage ? resultRows.slice(1) : resultRows;
     for (const dataRow of dataRows) {
-      const cells = (dataRow.Data ?? []).map((datum) => datum.VarCharValue);
-      rows.push(parseRow(cells));
+      rows.push((dataRow.Data ?? []).map((datum) => datum.VarCharValue));
     }
     nextToken = results.NextToken;
     isFirstPage = false;
   } while (nextToken != null);
   return rows;
 }
-async function runHourlyAthenaQuery({
+async function startAndFetchRows({
   athena,
   database,
-  table,
   workgroup,
   outputLocation,
-  source,
-  hourKey,
-  topPaths
+  queryString
 }) {
-  const queryString = buildHourlyQuery({ table, source, hourKey, topPaths });
   const started = await athena.send(new StartQueryExecutionCommand({
     QueryString: queryString,
     QueryExecutionContext: { Database: database },
@@ -370,7 +365,27 @@ async function runHourlyAthenaQuery({
     throw new Error("Athena StartQueryExecution returned no QueryExecutionId");
   }
   await waitForQuery({ athena, queryExecutionId });
-  const rows = await fetchRows({ athena, queryExecutionId });
+  return fetchRawRows({ athena, queryExecutionId });
+}
+async function runHourlyAthenaQuery({
+  athena,
+  database,
+  table,
+  workgroup,
+  outputLocation,
+  source,
+  hourKey,
+  topPaths
+}) {
+  const queryString = buildHourlyQuery({ table, source, hourKey, topPaths });
+  const rawRows = await startAndFetchRows({
+    athena,
+    database,
+    workgroup,
+    outputLocation,
+    queryString
+  });
+  const rows = rawRows.map((cells) => parseRow(cells));
   return assembleResult({ rows, topPaths });
 }
 // packages/analytics-processor/src/breakdowns.ts
@@ -489,7 +504,11 @@ var configSchema = v3.object({
   pk: v3.literal("system#config"),
   sk: v3.literal("analytics"),
   startKey: v3.string(),
-  startedAt: v3.string()
+  startedAt: v3.string(),
+  sitemapUrl: v3.optional(v3.string())
+});
+var sitemapConfigSchema = v3.looseObject({
+  sitemapUrl: v3.optional(v3.string())
 });
 var configKey = { pk: "system#config", sk: "analytics" };
 function isConditionalCheckFailure(error) {
@@ -510,6 +529,23 @@ class Config {
       return null;
     return v3.parse(configSchema, item);
   };
+  getSitemapConfig = async () => {
+    const { Item: item } = await this.props.dynamo.send(new GetCommand2({ TableName: this.props.tableName, Key: configKey }));
+    if (item == null)
+      return {};
+    return v3.parse(sitemapConfigSchema, item);
+  };
+  setSitemapUrl = async ({
+    sitemapUrl
+  }) => {
+    await this.props.dynamo.send(new UpdateCommand2({
+      TableName: this.props.tableName,
+      Key: configKey,
+      UpdateExpression: "SET #sitemapUrl = :url",
+      ExpressionAttributeNames: { "#sitemapUrl": "sitemapUrl" },
+      ExpressionAttributeValues: { ":url": sitemapUrl }
+    }));
+  };
   ensureStart = async (props) => {
     try {
       await this.props.dynamo.send(new UpdateCommand2({
@@ -1031,6 +1067,423 @@ function pad2(value) {
 function pad4(value) {
   return value.toString().padStart(4, "0");
 }
+// packages/analytics-processor/src/sitemap.ts
+import { UpdateCommand as UpdateCommand3 } from "@aws-sdk/lib-dynamodb";
+import { queryAll as queryAll2 } from "@beesolve/dynamo-helpers";
+import * as v4 from "valibot";
+function canonicalPath(path) {
+  if (path.length > 1 && path.endsWith("/")) {
+    return path.slice(0, -1);
+  }
+  return path;
+}
+function normalizeSitemapLoc(loc) {
+  let parsed;
+  try {
+    parsed = new URL(loc);
+  } catch {
+    throw new Error(`Unparseable sitemap <loc>: ${loc}`);
+  }
+  return canonicalPath(parsed.pathname);
+}
+function parseSitemap(xml) {
+  if (sitemapIndexPattern.test(xml)) {
+    throw new Error("Sitemap index documents are not supported");
+  }
+  const paths = [];
+  const seen = new Set;
+  for (const match of xml.matchAll(locPattern)) {
+    const raw = match[1];
+    if (raw == null)
+      continue;
+    const loc = unescapeXml(raw.trim());
+    if (loc === "")
+      continue;
+    const path = normalizeSitemapLoc(loc);
+    if (seen.has(path))
+      continue;
+    seen.add(path);
+    paths.push(path);
+  }
+  return paths;
+}
+var sitemapUserAgent = "@beesolve/analytics-processor sitemap fetcher (+https://github.com/BeeSolve/packages)";
+async function fetchSitemapPaths(url) {
+  const res = await fetch(url, { headers: { "user-agent": sitemapUserAgent } });
+  if (!res.ok) {
+    throw new Error(`Failed to fetch sitemap ${url}: ${res.status} ${res.statusText}`);
+  }
+  const text = await res.text();
+  return parseSitemap(text);
+}
+function hourKeyParts2(hourKey) {
+  const match = /^(\d{4})-(\d{2})-(\d{2})-(\d{2})$/.exec(hourKey);
+  if (match == null) {
+    throw new Error(`Invalid hourKey: ${hourKey}`);
+  }
+  const [, year, month, day, hour] = match;
+  if (year == null || month == null || day == null || hour == null) {
+    throw new Error(`Invalid hourKey: ${hourKey}`);
+  }
+  return { year, month, day, hour, monthKey: `${year}-${month}`, dayAttr: `d${day}` };
+}
+function buildSitemapHourlyQuery({
+  table,
+  source,
+  hourKey,
+  paths
+}) {
+  const parts = hourKeyParts2(hourKey);
+  const columns = columnMaps[source];
+  const canonicalUri = canonicalUriExpr(columns.uriStem);
+  const hourPredicate = source === "v2" ? `year = '${parts.year}' AND month = '${parts.month}' AND day = '${parts.day}' AND hour = '${parts.hour}'` : `"date" = DATE '${parts.year}-${parts.month}-${parts.day}' AND substr(${columns.timestamp}, 1, 2) = '${parts.hour}'`;
+  const whereClause = paths.length === 0 ? "1 = 0" : `${hourPredicate} AND ${canonicalUri} IN (${paths.map((path) => `'${escapeSqlString(path)}'`).join(", ")})`;
+  return [
+    "SELECT",
+    `  ${canonicalUri} AS path_value,`,
+    "  count(*) AS hits",
+    `FROM "${table}"`,
+    `WHERE ${whereClause}`,
+    `GROUP BY ${canonicalUri}`
+  ].join(`
+`);
+}
+async function runSitemapHourlyQuery({
+  athena,
+  database,
+  table,
+  workgroup,
+  outputLocation,
+  source,
+  hourKey,
+  paths
+}) {
+  if (paths.length === 0) {
+    return [];
+  }
+  const queryString = buildSitemapHourlyQuery({ table, source, hourKey, paths });
+  const rawRows = await startAndFetchRows({
+    athena,
+    database,
+    workgroup,
+    outputLocation,
+    queryString
+  });
+  const byPath = {};
+  for (const cells of rawRows) {
+    const path = cells[0];
+    const hits = toCount(cells[1]);
+    if (path == null || hits === 0)
+      continue;
+    byPath[path] = (byPath[path] ?? 0) + hits;
+  }
+  return Object.entries(byPath).map(([path, hits]) => ({ path, hits }));
+}
+function buildSitemapMonthlyQuery({
+  table,
+  source,
+  monthKey,
+  paths
+}) {
+  const match = /^(\d{4})-(\d{2})$/.exec(monthKey);
+  if (match == null) {
+    throw new Error(`Invalid monthKey: ${monthKey}`);
+  }
+  const [, year, month] = match;
+  if (year == null || month == null) {
+    throw new Error(`Invalid monthKey: ${monthKey}`);
+  }
+  const columns = columnMaps[source];
+  const canonicalUri = canonicalUriExpr(columns.uriStem);
+  const dayExpr = source === "v2" ? "day" : `day_of_month(CAST("date" AS date))`;
+  const monthPredicate = source === "v2" ? `year = '${year}' AND month = '${month}'` : `"date" BETWEEN DATE '${year}-${month}-01' AND DATE '${year}-${month}-31'`;
+  const whereClause = paths.length === 0 ? "1 = 0" : `${monthPredicate} AND ${canonicalUri} IN (${paths.map((path) => `'${escapeSqlString(path)}'`).join(", ")})`;
+  return [
+    "SELECT",
+    `  ${dayExpr} AS day_value,`,
+    `  ${canonicalUri} AS path_value,`,
+    "  count(*) AS hits",
+    `FROM "${table}"`,
+    `WHERE ${whereClause}`,
+    `GROUP BY ${dayExpr}, ${canonicalUri}`
+  ].join(`
+`);
+}
+async function runSitemapMonthlyQuery({
+  athena,
+  database,
+  table,
+  workgroup,
+  outputLocation,
+  source,
+  monthKey,
+  paths
+}) {
+  if (paths.length === 0) {
+    return [];
+  }
+  const queryString = buildSitemapMonthlyQuery({ table, source, monthKey, paths });
+  const rawRows = await startAndFetchRows({
+    athena,
+    database,
+    workgroup,
+    outputLocation,
+    queryString
+  });
+  const byPath = {};
+  for (const cells of rawRows) {
+    const dayCell = cells[0];
+    const path = cells[1];
+    const hits = toCount(cells[2]);
+    if (path == null || dayCell == null || hits === 0)
+      continue;
+    const dayKey = String(dayCell).padStart(2, "0");
+    const existing = byPath[path] ?? { days: {}, hits: 0 };
+    existing.days[dayKey] = (existing.days[dayKey] ?? 0) + hits;
+    existing.hits += hits;
+    byPath[path] = existing;
+  }
+  return Object.entries(byPath).map(([path, entry]) => ({
+    path,
+    month: monthKey,
+    days: entry.days,
+    hits: entry.hits
+  }));
+}
+async function processSitemapHour({
+  hourKey,
+  source,
+  paths,
+  models,
+  athena,
+  options,
+  dynamo
+}) {
+  const alreadyApplied = await models.markers.isApplied({ grain: "sitemap", periodKey: hourKey });
+  if (alreadyApplied)
+    return { status: "skipped", urls: 0 };
+  const parts = hourKeyParts2(hourKey);
+  const counts = paths.length === 0 ? [] : await runSitemapHourlyQuery({
+    athena: athena.client,
+    database: athena.database,
+    table: athena.table,
+    workgroup: athena.workgroup,
+    outputLocation: athena.outputLocation,
+    source,
+    hourKey,
+    paths
+  });
+  const ttl = computeMarkerTtl(options?.markerTtlDays);
+  const markerPut = models.markers.buildMarkerPut({
+    grain: "sitemap",
+    periodKey: hourKey,
+    source,
+    ttl
+  });
+  const addUpdates = counts.map((count) => models.sitemapUrlStats.buildAdd({
+    path: count.path,
+    month: parts.monthKey,
+    dayAttr: parts.dayAttr,
+    hits: count.hits
+  }));
+  const result = await sendTransacted({ dynamo, leading: [markerPut], rest: addUpdates });
+  return { status: result.status, urls: result.status === "applied" ? counts.length : 0 };
+}
+async function processSitemapMonth({
+  monthKey,
+  source,
+  paths,
+  models,
+  athena
+}) {
+  if (paths.length === 0) {
+    return { urls: 0 };
+  }
+  const counts = await runSitemapMonthlyQuery({
+    athena: athena.client,
+    database: athena.database,
+    table: athena.table,
+    workgroup: athena.workgroup,
+    outputLocation: athena.outputLocation,
+    source,
+    monthKey,
+    paths
+  });
+  await models.sitemapUrlStats.overwriteMany(counts);
+  return { urls: counts.length };
+}
+var sitemapUrlStatSchema = v4.looseObject({
+  pk: v4.string(),
+  sk: v4.string(),
+  path: v4.string(),
+  month: v4.string(),
+  hits: v4.number(),
+  updatedAt: v4.number()
+});
+
+class SitemapUrlStats {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  buildAdd = ({
+    path,
+    month,
+    dayAttr,
+    hits
+  }) => {
+    const update = {
+      TableName: this.props.tableName,
+      Key: { pk: partitionKey3(path), sk: month },
+      UpdateExpression: "SET #path = :path, #month = :month, #updatedAt = :now ADD #hits :hits, #day :hits",
+      ExpressionAttributeNames: {
+        "#path": "path",
+        "#month": "month",
+        "#updatedAt": "updatedAt",
+        "#hits": "hits",
+        "#day": dayAttr
+      },
+      ExpressionAttributeValues: {
+        ":path": path,
+        ":month": month,
+        ":now": Date.now(),
+        ":hits": hits
+      }
+    };
+    return { Update: update };
+  };
+  overwriteMany = async (stats) => {
+    for (const stat of stats) {
+      await this.props.dynamo.send(new UpdateCommand3(this.buildOverwrite(stat)));
+    }
+  };
+  buildOverwrite = ({
+    path,
+    month,
+    days,
+    hits
+  }) => {
+    const names = {
+      "#path": "path",
+      "#month": "month",
+      "#hits": "hits",
+      "#updatedAt": "updatedAt"
+    };
+    const valuesMap = {
+      ":path": path,
+      ":month": month,
+      ":hits": hits,
+      ":now": Date.now()
+    };
+    const setParts = ["#path = :path", "#month = :month", "#hits = :hits", "#updatedAt = :now"];
+    for (const [dayKey, count] of Object.entries(days)) {
+      const dayAttr = `d${dayKey}`;
+      names[`#${dayAttr}`] = dayAttr;
+      valuesMap[`:${dayAttr}`] = count;
+      setParts.push(`#${dayAttr} = :${dayAttr}`);
+    }
+    return {
+      TableName: this.props.tableName,
+      Key: { pk: partitionKey3(path), sk: month },
+      UpdateExpression: `SET ${setParts.join(", ")}`,
+      ExpressionAttributeNames: names,
+      ExpressionAttributeValues: valuesMap
+    };
+  };
+  getForUrl = async ({
+    path,
+    startMonth,
+    endMonth
+  }) => {
+    const items = await queryAll2({
+      dynamo: this.props.dynamo,
+      input: {
+        TableName: this.props.tableName,
+        KeyConditionExpression: "#pk = :pk AND #sk BETWEEN :start AND :end",
+        ExpressionAttributeNames: { "#pk": "pk", "#sk": "sk" },
+        ExpressionAttributeValues: {
+          ":pk": partitionKey3(path),
+          ":start": startMonth,
+          ":end": endMonth
+        }
+      }
+    });
+    return items.map((item) => this.parseOne(item));
+  };
+  getForMonth = async ({
+    month
+  }) => {
+    if (this.props.reverseIndexName == null) {
+      throw new Error("getForMonth requires a reverseIndexName");
+    }
+    const items = await queryAll2({
+      dynamo: this.props.dynamo,
+      input: {
+        TableName: this.props.tableName,
+        IndexName: this.props.reverseIndexName,
+        KeyConditionExpression: "#sk = :month",
+        ExpressionAttributeNames: { "#sk": "sk" },
+        ExpressionAttributeValues: { ":month": month }
+      }
+    });
+    return items.map((item) => this.parseOne(item));
+  };
+  parseOne = (item) => {
+    const result = v4.safeParse(sitemapUrlStatSchema, item);
+    if (!result.success) {
+      console.error(v4.flatten(result.issues));
+      throw new Error("Malformed stored sitemap url stat record");
+    }
+    const days = {};
+    for (const [key, value] of Object.entries(result.output)) {
+      const dayMatch = /^d(\d{2})$/.exec(key);
+      if (dayMatch?.[1] != null && typeof value === "number") {
+        days[dayMatch[1]] = value;
+      }
+    }
+    return {
+      pk: result.output.pk,
+      sk: result.output.sk,
+      path: result.output.path,
+      month: result.output.month,
+      hits: result.output.hits,
+      updatedAt: result.output.updatedAt,
+      days
+    };
+  };
+}
+function partitionKey3(path) {
+  return `sitemapurl#${path}`;
+}
+var locPattern = /<loc>([\s\S]*?)<\/loc>/gi;
+var sitemapIndexPattern = /<sitemapindex/i;
+var xmlEntities = {
+  "&amp;": "&",
+  "&lt;": "<",
+  "&gt;": ">",
+  "&quot;": '"',
+  "&#39;": "'"
+};
+function unescapeXml(value) {
+  return value.replace(/&amp;|&lt;|&gt;|&quot;|&#39;/g, (entity) => xmlEntities[entity] ?? entity);
+}
+function canonicalUriExpr(uriColumn) {
+  return `CASE WHEN length(${uriColumn}) > 1 AND substr(${uriColumn}, -1) = '/' THEN substr(${uriColumn}, 1, length(${uriColumn}) - 1) ELSE ${uriColumn} END`;
+}
+function escapeSqlString(value) {
+  return value.replace(/'/g, "''");
+}
+function toCount(value) {
+  if (value == null || value === "")
+    return 0;
+  const parsed = Number(value);
+  return Number.isNaN(parsed) ? 0 : parsed;
+}
+var defaultMarkerTtlDays = 90;
+function computeMarkerTtl(markerTtlDays) {
+  const days = markerTtlDays ?? defaultMarkerTtlDays;
+  return Math.floor(Date.now() / 1000) + days * 24 * 60 * 60;
+}
 // packages/analytics-processor/src/time.ts
 function pad22(value) {
   return value.toString().padStart(2, "0");
@@ -1116,9 +1569,9 @@ function hoursBetween({ from, to }) {
   return keys;
 }
 // packages/analytics-processor/src/totals.ts
-import { GetCommand as GetCommand3, UpdateCommand as UpdateCommand3 } from "@aws-sdk/lib-dynamodb";
-import { queryAll as queryAll2 } from "@beesolve/dynamo-helpers";
-import * as v4 from "valibot";
+import { GetCommand as GetCommand3, UpdateCommand as UpdateCommand4 } from "@aws-sdk/lib-dynamodb";
+import { queryAll as queryAll3 } from "@beesolve/dynamo-helpers";
+import * as v5 from "valibot";
 var additiveFields = [
   "hits",
   "success",
@@ -1134,23 +1587,23 @@ var hourlyDetailFields = [
   "cacheHitRatio",
   "uniqueIpCount"
 ];
-var totalsSchema = v4.object({
-  pk: v4.string(),
-  sk: v4.string(),
-  grain: v4.picklist(grains),
-  timeKey: v4.string(),
-  hits: v4.number(),
-  success: v4.number(),
-  errors: v4.number(),
-  errors4xx: v4.number(),
-  errors5xx: v4.number(),
-  bytesSent: v4.number(),
-  bytesReceived: v4.number(),
-  p50TimeTaken: v4.optional(v4.number()),
-  p95TimeTaken: v4.optional(v4.number()),
-  cacheHitRatio: v4.optional(v4.number()),
-  uniqueIpCount: v4.optional(v4.number()),
-  ttl: v4.optional(v4.number())
+var totalsSchema = v5.object({
+  pk: v5.string(),
+  sk: v5.string(),
+  grain: v5.picklist(grains),
+  timeKey: v5.string(),
+  hits: v5.number(),
+  success: v5.number(),
+  errors: v5.number(),
+  errors4xx: v5.number(),
+  errors5xx: v5.number(),
+  bytesSent: v5.number(),
+  bytesReceived: v5.number(),
+  p50TimeTaken: v5.optional(v5.number()),
+  p95TimeTaken: v5.optional(v5.number()),
+  cacheHitRatio: v5.optional(v5.number()),
+  uniqueIpCount: v5.optional(v5.number()),
+  ttl: v5.optional(v5.number())
 });
 
 class Totals {
@@ -1160,11 +1613,11 @@ class Totals {
   }
   addHourly = async (props) => {
     const update = this.buildHourlyUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand3(update.Update));
+    await this.props.dynamo.send(new UpdateCommand4(update.Update));
   };
   addAggregate = async (props) => {
     const update = this.buildAggregateUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand3(update.Update));
+    await this.props.dynamo.send(new UpdateCommand4(update.Update));
   };
   buildHourlyUpdate = (props) => ({
     Update: this.buildUpdateMembers({
@@ -1182,14 +1635,14 @@ class Totals {
     })
   });
   queryWindow = async (props) => {
-    const items = await queryAll2({
+    const items = await queryAll3({
       dynamo: this.props.dynamo,
       input: {
         TableName: this.props.tableName,
         KeyConditionExpression: "#pk = :pk AND #sk BETWEEN :start AND :end",
         ExpressionAttributeNames: { "#pk": "pk", "#sk": "sk" },
         ExpressionAttributeValues: {
-          ":pk": partitionKey3(props.grain),
+          ":pk": partitionKey4(props.grain),
           ":start": props.startKey,
           ":end": props.endKey
         }
@@ -1200,7 +1653,7 @@ class Totals {
   getOne = async (props) => {
     const { Item: item } = await this.props.dynamo.send(new GetCommand3({
       TableName: this.props.tableName,
-      Key: { pk: partitionKey3(props.grain), sk: props.timeKey }
+      Key: { pk: partitionKey4(props.grain), sk: props.timeKey }
     }));
     if (item == null)
       return;
@@ -1236,28 +1689,29 @@ class Totals {
     }
     return {
       TableName: this.props.tableName,
-      Key: { pk: partitionKey3(props.grain), sk: props.item.timeKey },
+      Key: { pk: partitionKey4(props.grain), sk: props.item.timeKey },
       UpdateExpression: `SET ${setParts.join(", ")} ADD ${addParts.join(", ")}`,
       ExpressionAttributeNames: names,
       ExpressionAttributeValues: values
     };
   };
   parseOne = (item) => {
-    const result = v4.safeParse(totalsSchema, item);
+    const result = v5.safeParse(totalsSchema, item);
     if (!result.success) {
-      console.error(v4.flatten(result.issues));
+      console.error(v5.flatten(result.issues));
       throw new Error("Malformed stored totals record");
     }
     return result.output;
   };
 }
-function partitionKey3(grain) {
+function partitionKey4(grain) {
   return `totals#${grain}`;
 }
 export {
   AppliedMarkers,
   Breakdowns,
   Config,
+  SitemapUrlStats,
   Totals,
   appliedMarkerSchema,
   appliedSources,
@@ -1266,6 +1720,9 @@ export {
   breakdownSchema,
   bucketStates,
   buildHourlyQuery,
+  buildSitemapHourlyQuery,
+  buildSitemapMonthlyQuery,
+  canonicalPath,
   chunkTransactItems,
   classifyBucketState,
   classifyDevice,
@@ -1273,20 +1730,32 @@ export {
   configSchema,
   dayKey,
   dimensions,
+  fetchSitemapPaths,
   grainForPeriodKey,
   grains,
   hourKey,
   hourKeyParts,
   hoursBetween,
+  isMarkerRaceError,
   maxWindowFor,
   monthKey,
+  normalizeSitemapLoc,
   parsePeriodKey,
+  parseSitemap,
   processHour,
+  processSitemapHour,
+  processSitemapMonth,
   requiredRecordFields,
   resolveBreakdownLive,
   resolveTotalsLive,
   runHourlyAthenaQuery,
+  runSitemapHourlyQuery,
+  runSitemapMonthlyQuery,
+  sendTransacted,
   shiftAnchor,
+  sitemapUrlStatSchema,
+  sitemapUserAgent,
+  startAndFetchRows,
   sumBreakdown,
   sumTotals,
   topNWithOther,