@beesolve/analytics-processor 0.0.0-stage → 0.2.0

package/dist/index.js

@@ -0,0 +1,1011 @@
+// packages/analytics-processor/index.ts
+export * from "@beesolve/ip-info";
+
+// packages/analytics-processor/src/appliedMarkers.ts
+import { GetCommand } from "@aws-sdk/lib-dynamodb";
+import * as v from "valibot";
+
+// packages/analytics-processor/src/keys.ts
+var grains = ["hour", "day", "month"];
+var dimensions = ["status", "country", "device", "path"];
+var appliedSources = ["v1", "v2"];
+
+// packages/analytics-processor/src/appliedMarkers.ts
+var appliedMarkerSchema = v.object({
+  pk: v.string(),
+  sk: v.string(),
+  source: v.picklist(appliedSources),
+  appliedAt: v.number(),
+  ttl: v.optional(v.number())
+});
+
+class AppliedMarkers {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  isApplied = async (props) => {
+    const { Item: item } = await this.props.dynamo.send(new GetCommand({
+      TableName: this.props.tableName,
+      Key: { pk: partitionKey(props), sk: "applied" }
+    }));
+    return item != null;
+  };
+  buildMarkerPut = (props) => {
+    const item = {
+      pk: partitionKey(props),
+      sk: "applied",
+      source: props.source,
+      appliedAt: Date.now()
+    };
+    if (props.ttl != null) {
+      item.ttl = props.ttl;
+    }
+    return {
+      Put: {
+        TableName: this.props.tableName,
+        Item: item,
+        ConditionExpression: "attribute_not_exists(pk) AND attribute_not_exists(sk)"
+      }
+    };
+  };
+}
+function partitionKey(props) {
+  return `applied#${props.grain}#${props.periodKey}`;
+}
+// packages/analytics-processor/src/athena.ts
+import {
+  GetQueryExecutionCommand,
+  GetQueryResultsCommand,
+  StartQueryExecutionCommand
+} from "@aws-sdk/client-athena";
+
+// packages/analytics-processor/src/reduce.ts
+function sumTotals(items) {
+  const summed = {
+    hits: 0,
+    success: 0,
+    errors: 0,
+    errors4xx: 0,
+    errors5xx: 0,
+    bytesSent: 0,
+    bytesReceived: 0
+  };
+  return items.reduce((accumulator, item) => ({
+    hits: accumulator.hits + item.hits,
+    success: accumulator.success + item.success,
+    errors: accumulator.errors + item.errors,
+    errors4xx: accumulator.errors4xx + item.errors4xx,
+    errors5xx: accumulator.errors5xx + item.errors5xx,
+    bytesSent: accumulator.bytesSent + item.bytesSent,
+    bytesReceived: accumulator.bytesReceived + item.bytesReceived
+  }), summed);
+}
+function sumBreakdown(items) {
+  const byValue = {};
+  for (const item of items) {
+    const existing = byValue[item.value];
+    if (existing == null) {
+      byValue[item.value] = { value: item.value, hits: item.hits, bytes: item.bytes };
+    } else {
+      byValue[item.value] = {
+        value: item.value,
+        hits: existing.hits + item.hits,
+        bytes: existing.bytes + item.bytes
+      };
+    }
+  }
+  return Object.values(byValue);
+}
+function topNWithOther({
+  items,
+  n,
+  otherValue
+}) {
+  const sorted = [...items].sort((left, right) => right.hits - left.hits);
+  const top = sorted.slice(0, n);
+  const remainder = sorted.slice(n);
+  if (remainder.length === 0)
+    return top;
+  const other = remainder.reduce((accumulator, item) => ({
+    value: otherValue,
+    hits: accumulator.hits + item.hits,
+    bytes: accumulator.bytes + item.bytes
+  }), { value: otherValue, hits: 0, bytes: 0 });
+  return [...top, other];
+}
+
+// packages/analytics-processor/src/athena.ts
+var requiredRecordFields = [
+  "timestamp",
+  "sc_status",
+  "sc_bytes",
+  "cs_bytes",
+  "time_taken",
+  "x_edge_result_type",
+  "x_edge_detailed_result_type",
+  "cs_uri_stem",
+  "c_ip",
+  "cs_user_agent"
+];
+var columnMaps = {
+  v2: {
+    status: "sc_status",
+    bytesSent: "sc_bytes",
+    bytesReceived: "cs_bytes",
+    timeTaken: "time_taken",
+    resultType: "x_edge_result_type",
+    detailedResultType: "x_edge_detailed_result_type",
+    uriStem: "cs_uri_stem",
+    clientIp: "c_ip",
+    userAgent: "cs_user_agent",
+    timestamp: "timestamp"
+  },
+  v1: {
+    status: 'CAST("sc-status" AS integer)',
+    bytesSent: 'CAST("sc-bytes" AS bigint)',
+    bytesReceived: 'CAST("cs-bytes" AS bigint)',
+    timeTaken: 'CAST("time-taken" AS double)',
+    resultType: '"x-edge-result-type"',
+    detailedResultType: '"x-edge-detailed-result-type"',
+    uriStem: '"cs-uri-stem"',
+    clientIp: '"c-ip"',
+    userAgent: '"cs(User-Agent)"',
+    timestamp: '"time"'
+  }
+};
+function hourKeyParts(hourKey) {
+  const match = /^(\d{4})-(\d{2})-(\d{2})-(\d{2})$/.exec(hourKey);
+  if (match == null) {
+    throw new Error(`Invalid hourKey: ${hourKey}`);
+  }
+  const [, year, month, day, hour] = match;
+  if (year == null || month == null || day == null || hour == null) {
+    throw new Error(`Invalid hourKey: ${hourKey}`);
+  }
+  return { year, month, day, hour };
+}
+function deviceCaseExpression(userAgentColumn) {
+  return [
+    "CASE",
+    `WHEN lower(${userAgentColumn}) LIKE '%bot%' OR lower(${userAgentColumn}) LIKE '%crawler%' OR lower(${userAgentColumn}) LIKE '%spider%' OR lower(${userAgentColumn}) LIKE '%slurp%' THEN 'bot'`,
+    `WHEN lower(${userAgentColumn}) LIKE '%mobi%' OR lower(${userAgentColumn}) LIKE '%android%' OR lower(${userAgentColumn}) LIKE '%iphone%' OR lower(${userAgentColumn}) LIKE '%ipad%' THEN 'mobile'`,
+    `WHEN lower(${userAgentColumn}) LIKE '%windows nt%' OR lower(${userAgentColumn}) LIKE '%macintosh%' OR lower(${userAgentColumn}) LIKE '%x11%' OR lower(${userAgentColumn}) LIKE '%linux%' THEN 'desktop'`,
+    "ELSE 'other'",
+    "END"
+  ].join(" ");
+}
+function predicateFor({
+  source,
+  hourKey,
+  columns
+}) {
+  const parts = hourKeyParts(hourKey);
+  if (source === "v2") {
+    return [
+      `year = '${parts.year}'`,
+      `month = '${parts.month}'`,
+      `day = '${parts.day}'`,
+      `hour = '${parts.hour}'`
+    ].join(" AND ");
+  }
+  return [
+    `"date" = DATE '${parts.year}-${parts.month}-${parts.day}'`,
+    `substr(${columns.timestamp}, 1, 2) = '${parts.hour}'`
+  ].join(" AND ");
+}
+function buildHourlyQuery({
+  table,
+  source,
+  hourKey,
+  topPaths: _topPaths
+}) {
+  const columns = columnMaps[source];
+  const predicate = predicateFor({ source, hourKey, columns });
+  const device = deviceCaseExpression(columns.userAgent);
+  const groupingSets = [
+    "()",
+    `(${columns.status})`,
+    "(device)",
+    `(${columns.uriStem})`,
+    `(${columns.clientIp})`
+  ].join(", ");
+  return [
+    "SELECT",
+    `  ${columns.status} AS status_value,`,
+    `  ${device} AS device,`,
+    `  ${columns.uriStem} AS path_value,`,
+    `  ${columns.clientIp} AS ip_value,`,
+    "  count(*) AS hits,",
+    `  sum(CASE WHEN ${columns.status} BETWEEN 200 AND 399 THEN 1 ELSE 0 END) AS success,`,
+    `  sum(CASE WHEN ${columns.status} BETWEEN 400 AND 599 THEN 1 ELSE 0 END) AS errors,`,
+    `  sum(CASE WHEN ${columns.status} BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS errors4xx,`,
+    `  sum(CASE WHEN ${columns.status} BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS errors5xx,`,
+    `  sum(${columns.bytesSent}) AS bytes_sent,`,
+    `  sum(${columns.bytesReceived}) AS bytes_received,`,
+    `  approx_percentile(${columns.timeTaken}, 0.5) AS p50_time_taken,`,
+    `  approx_percentile(${columns.timeTaken}, 0.95) AS p95_time_taken,`,
+    `  sum(CASE WHEN ${columns.resultType} = 'Hit' THEN 1 ELSE 0 END) AS cache_hits,`,
+    `  count(DISTINCT ${columns.clientIp}) AS unique_ip_count`,
+    `FROM "${table}"`,
+    `WHERE ${predicate}`,
+    `GROUP BY GROUPING SETS (${groupingSets})`
+  ].join(`
+`);
+}
+function toNumber(value) {
+  if (value == null || value === "")
+    return 0;
+  const parsed = Number(value);
+  return Number.isNaN(parsed) ? 0 : parsed;
+}
+function cellValue(value) {
+  if (value == null || value === "")
+    return;
+  return value;
+}
+function parseRow(columns) {
+  return {
+    status: cellValue(columns[0]),
+    device: cellValue(columns[1]),
+    path: cellValue(columns[2]),
+    ip: cellValue(columns[3]),
+    hits: toNumber(columns[4]),
+    success: toNumber(columns[5]),
+    errors: toNumber(columns[6]),
+    errors4xx: toNumber(columns[7]),
+    errors5xx: toNumber(columns[8]),
+    bytesSent: toNumber(columns[9]),
+    bytesReceived: toNumber(columns[10]),
+    p50TimeTaken: toNumber(columns[11]),
+    p95TimeTaken: toNumber(columns[12]),
+    cacheHits: toNumber(columns[13]),
+    uniqueIpCount: toNumber(columns[14])
+  };
+}
+function assembleResult({
+  rows,
+  topPaths
+}) {
+  const totalsRow = rows.find((row) => row.status == null && row.device == null && row.path == null && row.ip == null);
+  const totals = totalsRow == null ? {
+    hits: 0,
+    success: 0,
+    errors: 0,
+    errors4xx: 0,
+    errors5xx: 0,
+    bytesSent: 0,
+    bytesReceived: 0,
+    p50TimeTaken: 0,
+    p95TimeTaken: 0,
+    cacheHitRatio: 0,
+    uniqueIpCount: 0
+  } : {
+    hits: totalsRow.hits,
+    success: totalsRow.success,
+    errors: totalsRow.errors,
+    errors4xx: totalsRow.errors4xx,
+    errors5xx: totalsRow.errors5xx,
+    bytesSent: totalsRow.bytesSent,
+    bytesReceived: totalsRow.bytesReceived,
+    p50TimeTaken: totalsRow.p50TimeTaken,
+    p95TimeTaken: totalsRow.p95TimeTaken,
+    cacheHitRatio: totalsRow.hits === 0 ? 0 : totalsRow.cacheHits / totalsRow.hits,
+    uniqueIpCount: totalsRow.uniqueIpCount
+  };
+  const byStatus = rows.filter((row) => row.status != null && row.device == null && row.path == null && row.ip == null).map((row) => ({ value: row.status ?? "", hits: row.hits, bytes: row.bytesSent }));
+  const byDevice = rows.filter((row) => row.device != null && row.status == null && row.path == null && row.ip == null).map((row) => ({ value: row.device ?? "", hits: row.hits, bytes: row.bytesSent }));
+  const byPathAll = rows.filter((row) => row.path != null && row.status == null && row.device == null && row.ip == null).map((row) => ({ value: row.path ?? "", hits: row.hits, bytes: row.bytesSent }));
+  const byPath = topNWithOther({ items: byPathAll, n: topPaths, otherValue: "__other__" });
+  const topIps = rows.filter((row) => row.ip != null && row.status == null && row.device == null && row.path == null).map((row) => ({ ip: row.ip ?? "", hits: row.hits })).sort((left, right) => right.hits - left.hits).slice(0, topPaths);
+  return { totals, byStatus, byDevice, byPath, topIps };
+}
+async function waitForQuery({
+  athena,
+  queryExecutionId
+}) {
+  let delayMs = 250;
+  const maxDelayMs = 5000;
+  while (true) {
+    const execution = await athena.send(new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId }));
+    const state = execution.QueryExecution?.Status?.State;
+    if (state === "SUCCEEDED")
+      return;
+    if (state === "FAILED" || state === "CANCELLED") {
+      const reason = execution.QueryExecution?.Status?.StateChangeReason ?? "unknown reason";
+      throw new Error(`Athena query ${state}: ${reason}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, delayMs));
+    delayMs = Math.min(delayMs * 2, maxDelayMs);
+  }
+}
+async function fetchRows({
+  athena,
+  queryExecutionId
+}) {
+  const rows = [];
+  let nextToken;
+  let isFirstPage = true;
+  do {
+    const results = await athena.send(new GetQueryResultsCommand({ QueryExecutionId: queryExecutionId, NextToken: nextToken }));
+    const resultRows = results.ResultSet?.Rows ?? [];
+    const dataRows = isFirstPage ? resultRows.slice(1) : resultRows;
+    for (const dataRow of dataRows) {
+      const cells = (dataRow.Data ?? []).map((datum) => datum.VarCharValue);
+      rows.push(parseRow(cells));
+    }
+    nextToken = results.NextToken;
+    isFirstPage = false;
+  } while (nextToken != null);
+  return rows;
+}
+async function runHourlyAthenaQuery({
+  athena,
+  database,
+  table,
+  workgroup,
+  outputLocation,
+  source,
+  hourKey,
+  topPaths
+}) {
+  const queryString = buildHourlyQuery({ table, source, hourKey, topPaths });
+  const started = await athena.send(new StartQueryExecutionCommand({
+    QueryString: queryString,
+    QueryExecutionContext: { Database: database },
+    WorkGroup: workgroup,
+    ResultConfiguration: outputLocation == null ? undefined : { OutputLocation: outputLocation }
+  }));
+  const queryExecutionId = started.QueryExecutionId;
+  if (queryExecutionId == null) {
+    throw new Error("Athena StartQueryExecution returned no QueryExecutionId");
+  }
+  await waitForQuery({ athena, queryExecutionId });
+  const rows = await fetchRows({ athena, queryExecutionId });
+  return assembleResult({ rows, topPaths });
+}
+// packages/analytics-processor/src/breakdowns.ts
+import { UpdateCommand } from "@aws-sdk/lib-dynamodb";
+import { queryAll } from "@beesolve/dynamo-helpers";
+import * as v2 from "valibot";
+var breakdownSchema = v2.object({
+  pk: v2.string(),
+  sk: v2.string(),
+  dimension: v2.picklist(dimensions),
+  grain: v2.picklist(grains),
+  periodKey: v2.string(),
+  value: v2.string(),
+  hits: v2.number(),
+  bytes: v2.number(),
+  ttl: v2.optional(v2.number())
+});
+
+class Breakdowns {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  addMany = async (props) => {
+    const updates = this.buildUpdates(props);
+    for (const update of updates) {
+      await this.props.dynamo.send(new UpdateCommand(update.Update));
+    }
+  };
+  buildUpdates = (props) => props.items.map((item) => ({
+    Update: this.buildUpdateMembers({
+      dimension: props.dimension,
+      grain: props.grain,
+      periodKey: props.periodKey,
+      item,
+      ttl: props.ttl
+    })
+  }));
+  getForPeriod = async (props) => {
+    const items = await queryAll({
+      dynamo: this.props.dynamo,
+      input: {
+        TableName: this.props.tableName,
+        KeyConditionExpression: "#pk = :pk",
+        ExpressionAttributeNames: { "#pk": "pk" },
+        ExpressionAttributeValues: {
+          ":pk": partitionKey2({
+            dimension: props.dimension,
+            grain: props.grain,
+            periodKey: props.periodKey
+          })
+        }
+      }
+    });
+    return items.map((item) => this.parseOne(item));
+  };
+  buildUpdateMembers = (props) => {
+    const names = {
+      "#dimension": "dimension",
+      "#grain": "grain",
+      "#periodKey": "periodKey",
+      "#value": "value",
+      "#hits": "hits",
+      "#bytes": "bytes"
+    };
+    const values = {
+      ":dimension": props.dimension,
+      ":grain": props.grain,
+      ":periodKey": props.periodKey,
+      ":value": props.item.value,
+      ":hits": props.item.hits,
+      ":bytes": props.item.bytes
+    };
+    const setParts = [
+      "#dimension = :dimension",
+      "#grain = :grain",
+      "#periodKey = :periodKey",
+      "#value = :value"
+    ];
+    if (props.ttl != null) {
+      names["#ttl"] = "ttl";
+      values[":ttl"] = props.ttl;
+      setParts.push("#ttl = :ttl");
+    }
+    return {
+      TableName: this.props.tableName,
+      Key: {
+        pk: partitionKey2({
+          dimension: props.dimension,
+          grain: props.grain,
+          periodKey: props.periodKey
+        }),
+        sk: props.item.value
+      },
+      UpdateExpression: `SET ${setParts.join(", ")} ADD #hits :hits, #bytes :bytes`,
+      ExpressionAttributeNames: names,
+      ExpressionAttributeValues: values
+    };
+  };
+  parseOne = (item) => {
+    const result = v2.safeParse(breakdownSchema, item);
+    if (!result.success) {
+      console.error(v2.flatten(result.issues));
+      throw new Error("Malformed stored breakdown record");
+    }
+    return result.output;
+  };
+}
+function partitionKey2(props) {
+  return `breakdown#${props.dimension}#${props.grain}#${props.periodKey}`;
+}
+// packages/analytics-processor/src/device.ts
+var botPattern = /bot|crawler|spider|crawl|slurp|mediapartners|facebookexternalhit|bingpreview|duckduckbot|baiduspider|yandex|semrush|ahrefs|headlesschrome|python-requests|curl|wget|monitor|pingdom|uptime/i;
+var mobilePattern = /mobi|android|iphone|ipad|ipod|iemobile|blackberry|opera mini|windows phone|webos/i;
+var desktopPattern = /windows nt|macintosh|mac os x|x11|cros|linux/i;
+function classifyDevice(userAgent) {
+  if (userAgent === "")
+    return "other";
+  if (botPattern.test(userAgent))
+    return "bot";
+  if (mobilePattern.test(userAgent))
+    return "mobile";
+  if (desktopPattern.test(userAgent))
+    return "desktop";
+  return "other";
+}
+// packages/analytics-processor/src/processHour.ts
+import { TransactWriteCommand } from "@aws-sdk/lib-dynamodb";
+import { splitArrayToChunks } from "@beesolve/helpers";
+var transactionLimit = 100;
+var unresolvedCountry = "ZZ";
+async function processHour({
+  hourKey,
+  source,
+  models,
+  athena,
+  options,
+  dynamo
+}) {
+  const alreadyApplied = await models.markers.isApplied({ grain: "hour", periodKey: hourKey });
+  if (alreadyApplied)
+    return { status: "skipped" };
+  const query = await runHourlyAthenaQuery({
+    athena: athena.client,
+    database: athena.database,
+    table: athena.table,
+    workgroup: athena.workgroup,
+    outputLocation: athena.outputLocation,
+    source,
+    hourKey,
+    topPaths: options.topPaths
+  });
+  const byCountry = await resolveCountryBreakdown({
+    topIps: query.topIps,
+    ipCache: models.ipCache,
+    hasApiKey: options.ipInfoApiKey != null
+  });
+  const ttl = computeHourlyTtl(options.hourlyTtlDays);
+  const markerPut = models.markers.buildMarkerPut({
+    grain: "hour",
+    periodKey: hourKey,
+    source,
+    ttl
+  });
+  const totalsUpdate = models.totals.buildHourlyUpdate({
+    item: {
+      timeKey: hourKey,
+      hits: query.totals.hits,
+      success: query.totals.success,
+      errors: query.totals.errors,
+      errors4xx: query.totals.errors4xx,
+      errors5xx: query.totals.errors5xx,
+      bytesSent: query.totals.bytesSent,
+      bytesReceived: query.totals.bytesReceived,
+      p50TimeTaken: query.totals.p50TimeTaken,
+      p95TimeTaken: query.totals.p95TimeTaken,
+      cacheHitRatio: query.totals.cacheHitRatio,
+      uniqueIpCount: query.totals.uniqueIpCount
+    },
+    ttl
+  });
+  const breakdownUpdates = buildBreakdownUpdates({
+    breakdowns: models.breakdowns,
+    grain: "hour",
+    periodKey: hourKey,
+    ttl,
+    byStatus: query.byStatus,
+    byDevice: query.byDevice,
+    byPath: query.byPath,
+    byCountry
+  });
+  return sendTransacted({
+    dynamo,
+    leading: [markerPut, totalsUpdate],
+    rest: breakdownUpdates
+  });
+}
+async function applyDailyRollup({
+  dayKey,
+  models,
+  dynamo
+}) {
+  const alreadyApplied = await models.markers.isApplied({ grain: "day", periodKey: dayKey });
+  if (alreadyApplied)
+    return { status: "skipped" };
+  const hourTotals = await models.totals.queryWindow({
+    grain: "hour",
+    startKey: `${dayKey}-00`,
+    endKey: `${dayKey}-23`
+  });
+  const summed = sumTotals(hourTotals);
+  const byDimension = await aggregateFinerBreakdowns({
+    breakdowns: models.breakdowns,
+    grain: "hour",
+    periodKeys: hourWindowKeys(dayKey)
+  });
+  const totalsUpdate = models.totals.buildAggregateUpdate({
+    grain: "day",
+    item: { timeKey: dayKey, ...summed }
+  });
+  const breakdownUpdates = buildBreakdownUpdates({
+    breakdowns: models.breakdowns,
+    grain: "day",
+    periodKey: dayKey,
+    byStatus: byDimension.status,
+    byDevice: byDimension.device,
+    byPath: byDimension.path,
+    byCountry: byDimension.country
+  });
+  const markerPut = models.markers.buildMarkerPut({
+    grain: "day",
+    periodKey: dayKey,
+    source: "v2"
+  });
+  return sendTransacted({
+    dynamo,
+    leading: [markerPut, totalsUpdate],
+    rest: breakdownUpdates
+  });
+}
+async function applyMonthlyRollup({
+  monthKey,
+  models,
+  dynamo
+}) {
+  const alreadyApplied = await models.markers.isApplied({ grain: "month", periodKey: monthKey });
+  if (alreadyApplied)
+    return { status: "skipped" };
+  const dayTotals = await models.totals.queryWindow({
+    grain: "day",
+    startKey: `${monthKey}-01`,
+    endKey: `${monthKey}-31`
+  });
+  const summed = sumTotals(dayTotals);
+  const byDimension = await aggregateFinerBreakdowns({
+    breakdowns: models.breakdowns,
+    grain: "day",
+    periodKeys: dayTotals.map((item) => item.timeKey)
+  });
+  const totalsUpdate = models.totals.buildAggregateUpdate({
+    grain: "month",
+    item: { timeKey: monthKey, ...summed }
+  });
+  const breakdownUpdates = buildBreakdownUpdates({
+    breakdowns: models.breakdowns,
+    grain: "month",
+    periodKey: monthKey,
+    byStatus: byDimension.status,
+    byDevice: byDimension.device,
+    byPath: byDimension.path,
+    byCountry: byDimension.country
+  });
+  const markerPut = models.markers.buildMarkerPut({
+    grain: "month",
+    periodKey: monthKey,
+    source: "v2"
+  });
+  return sendTransacted({
+    dynamo,
+    leading: [markerPut, totalsUpdate],
+    rest: breakdownUpdates
+  });
+}
+function chunkTransactItems({
+  leading,
+  rest,
+  limit = transactionLimit
+}) {
+  const firstCapacity = Math.max(limit - leading.length, 0);
+  const firstChunk = [...leading, ...rest.slice(0, firstCapacity)];
+  const tail = splitArrayToChunks(rest.slice(firstCapacity), limit);
+  return [firstChunk, ...tail];
+}
+async function sendTransacted({
+  dynamo,
+  leading,
+  rest
+}) {
+  const chunks = chunkTransactItems({ leading, rest });
+  try {
+    for (const chunk of chunks) {
+      if (chunk.length === 0)
+        continue;
+      await dynamo.send(new TransactWriteCommand({ TransactItems: chunk }));
+    }
+  } catch (error) {
+    if (isMarkerRaceError(error))
+      return { status: "skipped" };
+    throw error;
+  }
+  return { status: "applied" };
+}
+function isMarkerRaceError(error) {
+  if (error == null || typeof error !== "object")
+    return false;
+  const name = "name" in error ? error.name : undefined;
+  return name === "TransactionCanceledException" || name === "ConditionalCheckFailedException";
+}
+function buildBreakdownUpdates({
+  breakdowns,
+  grain,
+  periodKey,
+  ttl,
+  byStatus,
+  byDevice,
+  byPath,
+  byCountry
+}) {
+  const dimensionRows = [
+    { dimension: "status", items: byStatus },
+    { dimension: "device", items: byDevice },
+    { dimension: "country", items: byCountry },
+    { dimension: "path", items: byPath }
+  ];
+  const updates = [];
+  for (const row of dimensionRows) {
+    if (row.items.length === 0)
+      continue;
+    updates.push(...breakdowns.buildUpdates({
+      dimension: row.dimension,
+      grain,
+      periodKey,
+      items: row.items,
+      ttl
+    }));
+  }
+  return updates;
+}
+async function resolveCountryBreakdown({
+  topIps,
+  ipCache,
+  hasApiKey
+}) {
+  if (topIps.length === 0)
+    return [];
+  const resolved = hasApiKey ? await ipCache.enrichMany({ ips: topIps.map((entry) => entry.ip) }) : {};
+  const hitsByCountry = {};
+  for (const entry of topIps) {
+    const info = resolved[entry.ip];
+    const country = info?.countryCode ?? unresolvedCountry;
+    hitsByCountry[country] = (hitsByCountry[country] ?? 0) + entry.hits;
+  }
+  return Object.entries(hitsByCountry).map(([country, hits]) => ({
+    value: country,
+    hits,
+    bytes: 0
+  }));
+}
+async function aggregateFinerBreakdowns({
+  breakdowns,
+  grain,
+  periodKeys
+}) {
+  const status = [];
+  const device = [];
+  const path = [];
+  const country = [];
+  for (const periodKey of periodKeys) {
+    const [statusItems, deviceItems, pathItems, countryItems] = await Promise.all([
+      breakdowns.getForPeriod({ dimension: "status", grain, periodKey }),
+      breakdowns.getForPeriod({ dimension: "device", grain, periodKey }),
+      breakdowns.getForPeriod({ dimension: "path", grain, periodKey }),
+      breakdowns.getForPeriod({ dimension: "country", grain, periodKey })
+    ]);
+    status.push(...statusItems.map(toBreakdownRow));
+    device.push(...deviceItems.map(toBreakdownRow));
+    path.push(...pathItems.map(toBreakdownRow));
+    country.push(...countryItems.map(toBreakdownRow));
+  }
+  return {
+    status: sumBreakdown(status),
+    device: sumBreakdown(device),
+    path: sumBreakdown(path),
+    country: sumBreakdown(country)
+  };
+}
+function hourWindowKeys(dayKey) {
+  const keys = [];
+  for (let hour = 0;hour < 24; hour += 1) {
+    keys.push(`${dayKey}-${hour.toString().padStart(2, "0")}`);
+  }
+  return keys;
+}
+function computeHourlyTtl(hourlyTtlDays) {
+  const nowSeconds = Math.floor(Date.now() / 1000);
+  return nowSeconds + hourlyTtlDays * 24 * 60 * 60;
+}
+function toBreakdownRow(item) {
+  return { value: item.value, hits: item.hits, bytes: item.bytes };
+}
+// packages/analytics-processor/src/time.ts
+function pad2(value) {
+  return value.toString().padStart(2, "0");
+}
+function pad4(value) {
+  return value.toString().padStart(4, "0");
+}
+function yearKey(date) {
+  return pad4(date.getUTCFullYear());
+}
+function monthKey(date) {
+  return `${pad4(date.getUTCFullYear())}-${pad2(date.getUTCMonth() + 1)}`;
+}
+function dayKey(date) {
+  return `${monthKey(date)}-${pad2(date.getUTCDate())}`;
+}
+function hourKey(date) {
+  return `${dayKey(date)}-${pad2(date.getUTCHours())}`;
+}
+function maxWindowFor(grain) {
+  if (grain === "hour")
+    return 24;
+  if (grain === "day")
+    return 30;
+  return 12;
+}
+function keyFor(grain, date) {
+  if (grain === "hour")
+    return hourKey(date);
+  if (grain === "day")
+    return dayKey(date);
+  return monthKey(date);
+}
+function shiftAnchor({
+  grain,
+  anchor,
+  size,
+  direction
+}) {
+  const delta = direction === "next" ? size : -size;
+  const shifted = new Date(anchor.getTime());
+  if (grain === "hour") {
+    shifted.setUTCHours(shifted.getUTCHours() + delta);
+  } else if (grain === "day") {
+    shifted.setUTCDate(shifted.getUTCDate() + delta);
+  } else {
+    shifted.setUTCMonth(shifted.getUTCMonth() + delta);
+  }
+  return shifted;
+}
+function windowKeys({ grain, anchor, size }) {
+  const endKey = keyFor(grain, anchor);
+  const start = shiftAnchor({ grain, anchor, size: size - 1, direction: "prev" });
+  const startKey = keyFor(grain, start);
+  return { startKey, endKey };
+}
+function hoursBetween({ from, to }) {
+  const keys = [];
+  const cursor = new Date(Date.UTC(from.getUTCFullYear(), from.getUTCMonth(), from.getUTCDate(), from.getUTCHours()));
+  const end = Date.UTC(to.getUTCFullYear(), to.getUTCMonth(), to.getUTCDate(), to.getUTCHours());
+  while (cursor.getTime() <= end) {
+    keys.push(hourKey(cursor));
+    cursor.setUTCHours(cursor.getUTCHours() + 1);
+  }
+  return keys;
+}
+// packages/analytics-processor/src/totals.ts
+import { GetCommand as GetCommand2, UpdateCommand as UpdateCommand2 } from "@aws-sdk/lib-dynamodb";
+import { queryAll as queryAll2 } from "@beesolve/dynamo-helpers";
+import * as v3 from "valibot";
+var additiveFields = [
+  "hits",
+  "success",
+  "errors",
+  "errors4xx",
+  "errors5xx",
+  "bytesSent",
+  "bytesReceived"
+];
+var hourlyDetailFields = [
+  "p50TimeTaken",
+  "p95TimeTaken",
+  "cacheHitRatio",
+  "uniqueIpCount"
+];
+var totalsSchema = v3.object({
+  pk: v3.string(),
+  sk: v3.string(),
+  grain: v3.picklist(grains),
+  timeKey: v3.string(),
+  hits: v3.number(),
+  success: v3.number(),
+  errors: v3.number(),
+  errors4xx: v3.number(),
+  errors5xx: v3.number(),
+  bytesSent: v3.number(),
+  bytesReceived: v3.number(),
+  p50TimeTaken: v3.optional(v3.number()),
+  p95TimeTaken: v3.optional(v3.number()),
+  cacheHitRatio: v3.optional(v3.number()),
+  uniqueIpCount: v3.optional(v3.number()),
+  ttl: v3.optional(v3.number())
+});
+
+class Totals {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  addHourly = async (props) => {
+    const update = this.buildHourlyUpdate(props);
+    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+  };
+  addAggregate = async (props) => {
+    const update = this.buildAggregateUpdate(props);
+    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+  };
+  buildHourlyUpdate = (props) => ({
+    Update: this.buildUpdateMembers({
+      grain: "hour",
+      item: props.item,
+      ttl: props.ttl,
+      includeHourlyDetail: true
+    })
+  });
+  buildAggregateUpdate = (props) => ({
+    Update: this.buildUpdateMembers({
+      grain: props.grain,
+      item: props.item,
+      includeHourlyDetail: false
+    })
+  });
+  queryWindow = async (props) => {
+    const items = await queryAll2({
+      dynamo: this.props.dynamo,
+      input: {
+        TableName: this.props.tableName,
+        KeyConditionExpression: "#pk = :pk AND #sk BETWEEN :start AND :end",
+        ExpressionAttributeNames: { "#pk": "pk", "#sk": "sk" },
+        ExpressionAttributeValues: {
+          ":pk": partitionKey3(props.grain),
+          ":start": props.startKey,
+          ":end": props.endKey
+        }
+      }
+    });
+    return items.map((item) => this.parseOne(item));
+  };
+  getOne = async (props) => {
+    const { Item: item } = await this.props.dynamo.send(new GetCommand2({
+      TableName: this.props.tableName,
+      Key: { pk: partitionKey3(props.grain), sk: props.timeKey }
+    }));
+    if (item == null)
+      return;
+    return this.parseOne(item);
+  };
+  buildUpdateMembers = (props) => {
+    const names = { "#grain": "grain", "#timeKey": "timeKey" };
+    const values = {
+      ":grain": props.grain,
+      ":timeKey": props.item.timeKey
+    };
+    const setParts = ["#grain = :grain", "#timeKey = :timeKey"];
+    const addParts = [];
+    for (const field of additiveFields) {
+      names[`#${field}`] = field;
+      values[`:${field}`] = props.item[field];
+      addParts.push(`#${field} :${field}`);
+    }
+    if (props.includeHourlyDetail) {
+      for (const field of hourlyDetailFields) {
+        const value = props.item[field];
+        if (value == null)
+          continue;
+        names[`#${field}`] = field;
+        values[`:${field}`] = value;
+        setParts.push(`#${field} = :${field}`);
+      }
+    }
+    if (props.ttl != null) {
+      names["#ttl"] = "ttl";
+      values[":ttl"] = props.ttl;
+      setParts.push("#ttl = :ttl");
+    }
+    return {
+      TableName: this.props.tableName,
+      Key: { pk: partitionKey3(props.grain), sk: props.item.timeKey },
+      UpdateExpression: `SET ${setParts.join(", ")} ADD ${addParts.join(", ")}`,
+      ExpressionAttributeNames: names,
+      ExpressionAttributeValues: values
+    };
+  };
+  parseOne = (item) => {
+    const result = v3.safeParse(totalsSchema, item);
+    if (!result.success) {
+      console.error(v3.flatten(result.issues));
+      throw new Error("Malformed stored totals record");
+    }
+    return result.output;
+  };
+}
+function partitionKey3(grain) {
+  return `totals#${grain}`;
+}
+export {
+  AppliedMarkers,
+  Breakdowns,
+  Totals,
+  appliedMarkerSchema,
+  appliedSources,
+  applyDailyRollup,
+  applyMonthlyRollup,
+  breakdownSchema,
+  buildHourlyQuery,
+  chunkTransactItems,
+  classifyDevice,
+  columnMaps,
+  dayKey,
+  dimensions,
+  grains,
+  hourKey,
+  hourKeyParts,
+  hoursBetween,
+  maxWindowFor,
+  monthKey,
+  processHour,
+  requiredRecordFields,
+  runHourlyAthenaQuery,
+  shiftAnchor,
+  sumBreakdown,
+  sumTotals,
+  topNWithOther,
+  totalsSchema,
+  windowKeys,
+  yearKey
+};