@beesolve/analytics-processor 0.2.5 → 0.3.0

package/dist/index.js

@@ -130,10 +130,10 @@ var requiredRecordFields = [
 ];
 var columnMaps = {
   v2: {
-    status: "sc_status",
-    bytesSent: "sc_bytes",
-    bytesReceived: "cs_bytes",
-    timeTaken: "time_taken",
+    status: "CAST(sc_status AS integer)",
+    bytesSent: "CAST(sc_bytes AS bigint)",
+    bytesReceived: "CAST(cs_bytes AS bigint)",
+    timeTaken: "CAST(time_taken AS double)",
     resultType: "x_edge_result_type",
     detailedResultType: "x_edge_detailed_result_type",
     uriStem: "cs_uri_stem",
@@ -203,32 +203,41 @@ function buildHourlyQuery({
   const columns = columnMaps[source];
   const predicate = predicateFor({ source, hourKey, columns });
   const device = deviceCaseExpression(columns.userAgent);
-  const groupingSets = [
-    "()",
-    `(${columns.status})`,
-    "(device)",
-    `(${columns.uriStem})`,
-    `(${columns.clientIp})`
-  ].join(", ");
+  const inner = [
+    "  SELECT",
+    `    ${columns.status} AS st,`,
+    `    ${columns.bytesSent} AS scb,`,
+    `    ${columns.bytesReceived} AS csb,`,
+    `    ${columns.timeTaken} AS tt,`,
+    `    ${columns.resultType} AS rt,`,
+    `    ${columns.uriStem} AS uri,`,
+    `    ${columns.clientIp} AS ip,`,
+    `    ${device} AS device`,
+    `  FROM "${table}"`,
+    `  WHERE ${predicate}`
+  ].join(`
+`);
+  const groupingSets = ["()", "(st)", "(device)", "(uri)", "(ip)"].join(", ");
   return [
     "SELECT",
-    `  ${columns.status} AS status_value,`,
-    `  ${device} AS device,`,
-    `  ${columns.uriStem} AS path_value,`,
-    `  ${columns.clientIp} AS ip_value,`,
+    "  st AS status_value,",
+    "  device,",
+    "  uri AS path_value,",
+    "  ip AS ip_value,",
     "  count(*) AS hits,",
-    `  sum(CASE WHEN ${columns.status} BETWEEN 200 AND 399 THEN 1 ELSE 0 END) AS success,`,
-    `  sum(CASE WHEN ${columns.status} BETWEEN 400 AND 599 THEN 1 ELSE 0 END) AS errors,`,
-    `  sum(CASE WHEN ${columns.status} BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS errors4xx,`,
-    `  sum(CASE WHEN ${columns.status} BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS errors5xx,`,
-    `  sum(${columns.bytesSent}) AS bytes_sent,`,
-    `  sum(${columns.bytesReceived}) AS bytes_received,`,
-    `  approx_percentile(${columns.timeTaken}, 0.5) AS p50_time_taken,`,
-    `  approx_percentile(${columns.timeTaken}, 0.95) AS p95_time_taken,`,
-    `  sum(CASE WHEN ${columns.resultType} = 'Hit' THEN 1 ELSE 0 END) AS cache_hits,`,
-    `  count(DISTINCT ${columns.clientIp}) AS unique_ip_count`,
-    `FROM "${table}"`,
-    `WHERE ${predicate}`,
+    "  sum(CASE WHEN st BETWEEN 200 AND 399 THEN 1 ELSE 0 END) AS success,",
+    "  sum(CASE WHEN st BETWEEN 400 AND 599 THEN 1 ELSE 0 END) AS errors,",
+    "  sum(CASE WHEN st BETWEEN 400 AND 499 THEN 1 ELSE 0 END) AS errors4xx,",
+    "  sum(CASE WHEN st BETWEEN 500 AND 599 THEN 1 ELSE 0 END) AS errors5xx,",
+    "  sum(scb) AS bytes_sent,",
+    "  sum(csb) AS bytes_received,",
+    "  approx_percentile(tt, 0.5) AS p50_time_taken,",
+    "  approx_percentile(tt, 0.95) AS p95_time_taken,",
+    "  sum(CASE WHEN rt = 'Hit' THEN 1 ELSE 0 END) AS cache_hits,",
+    "  count(DISTINCT ip) AS unique_ip_count",
+    "FROM (",
+    inner,
+    ")",
     `GROUP BY GROUPING SETS (${groupingSets})`
   ].join(`
 `);
@@ -473,6 +482,57 @@ class Breakdowns {
 function partitionKey2(props) {
   return `breakdown#${props.dimension}#${props.grain}#${props.periodKey}`;
 }
+// packages/analytics-processor/src/config.ts
+import { GetCommand as GetCommand2, UpdateCommand as UpdateCommand2 } from "@aws-sdk/lib-dynamodb";
+import * as v3 from "valibot";
+var configSchema = v3.object({
+  pk: v3.literal("system#config"),
+  sk: v3.literal("analytics"),
+  startKey: v3.string(),
+  startedAt: v3.string()
+});
+var configKey = { pk: "system#config", sk: "analytics" };
+function isConditionalCheckFailure(error) {
+  if (error == null || typeof error !== "object")
+    return false;
+  const name = "name" in error ? error.name : undefined;
+  return name === "ConditionalCheckFailedException";
+}
+
+class Config {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  get = async () => {
+    const { Item: item } = await this.props.dynamo.send(new GetCommand2({ TableName: this.props.tableName, Key: configKey }));
+    if (item == null)
+      return null;
+    return v3.parse(configSchema, item);
+  };
+  ensureStart = async (props) => {
+    try {
+      await this.props.dynamo.send(new UpdateCommand2({
+        TableName: this.props.tableName,
+        Key: configKey,
+        UpdateExpression: "SET #startKey = :hourKey, #startedAt = if_not_exists(#startedAt, :now)",
+        ConditionExpression: "attribute_not_exists(#startKey) OR :hourKey < #startKey",
+        ExpressionAttributeNames: {
+          "#startKey": "startKey",
+          "#startedAt": "startedAt"
+        },
+        ExpressionAttributeValues: {
+          ":hourKey": props.hourKey,
+          ":now": new Date().toISOString()
+        }
+      }));
+    } catch (error) {
+      if (isConditionalCheckFailure(error))
+        return;
+      throw error;
+    }
+  };
+}
 // packages/analytics-processor/src/device.ts
 var botPattern = /bot|crawler|spider|crawl|slurp|mediapartners|facebookexternalhit|bingpreview|duckduckbot|baiduspider|yandex|semrush|ahrefs|headlesschrome|python-requests|curl|wget|monitor|pingdom|uptime/i;
 var mobilePattern = /mobi|android|iphone|ipad|ipod|iemobile|blackberry|opera mini|windows phone|webos/i;
@@ -553,11 +613,15 @@ async function processHour({
     byPath: query.byPath,
     byCountry
   });
-  return sendTransacted({
+  const result = await sendTransacted({
     dynamo,
     leading: [markerPut, totalsUpdate],
     rest: breakdownUpdates
   });
+  if (result.status === "applied" && query.totals.hits > 0 && models.config != null) {
+    await models.config.ensureStart({ hourKey });
+  }
+  return result;
 }
 async function applyDailyRollup({
   dayKey,
@@ -772,24 +836,196 @@ function computeHourlyTtl(hourlyTtlDays) {
 function toBreakdownRow(item) {
   return { value: item.value, hits: item.hits, bytes: item.bytes };
 }
-// packages/analytics-processor/src/time.ts
+// packages/analytics-processor/src/resolveLive.ts
+var emptyTotals = {
+  hits: 0,
+  success: 0,
+  errors: 0,
+  errors4xx: 0,
+  errors5xx: 0,
+  bytesSent: 0,
+  bytesReceived: 0
+};
+async function resolveTotalsLive({
+  models,
+  grain,
+  periodKey,
+  now
+}) {
+  if (grain === "hour") {
+    const stored = await models.totals.getOne({ grain: "hour", timeKey: periodKey });
+    return stored == null ? emptyTotals : pickTotals(stored);
+  }
+  if (await isSealed({ models, grain, periodKey, now })) {
+    const stored = await models.totals.getOne({ grain, timeKey: periodKey });
+    if (stored != null)
+      return pickTotals(stored);
+  }
+  if (grain === "day") {
+    const lastHour = isCurrentPeriod({ grain: "day", periodKey, now }) ? now.getUTCHours() : 23;
+    const hours = await models.totals.queryWindow({
+      grain: "hour",
+      startKey: `${periodKey}-00`,
+      endKey: `${periodKey}-${pad2(lastHour)}`
+    });
+    return sumTotals(hours);
+  }
+  const dayKeys = dayKeysInProgress({ periodKey, now });
+  const sealedDays = await models.totals.queryWindow({
+    grain: "day",
+    startKey: `${periodKey}-01`,
+    endKey: `${periodKey}-31`
+  });
+  const sealedByKey = indexByTimeKey(sealedDays);
+  const resolvedDays = await Promise.all(dayKeys.map(async (dayKey) => {
+    const stored = sealedByKey[dayKey];
+    if (stored != null && !isCurrentPeriod({ grain: "day", periodKey: dayKey, now })) {
+      return pickTotals(stored);
+    }
+    return resolveTotalsLive({ models, grain: "day", periodKey: dayKey, now });
+  }));
+  return sumTotals(resolvedDays);
+}
+async function resolveBreakdownLive({
+  models,
+  dimension,
+  grain,
+  periodKey,
+  now
+}) {
+  if (grain === "hour") {
+    return toRows(await models.breakdowns.getForPeriod({ dimension, grain: "hour", periodKey }));
+  }
+  if (await isSealed({ models, grain, periodKey, now })) {
+    const stored = await models.breakdowns.getForPeriod({ dimension, grain, periodKey });
+    if (stored.length > 0)
+      return toRows(stored);
+  }
+  const finer = finerGrain(grain);
+  const childKeys = grain === "day" ? hourKeysInProgress({ periodKey, now }) : dayKeysInProgress({ periodKey, now });
+  const children = await Promise.all(childKeys.map((childKey) => resolveBreakdownLive({ models, dimension, grain: finer, periodKey: childKey, now })));
+  return sumBreakdown(children.flat());
+}
+async function isSealed({
+  models,
+  grain,
+  periodKey,
+  now
+}) {
+  if (isCurrentPeriod({ grain, periodKey, now }))
+    return false;
+  return models.markers.isApplied({ grain, periodKey });
+}
+function isCurrentPeriod({
+  grain,
+  periodKey,
+  now
+}) {
+  return periodKey === currentKeyFor({ grain, now });
+}
+function currentKeyFor({ grain, now }) {
+  const year = pad4(now.getUTCFullYear());
+  const month = pad2(now.getUTCMonth() + 1);
+  const day = pad2(now.getUTCDate());
+  const hour = pad2(now.getUTCHours());
+  if (grain === "month")
+    return `${year}-${month}`;
+  if (grain === "day")
+    return `${year}-${month}-${day}`;
+  return `${year}-${month}-${day}-${hour}`;
+}
+function finerGrain(grain) {
+  return grain === "month" ? "day" : "hour";
+}
+function hourKeysInProgress({
+  periodKey,
+  now
+}) {
+  const lastHour = isCurrentPeriod({ grain: "day", periodKey, now }) ? now.getUTCHours() : 23;
+  const keys = [];
+  for (let hour = 0;hour <= lastHour; hour += 1) {
+    keys.push(`${periodKey}-${pad2(hour)}`);
+  }
+  return keys;
+}
+function dayKeysInProgress({
+  periodKey,
+  now
+}) {
+  const lastDay = isCurrentPeriod({ grain: "month", periodKey, now }) ? now.getUTCDate() : daysInMonth(periodKey);
+  const keys = [];
+  for (let day = 1;day <= lastDay; day += 1) {
+    keys.push(`${periodKey}-${pad2(day)}`);
+  }
+  return keys;
+}
+function daysInMonth(monthKey) {
+  const [year, month] = monthKey.split("-").map(Number);
+  return new Date(Date.UTC(year ?? 1970, month ?? 1, 0)).getUTCDate();
+}
+function indexByTimeKey(items) {
+  const byKey = {};
+  for (const item of items) {
+    byKey[item.timeKey] = item;
+  }
+  return byKey;
+}
+function toRows(items) {
+  return items.map((item) => ({ value: item.value, hits: item.hits, bytes: item.bytes }));
+}
+function pickTotals(item) {
+  return {
+    hits: item.hits,
+    success: item.success,
+    errors: item.errors,
+    errors4xx: item.errors4xx,
+    errors5xx: item.errors5xx,
+    bytesSent: item.bytesSent,
+    bytesReceived: item.bytesReceived
+  };
+}
 function pad2(value) {
   return value.toString().padStart(2, "0");
 }
 function pad4(value) {
   return value.toString().padStart(4, "0");
 }
+// packages/analytics-processor/src/time.ts
+function pad22(value) {
+  return value.toString().padStart(2, "0");
+}
+function pad42(value) {
+  return value.toString().padStart(4, "0");
+}
 function yearKey(date) {
-  return pad4(date.getUTCFullYear());
+  return pad42(date.getUTCFullYear());
 }
 function monthKey(date) {
-  return `${pad4(date.getUTCFullYear())}-${pad2(date.getUTCMonth() + 1)}`;
+  return `${pad42(date.getUTCFullYear())}-${pad22(date.getUTCMonth() + 1)}`;
 }
 function dayKey(date) {
-  return `${monthKey(date)}-${pad2(date.getUTCDate())}`;
+  return `${monthKey(date)}-${pad22(date.getUTCDate())}`;
 }
 function hourKey(date) {
-  return `${dayKey(date)}-${pad2(date.getUTCHours())}`;
+  return `${dayKey(date)}-${pad22(date.getUTCHours())}`;
+}
+function parsePeriodKey(periodKey) {
+  const match = /^(\d{4})(?:-(\d{2}))?(?:-(\d{2}))?(?:-(\d{2}))?$/.exec(periodKey);
+  if (match == null) {
+    throw new Error(`Invalid period key: ${periodKey}`);
+  }
+  const [, year, month, day, hour] = match;
+  return new Date(Date.UTC(Number(year), month == null ? 0 : Number(month) - 1, day == null ? 1 : Number(day), hour == null ? 0 : Number(hour)));
+}
+function grainForPeriodKey(periodKey) {
+  const segments = periodKey.split("-").length;
+  if (segments === 1)
+    return "year";
+  if (segments === 2)
+    return "month";
+  if (segments === 3)
+    return "day";
+  return "hour";
 }
 function maxWindowFor(grain) {
   if (grain === "hour")
@@ -839,9 +1075,9 @@ function hoursBetween({ from, to }) {
   return keys;
 }
 // packages/analytics-processor/src/totals.ts
-import { GetCommand as GetCommand2, UpdateCommand as UpdateCommand2 } from "@aws-sdk/lib-dynamodb";
+import { GetCommand as GetCommand3, UpdateCommand as UpdateCommand3 } from "@aws-sdk/lib-dynamodb";
 import { queryAll as queryAll2 } from "@beesolve/dynamo-helpers";
-import * as v3 from "valibot";
+import * as v4 from "valibot";
 var additiveFields = [
   "hits",
   "success",
@@ -857,23 +1093,23 @@ var hourlyDetailFields = [
   "cacheHitRatio",
   "uniqueIpCount"
 ];
-var totalsSchema = v3.object({
-  pk: v3.string(),
-  sk: v3.string(),
-  grain: v3.picklist(grains),
-  timeKey: v3.string(),
-  hits: v3.number(),
-  success: v3.number(),
-  errors: v3.number(),
-  errors4xx: v3.number(),
-  errors5xx: v3.number(),
-  bytesSent: v3.number(),
-  bytesReceived: v3.number(),
-  p50TimeTaken: v3.optional(v3.number()),
-  p95TimeTaken: v3.optional(v3.number()),
-  cacheHitRatio: v3.optional(v3.number()),
-  uniqueIpCount: v3.optional(v3.number()),
-  ttl: v3.optional(v3.number())
+var totalsSchema = v4.object({
+  pk: v4.string(),
+  sk: v4.string(),
+  grain: v4.picklist(grains),
+  timeKey: v4.string(),
+  hits: v4.number(),
+  success: v4.number(),
+  errors: v4.number(),
+  errors4xx: v4.number(),
+  errors5xx: v4.number(),
+  bytesSent: v4.number(),
+  bytesReceived: v4.number(),
+  p50TimeTaken: v4.optional(v4.number()),
+  p95TimeTaken: v4.optional(v4.number()),
+  cacheHitRatio: v4.optional(v4.number()),
+  uniqueIpCount: v4.optional(v4.number()),
+  ttl: v4.optional(v4.number())
 });
 
 class Totals {
@@ -883,11 +1119,11 @@ class Totals {
   }
   addHourly = async (props) => {
     const update = this.buildHourlyUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+    await this.props.dynamo.send(new UpdateCommand3(update.Update));
   };
   addAggregate = async (props) => {
     const update = this.buildAggregateUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+    await this.props.dynamo.send(new UpdateCommand3(update.Update));
   };
   buildHourlyUpdate = (props) => ({
     Update: this.buildUpdateMembers({
@@ -921,7 +1157,7 @@ class Totals {
     return items.map((item) => this.parseOne(item));
   };
   getOne = async (props) => {
-    const { Item: item } = await this.props.dynamo.send(new GetCommand2({
+    const { Item: item } = await this.props.dynamo.send(new GetCommand3({
       TableName: this.props.tableName,
       Key: { pk: partitionKey3(props.grain), sk: props.timeKey }
     }));
@@ -966,9 +1202,9 @@ class Totals {
     };
   };
   parseOne = (item) => {
-    const result = v3.safeParse(totalsSchema, item);
+    const result = v4.safeParse(totalsSchema, item);
     if (!result.success) {
-      console.error(v3.flatten(result.issues));
+      console.error(v4.flatten(result.issues));
       throw new Error("Malformed stored totals record");
     }
     return result.output;
@@ -980,6 +1216,7 @@ function partitionKey3(grain) {
 export {
   AppliedMarkers,
   Breakdowns,
+  Config,
   Totals,
   appliedMarkerSchema,
   appliedSources,
@@ -990,16 +1227,21 @@ export {
   chunkTransactItems,
   classifyDevice,
   columnMaps,
+  configSchema,
   dayKey,
   dimensions,
+  grainForPeriodKey,
   grains,
   hourKey,
   hourKeyParts,
   hoursBetween,
   maxWindowFor,
   monthKey,
+  parsePeriodKey,
   processHour,
   requiredRecordFields,
+  resolveBreakdownLive,
+  resolveTotalsLive,
   runHourlyAthenaQuery,
   shiftAnchor,
   sumBreakdown,