@beesolve/analytics-processor 0.2.6 → 0.4.0

package/dist/index.js

@@ -482,6 +482,57 @@ class Breakdowns {
 function partitionKey2(props) {
   return `breakdown#${props.dimension}#${props.grain}#${props.periodKey}`;
 }
+// packages/analytics-processor/src/config.ts
+import { GetCommand as GetCommand2, UpdateCommand as UpdateCommand2 } from "@aws-sdk/lib-dynamodb";
+import * as v3 from "valibot";
+var configSchema = v3.object({
+  pk: v3.literal("system#config"),
+  sk: v3.literal("analytics"),
+  startKey: v3.string(),
+  startedAt: v3.string()
+});
+var configKey = { pk: "system#config", sk: "analytics" };
+function isConditionalCheckFailure(error) {
+  if (error == null || typeof error !== "object")
+    return false;
+  const name = "name" in error ? error.name : undefined;
+  return name === "ConditionalCheckFailedException";
+}
+
+class Config {
+  props;
+  constructor(props) {
+    this.props = props;
+  }
+  get = async () => {
+    const { Item: item } = await this.props.dynamo.send(new GetCommand2({ TableName: this.props.tableName, Key: configKey }));
+    if (item == null)
+      return null;
+    return v3.parse(configSchema, item);
+  };
+  ensureStart = async (props) => {
+    try {
+      await this.props.dynamo.send(new UpdateCommand2({
+        TableName: this.props.tableName,
+        Key: configKey,
+        UpdateExpression: "SET #startKey = :hourKey, #startedAt = if_not_exists(#startedAt, :now)",
+        ConditionExpression: "attribute_not_exists(#startKey) OR :hourKey < #startKey",
+        ExpressionAttributeNames: {
+          "#startKey": "startKey",
+          "#startedAt": "startedAt"
+        },
+        ExpressionAttributeValues: {
+          ":hourKey": props.hourKey,
+          ":now": new Date().toISOString()
+        }
+      }));
+    } catch (error) {
+      if (isConditionalCheckFailure(error))
+        return;
+      throw error;
+    }
+  };
+}
 // packages/analytics-processor/src/device.ts
 var botPattern = /bot|crawler|spider|crawl|slurp|mediapartners|facebookexternalhit|bingpreview|duckduckbot|baiduspider|yandex|semrush|ahrefs|headlesschrome|python-requests|curl|wget|monitor|pingdom|uptime/i;
 var mobilePattern = /mobi|android|iphone|ipad|ipod|iemobile|blackberry|opera mini|windows phone|webos/i;
@@ -562,11 +613,15 @@ async function processHour({
     byPath: query.byPath,
     byCountry
   });
-  return sendTransacted({
+  const result = await sendTransacted({
     dynamo,
     leading: [markerPut, totalsUpdate],
     rest: breakdownUpdates
   });
+  if (result.status === "applied" && query.totals.hits > 0 && models.config != null) {
+    await models.config.ensureStart({ hourKey });
+  }
+  return result;
 }
 async function applyDailyRollup({
   dayKey,
@@ -781,24 +836,237 @@ function computeHourlyTtl(hourlyTtlDays) {
 function toBreakdownRow(item) {
   return { value: item.value, hits: item.hits, bytes: item.bytes };
 }
-// packages/analytics-processor/src/time.ts
+// packages/analytics-processor/src/resolveLive.ts
+var bucketStates = ["sealed", "partial", "pending"];
+var emptyTotals = {
+  hits: 0,
+  success: 0,
+  errors: 0,
+  errors4xx: 0,
+  errors5xx: 0,
+  bytesSent: 0,
+  bytesReceived: 0
+};
+async function resolveTotalsLive({
+  models,
+  grain,
+  periodKey,
+  now
+}) {
+  if (grain === "hour") {
+    const stored = await models.totals.getOne({ grain: "hour", timeKey: periodKey });
+    return stored == null ? emptyTotals : pickTotals(stored);
+  }
+  if (await isSealed({ models, grain, periodKey, now })) {
+    const stored = await models.totals.getOne({ grain, timeKey: periodKey });
+    if (stored != null)
+      return pickTotals(stored);
+  }
+  if (grain === "day") {
+    const lastHour = isCurrentPeriod({ grain: "day", periodKey, now }) ? now.getUTCHours() : 23;
+    const hours = await models.totals.queryWindow({
+      grain: "hour",
+      startKey: `${periodKey}-00`,
+      endKey: `${periodKey}-${pad2(lastHour)}`
+    });
+    return sumTotals(hours);
+  }
+  const storedDays = await models.totals.queryWindow({
+    grain: "day",
+    startKey: `${periodKey}-01`,
+    endKey: `${periodKey}-31`
+  });
+  const base = sumTotals(storedDays.filter((item) => !isCurrentPeriod({ grain: "day", periodKey: item.timeKey, now })));
+  const currentDayKey = currentChildKey({ parentGrain: "month", periodKey, now });
+  if (currentDayKey == null)
+    return base;
+  const currentDay = await resolveTotalsLive({
+    models,
+    grain: "day",
+    periodKey: currentDayKey,
+    now
+  });
+  return sumTotals([base, currentDay]);
+}
+async function resolveBreakdownLive({
+  models,
+  dimension,
+  grain,
+  periodKey,
+  now
+}) {
+  if (grain === "hour") {
+    return toRows(await models.breakdowns.getForPeriod({ dimension, grain: "hour", periodKey }));
+  }
+  if (await isSealed({ models, grain, periodKey, now })) {
+    const stored = await models.breakdowns.getForPeriod({ dimension, grain, periodKey });
+    if (stored.length > 0)
+      return toRows(stored);
+  }
+  if (grain === "day") {
+    const hourKeys = hourKeysInProgress({ periodKey, now });
+    const hourRows = await Promise.all(hourKeys.map((hourKey) => models.breakdowns.getForPeriod({ dimension, grain: "hour", periodKey: hourKey })));
+    return sumBreakdown(hourRows.flat().map(toRow));
+  }
+  const storedDayKeys = pastDayKeys({ periodKey, now });
+  const storedDayRows = await Promise.all(storedDayKeys.map((dayKey) => models.breakdowns.getForPeriod({ dimension, grain: "day", periodKey: dayKey })));
+  const rows = storedDayRows.flat().map(toRow);
+  const currentDayKey = currentChildKey({ parentGrain: "month", periodKey, now });
+  if (currentDayKey != null) {
+    const currentDay = await resolveBreakdownLive({
+      models,
+      dimension,
+      grain: "day",
+      periodKey: currentDayKey,
+      now
+    });
+    rows.push(...currentDay);
+  }
+  return sumBreakdown(rows);
+}
+async function classifyBucketState({
+  models,
+  grain,
+  periodKey,
+  now,
+  hasData
+}) {
+  if (await isSealed({ models, grain, periodKey, now }))
+    return "sealed";
+  if (hasData)
+    return "partial";
+  return "pending";
+}
+async function isSealed({
+  models,
+  grain,
+  periodKey,
+  now
+}) {
+  if (!isPastPeriod({ grain, periodKey, now }))
+    return false;
+  return models.markers.isApplied({ grain, periodKey });
+}
+function isCurrentPeriod({
+  grain,
+  periodKey,
+  now
+}) {
+  return periodKey === currentKeyFor({ grain, now });
+}
+function isPastPeriod({
+  grain,
+  periodKey,
+  now
+}) {
+  return periodKey < currentKeyFor({ grain, now });
+}
+function currentKeyFor({ grain, now }) {
+  const year = pad4(now.getUTCFullYear());
+  const month = pad2(now.getUTCMonth() + 1);
+  const day = pad2(now.getUTCDate());
+  const hour = pad2(now.getUTCHours());
+  if (grain === "month")
+    return `${year}-${month}`;
+  if (grain === "day")
+    return `${year}-${month}-${day}`;
+  return `${year}-${month}-${day}-${hour}`;
+}
+function currentChildKey({
+  parentGrain,
+  periodKey,
+  now
+}) {
+  if (!isCurrentPeriod({ grain: parentGrain, periodKey, now }))
+    return null;
+  return currentKeyFor({ grain: "day", now });
+}
+function hourKeysInProgress({
+  periodKey,
+  now
+}) {
+  const lastHour = isCurrentPeriod({ grain: "day", periodKey, now }) ? now.getUTCHours() : 23;
+  const keys = [];
+  for (let hour = 0;hour <= lastHour; hour += 1) {
+    keys.push(`${periodKey}-${pad2(hour)}`);
+  }
+  return keys;
+}
+function pastDayKeys({
+  periodKey,
+  now
+}) {
+  const isCurrentMonth = isCurrentPeriod({ grain: "month", periodKey, now });
+  const lastDay = isCurrentMonth ? now.getUTCDate() - 1 : daysInMonth(periodKey);
+  const keys = [];
+  for (let day = 1;day <= lastDay; day += 1) {
+    keys.push(`${periodKey}-${pad2(day)}`);
+  }
+  return keys;
+}
+function daysInMonth(monthKey) {
+  const [year, month] = monthKey.split("-").map(Number);
+  return new Date(Date.UTC(year ?? 1970, month ?? 1, 0)).getUTCDate();
+}
+function toRow(item) {
+  return { value: item.value, hits: item.hits, bytes: item.bytes };
+}
+function toRows(items) {
+  return items.map(toRow);
+}
+function pickTotals(item) {
+  return {
+    hits: item.hits,
+    success: item.success,
+    errors: item.errors,
+    errors4xx: item.errors4xx,
+    errors5xx: item.errors5xx,
+    bytesSent: item.bytesSent,
+    bytesReceived: item.bytesReceived
+  };
+}
 function pad2(value) {
   return value.toString().padStart(2, "0");
 }
 function pad4(value) {
   return value.toString().padStart(4, "0");
 }
+// packages/analytics-processor/src/time.ts
+function pad22(value) {
+  return value.toString().padStart(2, "0");
+}
+function pad42(value) {
+  return value.toString().padStart(4, "0");
+}
 function yearKey(date) {
-  return pad4(date.getUTCFullYear());
+  return pad42(date.getUTCFullYear());
 }
 function monthKey(date) {
-  return `${pad4(date.getUTCFullYear())}-${pad2(date.getUTCMonth() + 1)}`;
+  return `${pad42(date.getUTCFullYear())}-${pad22(date.getUTCMonth() + 1)}`;
 }
 function dayKey(date) {
-  return `${monthKey(date)}-${pad2(date.getUTCDate())}`;
+  return `${monthKey(date)}-${pad22(date.getUTCDate())}`;
 }
 function hourKey(date) {
-  return `${dayKey(date)}-${pad2(date.getUTCHours())}`;
+  return `${dayKey(date)}-${pad22(date.getUTCHours())}`;
+}
+function parsePeriodKey(periodKey) {
+  const match = /^(\d{4})(?:-(\d{2}))?(?:-(\d{2}))?(?:-(\d{2}))?$/.exec(periodKey);
+  if (match == null) {
+    throw new Error(`Invalid period key: ${periodKey}`);
+  }
+  const [, year, month, day, hour] = match;
+  return new Date(Date.UTC(Number(year), month == null ? 0 : Number(month) - 1, day == null ? 1 : Number(day), hour == null ? 0 : Number(hour)));
+}
+function grainForPeriodKey(periodKey) {
+  const segments = periodKey.split("-").length;
+  if (segments === 1)
+    return "year";
+  if (segments === 2)
+    return "month";
+  if (segments === 3)
+    return "day";
+  return "hour";
 }
 function maxWindowFor(grain) {
   if (grain === "hour")
@@ -848,9 +1116,9 @@ function hoursBetween({ from, to }) {
   return keys;
 }
 // packages/analytics-processor/src/totals.ts
-import { GetCommand as GetCommand2, UpdateCommand as UpdateCommand2 } from "@aws-sdk/lib-dynamodb";
+import { GetCommand as GetCommand3, UpdateCommand as UpdateCommand3 } from "@aws-sdk/lib-dynamodb";
 import { queryAll as queryAll2 } from "@beesolve/dynamo-helpers";
-import * as v3 from "valibot";
+import * as v4 from "valibot";
 var additiveFields = [
   "hits",
   "success",
@@ -866,23 +1134,23 @@ var hourlyDetailFields = [
   "cacheHitRatio",
   "uniqueIpCount"
 ];
-var totalsSchema = v3.object({
-  pk: v3.string(),
-  sk: v3.string(),
-  grain: v3.picklist(grains),
-  timeKey: v3.string(),
-  hits: v3.number(),
-  success: v3.number(),
-  errors: v3.number(),
-  errors4xx: v3.number(),
-  errors5xx: v3.number(),
-  bytesSent: v3.number(),
-  bytesReceived: v3.number(),
-  p50TimeTaken: v3.optional(v3.number()),
-  p95TimeTaken: v3.optional(v3.number()),
-  cacheHitRatio: v3.optional(v3.number()),
-  uniqueIpCount: v3.optional(v3.number()),
-  ttl: v3.optional(v3.number())
+var totalsSchema = v4.object({
+  pk: v4.string(),
+  sk: v4.string(),
+  grain: v4.picklist(grains),
+  timeKey: v4.string(),
+  hits: v4.number(),
+  success: v4.number(),
+  errors: v4.number(),
+  errors4xx: v4.number(),
+  errors5xx: v4.number(),
+  bytesSent: v4.number(),
+  bytesReceived: v4.number(),
+  p50TimeTaken: v4.optional(v4.number()),
+  p95TimeTaken: v4.optional(v4.number()),
+  cacheHitRatio: v4.optional(v4.number()),
+  uniqueIpCount: v4.optional(v4.number()),
+  ttl: v4.optional(v4.number())
 });
 
 class Totals {
@@ -892,11 +1160,11 @@ class Totals {
   }
   addHourly = async (props) => {
     const update = this.buildHourlyUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+    await this.props.dynamo.send(new UpdateCommand3(update.Update));
   };
   addAggregate = async (props) => {
     const update = this.buildAggregateUpdate(props);
-    await this.props.dynamo.send(new UpdateCommand2(update.Update));
+    await this.props.dynamo.send(new UpdateCommand3(update.Update));
   };
   buildHourlyUpdate = (props) => ({
     Update: this.buildUpdateMembers({
@@ -930,7 +1198,7 @@ class Totals {
     return items.map((item) => this.parseOne(item));
   };
   getOne = async (props) => {
-    const { Item: item } = await this.props.dynamo.send(new GetCommand2({
+    const { Item: item } = await this.props.dynamo.send(new GetCommand3({
       TableName: this.props.tableName,
       Key: { pk: partitionKey3(props.grain), sk: props.timeKey }
     }));
@@ -975,9 +1243,9 @@ class Totals {
     };
   };
   parseOne = (item) => {
-    const result = v3.safeParse(totalsSchema, item);
+    const result = v4.safeParse(totalsSchema, item);
     if (!result.success) {
-      console.error(v3.flatten(result.issues));
+      console.error(v4.flatten(result.issues));
       throw new Error("Malformed stored totals record");
     }
     return result.output;
@@ -989,26 +1257,34 @@ function partitionKey3(grain) {
 export {
   AppliedMarkers,
   Breakdowns,
+  Config,
   Totals,
   appliedMarkerSchema,
   appliedSources,
   applyDailyRollup,
   applyMonthlyRollup,
   breakdownSchema,
+  bucketStates,
   buildHourlyQuery,
   chunkTransactItems,
+  classifyBucketState,
   classifyDevice,
   columnMaps,
+  configSchema,
   dayKey,
   dimensions,
+  grainForPeriodKey,
   grains,
   hourKey,
   hourKeyParts,
   hoursBetween,
   maxWindowFor,
   monthKey,
+  parsePeriodKey,
   processHour,
   requiredRecordFields,
+  resolveBreakdownLive,
+  resolveTotalsLive,
   runHourlyAthenaQuery,
   shiftAnchor,
   sumBreakdown,