@hot-updater/server 1.0.0-rc.1 → 1.0.0-rc.16

package/src/plugins/insights/recordEvent.spec.ts

@@ -0,0 +1,443 @@
+import { DatabaseSync } from "node:sqlite";
+
+import type { BundleEventRow } from "@hot-updater/plugin-core";
+import {
+  countInsightsDistinct,
+  createMemoryAdapter,
+  type DatabaseAdapter,
+  type WriteOp,
+} from "@hot-updater/plugin-core/internal";
+import { createPluginTestHarness } from "@hot-updater/test-utils";
+import { describe, expect, it } from "vitest";
+
+import * as engine from "../../database";
+import type { HotUpdaterDatabase } from "../../database/database";
+import { createKvAdapter } from "../../database/kv/kvAdapter";
+import { createMemoryKeyValueStore } from "../../database/kv/kvTestStore";
+import { createSqlAdapter } from "../../database/sql/sqlAdapter";
+import { sqliteExecutor } from "../../database/sql/sqlTestExecutors";
+import {
+  insights,
+  insightsIdentity,
+  type InsightsIdentityParts,
+  type InsightsSchema,
+} from "./index";
+import { DAILY_EVENTS } from "./schema";
+
+const HOUR = 3_600_000;
+const DAY = 86_400_000;
+const T = Date.UTC(2026, 8, 23, 10, 15);
+const hour = (ms: number) => ms - (ms % HOUR);
+
+const uuid = (n: number) =>
+  `00000000-0000-7000-8000-${String(n).padStart(12, "0")}`;
+
+const event = (
+  n: number,
+  overrides: Partial<BundleEventRow> = {},
+): BundleEventRow =>
+  ({
+    id: uuid(n),
+    type: "UPDATE_APPLIED",
+    install_id: "install-1",
+    user_id: "user-1",
+    from_release_id: "release-1",
+    from_bundle_id: "bundle-1",
+    to_release_id: "release-2",
+    to_bundle_id: "bundle-2",
+    platform: "ios",
+    app_version: "1.0.0",
+    channel: "production",
+    metadata: {
+      username: null,
+      cohort: "1",
+      update_strategy: "appVersion",
+      fingerprint_hash: null,
+      sdk_version: null,
+    },
+    received_at_ms: T,
+    ...overrides,
+  }) as BundleEventRow;
+
+const backends: [string, () => DatabaseAdapter][] = [
+  ["memory", () => createMemoryAdapter()],
+  [
+    "SQLite",
+    () =>
+      createSqlAdapter({
+        executor: sqliteExecutor(new DatabaseSync(":memory:")),
+      }),
+  ],
+  [
+    "the key-value helper",
+    () => createKvAdapter({ store: createMemoryKeyValueStore() }),
+  ],
+];
+
+const identity = (
+  parts: Partial<InsightsIdentityParts> &
+    Pick<InsightsIdentityParts, "periodKind">,
+) =>
+  insightsIdentity({
+    scopeKind: "channel",
+    releaseKind: "all",
+    releaseId: "",
+    channel: "production",
+    platform: "ios",
+    appVersionKind: "all",
+    appVersion: "",
+    ...parts,
+  });
+
+describe.each(backends)("insights recordEvent on %s", (_name, adapter) => {
+  const setup = async (wrap = (inner: DatabaseAdapter) => inner) => {
+    const harness = await createPluginTestHarness(insights(), {
+      engine,
+      adapter: wrap(adapter()),
+    });
+    const db = harness.db as HotUpdaterDatabase<InsightsSchema>;
+    const at = (key: string, bucket: number) => ({
+      index: "window" as const,
+      where: { identity: key },
+      range: { gte: bucket, lte: bucket },
+      limit: 100,
+    });
+    const overview = async (key: string, bucket: number) =>
+      (await db.findAggregates("insights_overview", at(key, bucket))).rows[0];
+    const sketch = async (key: string, bucket: number) =>
+      (await db.findAggregates("insights_sketches", at(key, bucket))).rows[0];
+    const distribution = async () =>
+      (
+        await db.findAggregates("insights_distribution", {
+          index: "byScope",
+          where: { channel: "production", platform: "ios" },
+          limit: 100,
+        })
+      ).rows;
+    const byBundle = async (
+      field: "from_bundle_id" | "to_bundle_id",
+      bundleId: string,
+      type: string,
+    ) =>
+      (
+        await db.findAggregates("insights_latest_by_bundle", {
+          index: "byBundle",
+          where: {
+            platform: "ios",
+            channel: "production",
+            bundle_field: field,
+            bundle_id: bundleId,
+            type,
+          },
+          limit: 100,
+        })
+      ).rows;
+    const outcomes = async (type: string, ref: string) =>
+      (
+        await db.findAggregates("insights_outcomes", {
+          index: "byRef",
+          where: {
+            platform: "ios",
+            channel: "production",
+            type,
+            bundle_ref: ref,
+          },
+          limit: 100,
+        })
+      ).rows;
+    /** The per-day count of every event, which the global event list reads. */
+    const everyEvent = async () =>
+      (
+        await db.findAggregates("insights_outcomes", {
+          index: "byRef",
+          where: DAILY_EVENTS,
+          limit: 100,
+        })
+      ).rows;
+    return {
+      ...harness,
+      db,
+      overview,
+      sketch,
+      distribution,
+      byBundle,
+      outcomes,
+      everyEvent,
+    };
+  };
+
+  it("records the event with derived keys, the head, gauges, counters, and sketches", async () => {
+    const {
+      api,
+      db,
+      overview,
+      sketch,
+      distribution,
+      byBundle,
+      outcomes,
+      everyEvent,
+    } = await setup();
+    await api.recordEvent(event(1));
+
+    await expect(
+      db.findOne("bundle_events", { id: uuid(1) }),
+    ).resolves.toMatchObject({
+      day: T - (T % DAY),
+      movement_install_id: "install-1",
+      bundle_ref: ["to:bundle-2"],
+    });
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: "install-1" }),
+    ).resolves.toEqual({ ...event(1), current_release_id: "release-2" });
+    await expect(distribution()).resolves.toEqual([
+      {
+        channel: "production",
+        platform: "ios",
+        app_version: "1.0.0",
+        release_id: "release-2",
+        bucket_start_ms: hour(T),
+        latest_installations: 1,
+      },
+    ]);
+    for (const [field, bundle] of [
+      ["from_bundle_id", "bundle-1"],
+      ["to_bundle_id", "bundle-2"],
+    ] as const) {
+      await expect(
+        byBundle(field, bundle, "UPDATE_APPLIED"),
+      ).resolves.toMatchObject([{ installations: 1 }]);
+    }
+    await expect(
+      outcomes("UPDATE_APPLIED", "to:bundle-2"),
+    ).resolves.toMatchObject([{ bucket_start_ms: hour(T), events: 1 }]);
+    // Every event's row is a UTC day's, not an hour's.
+    await expect(everyEvent()).resolves.toEqual([
+      { ...DAILY_EVENTS, bucket_start_ms: T - (T % DAY), events: 1 },
+    ]);
+
+    const release = { scopeKind: "release", releaseKind: "specific" } as const;
+    const counters = { downloads: 0, launches: 1, failed_launches: 0 };
+    await expect(
+      overview(
+        identity({
+          ...release,
+          releaseId: "release-2",
+          periodKind: "lifetime",
+        }),
+        0,
+      ),
+    ).resolves.toMatchObject(counters);
+    for (const [periodKind, bucket] of [
+      ["hour", hour(T)],
+      ["day", T - (T % DAY)],
+    ] as const) {
+      await expect(
+        overview(identity({ periodKind }), bucket),
+      ).resolves.toMatchObject(counters);
+      const launches = await sketch(identity({ periodKind }), bucket);
+      expect(countInsightsDistinct(launches!.launch_users)).toBe(1);
+      for (const platform of ["ios", "all"] as const) {
+        const usage = await sketch(
+          identity({
+            scopeKind: "usage",
+            platform,
+            appVersionKind: "specific",
+            appVersion: "1.0.0",
+            periodKind,
+          }),
+          bucket,
+        );
+        expect(countInsightsDistinct(usage!.activity_users)).toBe(1);
+      }
+    }
+  });
+
+  it("keys an event by the one bundle its bundle filter reads", async () => {
+    const { api, db } = await setup();
+    await api.recordEvent(event(1));
+    await api.recordEvent(
+      event(2, { type: "RECOVERED", install_id: "install-2" }),
+    );
+
+    await expect(
+      db.findOne("bundle_events", { id: uuid(2) }),
+    ).resolves.toMatchObject({ bundle_ref: ["from:bundle-1"] });
+    const scope = { platform: "ios", channel: "production" } as const;
+    for (const [n, filter] of [
+      [1, { ...scope, type: "UPDATE_APPLIED", toBundleId: "bundle-2" }],
+      [2, { ...scope, type: "RECOVERED", fromBundleId: "bundle-1" }],
+    ] as const) {
+      await expect(
+        api.listEvents({
+          filter: { kind: "bundle", ...filter },
+          sinceMs: T,
+          beforeReceivedAtMs: T + HOUR,
+          limit: 10,
+        }),
+      ).resolves.toMatchObject([{ id: uuid(n) }]);
+    }
+  });
+
+  it("changes nothing for a repeated id, even with another payload", async () => {
+    const { api, db, outcomes, everyEvent } = await setup();
+    await api.recordEvent(event(1));
+    await api.recordEvent(
+      event(1, { install_id: "install-2", received_at_ms: T + HOUR }),
+    );
+
+    await expect(
+      db.findOne("bundle_events", { id: uuid(1) }),
+    ).resolves.toMatchObject({ install_id: "install-1" });
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: "install-2" }),
+    ).resolves.toBeNull();
+    await expect(
+      outcomes("UPDATE_APPLIED", "to:bundle-2"),
+    ).resolves.toMatchObject([{ events: 1 }]);
+    await expect(everyEvent()).resolves.toMatchObject([{ events: 1 }]);
+  });
+
+  it("moves the head only for a newer event and deletes gauge rows that reach zero", async () => {
+    const { api, db, distribution, byBundle, outcomes } = await setup();
+    await api.recordEvent(event(1));
+    await api.recordEvent(
+      event(2, {
+        type: "UPDATE_DOWNLOADED",
+        user_id: null,
+        from_release_id: "release-2",
+        from_bundle_id: "bundle-2",
+        to_release_id: "release-3",
+        to_bundle_id: "bundle-3",
+        received_at_ms: T + 2 * HOUR,
+      }),
+    );
+    await api.recordEvent(event(3, { received_at_ms: T + HOUR }));
+
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: "install-1" }),
+    ).resolves.toMatchObject({
+      id: uuid(2),
+      user_id: null,
+      type: "UPDATE_DOWNLOADED",
+      current_release_id: "release-2",
+    });
+    await expect(distribution()).resolves.toMatchObject([
+      {
+        release_id: "release-2",
+        bucket_start_ms: hour(T + 2 * HOUR),
+        latest_installations: 1,
+      },
+    ]);
+    await expect(
+      byBundle("to_bundle_id", "bundle-2", "UPDATE_APPLIED"),
+    ).resolves.toEqual([]);
+    await expect(
+      byBundle("to_bundle_id", "bundle-3", "UPDATE_DOWNLOADED"),
+    ).resolves.toMatchObject([{ installations: 1 }]);
+    await expect(
+      outcomes("UPDATE_APPLIED", "to:bundle-2"),
+    ).resolves.toMatchObject([
+      { bucket_start_ms: hour(T), events: 1 },
+      { bucket_start_ms: hour(T + HOUR), events: 1 },
+    ]);
+
+    await api.recordEvent(event(4, { received_at_ms: T + 2 * HOUR }));
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: "install-1" }),
+    ).resolves.toMatchObject({ id: uuid(4), type: "UPDATE_APPLIED" });
+    await expect(distribution()).resolves.toMatchObject([
+      {
+        release_id: "release-2",
+        bucket_start_ms: hour(T + 2 * HOUR),
+        latest_installations: 1,
+      },
+    ]);
+  });
+
+  it("reads the event and head in one round, gauge and sketch rows in a second, and writes once", async () => {
+    const calls: string[] = [];
+    const { api, measureReads } = await setup((inner) => ({
+      ...inner,
+      get: async (table, keys) => {
+        calls.push(`get ${table.name}`);
+        return inner.get(table, keys);
+      },
+      write: async (ops: readonly WriteOp[]) => {
+        calls.push("write");
+        return inner.write(ops);
+      },
+    }));
+    const measured = await measureReads(() => api.recordEvent(event(1)));
+
+    expect(measured.adapter.queries).toBe(0);
+    expect(calls).toEqual([
+      "get bundle_events",
+      "get bundle_event_heads",
+      "get insights_sketches",
+      "get insights_distribution",
+      "get insights_latest_by_bundle",
+      "write",
+    ]);
+  });
+
+  it("serializes events per installation and keeps gauges exact under concurrency", async () => {
+    const { api, db, distribution } = await setup();
+    const installs = ["a", "b", "c", "d"];
+    const events = installs.flatMap((install, position) =>
+      [0, 1, 2].map((step) =>
+        event(position * 10 + step + 1, {
+          install_id: `install-${install}`,
+          received_at_ms: T + ((step * 7 + position) % 3) * HOUR,
+        }),
+      ),
+    );
+    await Promise.all(events.map((row) => api.recordEvent(row)));
+
+    for (const install of installs) {
+      const latest = events
+        .filter((row) => row.install_id === `install-${install}`)
+        .toSorted(
+          (left, right) =>
+            left.received_at_ms - right.received_at_ms ||
+            (left.id < right.id ? -1 : 1),
+        )
+        .at(-1)!;
+      await expect(
+        db.findOne("bundle_event_heads", { install_id: `install-${install}` }),
+      ).resolves.toMatchObject({ id: latest.id });
+    }
+    const counted = (await distribution()).reduce(
+      (sum, row) => sum + row.latest_installations,
+      0,
+    );
+    expect(counted).toBe(installs.length);
+  });
+
+  it("records a retried report once, and a racing installation's report under its id not at all", async () => {
+    const { api, db, outcomes } = await setup();
+    await api.recordEvent(event(1));
+    // A retry repeats the report's id, even when it arrives an hour later.
+    await api.recordEvent(event(1, { received_at_ms: T + HOUR }));
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: "install-1" }),
+    ).resolves.toMatchObject({ id: uuid(1), received_at_ms: T });
+    await expect(
+      outcomes("UPDATE_APPLIED", "to:bundle-2"),
+    ).resolves.toMatchObject([{ events: 1 }]);
+
+    // The later writer reads the winner's row, at once or in the rerun after
+    // its own insert fails, and changes nothing.
+    const installs = ["install-2", "install-3"];
+    await Promise.all(
+      installs.map((install) =>
+        api.recordEvent(event(2, { install_id: install })),
+      ),
+    );
+    const stored = await db.findOne("bundle_events", { id: uuid(2) });
+    const loser = installs.find((install) => install !== stored?.install_id);
+    expect(installs).toContain(stored?.install_id);
+    await expect(
+      db.findOne("bundle_event_heads", { install_id: loser! }),
+    ).resolves.toBeNull();
+  });
+});

package/src/plugins/insights/recordEvent.ts

@@ -0,0 +1,238 @@
+import type { BundleEventRow } from "@hot-updater/plugin-core";
+import {
+  addInsightsDistinct,
+  assertBundleEventRow,
+  compareUtf8,
+  currentInsightsReleaseId,
+  insightsKey,
+  insightsOverviewDeltas,
+  insightsOverviewId,
+  type InsightsOverviewIdentity,
+} from "@hot-updater/plugin-core/internal";
+
+import type {
+  HotUpdaterDatabase,
+  HotUpdaterTransaction,
+} from "../../database/database";
+import { DAILY_EVENTS, DAY_MS, HOUR_MS, type InsightsSchema } from "./schema";
+
+/** The head columns `countHead` reads. */
+interface Head {
+  readonly install_id: string;
+  readonly id: string;
+  readonly received_at_ms: number;
+  readonly platform: string;
+  readonly channel: string;
+  readonly type: string;
+  readonly from_bundle_id: string | null;
+  readonly to_bundle_id: string;
+  readonly current_release_id: string | null;
+  readonly app_version: string;
+}
+
+const hourOf = (ms: number) => ms - (ms % HOUR_MS);
+const dayOf = (ms: number) => ms - (ms % DAY_MS);
+
+/** An overview or sketch row's scope and period; day periods roll up channel and usage rows. */
+export type InsightsIdentityParts = Omit<
+  InsightsOverviewIdentity,
+  "bucketStartMs" | "periodKind"
+> & { readonly periodKind: InsightsOverviewIdentity["periodKind"] | "day" };
+
+/** The `identity` of an overview or sketch row: its scope and period, hashed. */
+export const insightsIdentity = (identity: InsightsIdentityParts): string =>
+  insightsOverviewId({
+    ...identity,
+    periodKind: identity.periodKind as InsightsOverviewIdentity["periodKind"],
+    bucketStartMs: 0,
+  });
+
+/** The head columns besides its key: the whole event and its current release. */
+const headFields = (event: BundleEventRow) => {
+  const { install_id: _, ...fields } = event;
+  return { ...fields, current_release_id: currentInsightsReleaseId(event) };
+};
+
+/**
+ * The `insights_latest_by_bundle` field of a head's (from, to) pair. A head
+ * matches a `from` and a `to` predicate of one type at once only through its
+ * pair, so this gauge is what a count subtracts to count it once.
+ */
+export const PAIR_FIELD = "from_to";
+
+/** The pair gauge's `bundle_id`: a hash of the two bundle ids, which fit no single 36-character column. */
+export const bundlePairKey = (from: string, to: string): string =>
+  insightsKey(`${from.length}:${from}${to.length}:${to}`);
+
+/** Moves a head's gauges: the distribution row, one row per bundle it references, and its pair. */
+const countHead = (
+  tx: HotUpdaterTransaction<InsightsSchema>,
+  head: Head,
+  delta: 1 | -1,
+) => {
+  const bucket = hourOf(head.received_at_ms);
+  const shardBy = head.install_id;
+  tx.aggregate(
+    "insights_distribution",
+    {
+      channel: head.channel,
+      platform: head.platform,
+      app_version: head.app_version,
+      release_id: head.current_release_id ?? "",
+      bucket_start_ms: bucket,
+    },
+    { latest_installations: delta },
+    { shardBy },
+  );
+  for (const field of ["from_bundle_id", "to_bundle_id"] as const) {
+    const bundleId = head[field];
+    if (bundleId === null) continue;
+    tx.aggregate(
+      "insights_latest_by_bundle",
+      {
+        platform: head.platform,
+        channel: head.channel,
+        bundle_field: field,
+        bundle_id: bundleId,
+        type: head.type,
+        bucket_start_ms: bucket,
+      },
+      { installations: delta },
+      { shardBy },
+    );
+  }
+  if (head.from_bundle_id !== null) {
+    tx.aggregate(
+      "insights_latest_by_bundle",
+      {
+        platform: head.platform,
+        channel: head.channel,
+        bundle_field: PAIR_FIELD,
+        bundle_id: bundlePairKey(head.from_bundle_id, head.to_bundle_id),
+        type: head.type,
+        bucket_start_ms: bucket,
+      },
+      { installations: delta },
+      { shardBy },
+    );
+  }
+};
+
+/**
+ * Counters and sketches for one event: release, channel, and usage rows, with
+ * day rollups for channel and usage; and its outcome rows, its bundle
+ * filter's hour and every event's day.
+ */
+const countEvent = (
+  tx: HotUpdaterTransaction<InsightsSchema>,
+  event: BundleEventRow,
+) => {
+  const shardBy = event.install_id;
+  for (const delta of insightsOverviewDeltas(event)) {
+    const { bucketStartMs, ...parts } = delta.identity;
+    const periods =
+      parts.scopeKind === "channel" || parts.scopeKind === "usage"
+        ? ([
+            ["hour", bucketStartMs],
+            ["day", dayOf(event.received_at_ms)],
+          ] as const)
+        : ([[parts.periodKind, bucketStartMs]] as const);
+    for (const [periodKind, bucket] of periods) {
+      const key = {
+        identity: insightsIdentity({ ...parts, periodKind }),
+        bucket_start_ms: bucket,
+      };
+      const counters: {
+        downloads?: number;
+        launches?: number;
+        failed_launches?: number;
+      } = Object.fromEntries(
+        [
+          ["downloads", delta.downloads],
+          ["launches", delta.launches],
+          ["failed_launches", delta.failedLaunches],
+        ].filter(([, value]) => value !== 0),
+      );
+      if (Object.keys(counters).length > 0) {
+        tx.aggregate("insights_overview", key, counters, { shardBy });
+      }
+      const sketches = {
+        ...(delta.launchIdentity === undefined
+          ? {}
+          : { launch_users: addInsightsDistinct(null, delta.launchIdentity) }),
+        ...(delta.activityIdentity === undefined
+          ? {}
+          : {
+              activity_users: addInsightsDistinct(null, delta.activityIdentity),
+            }),
+      };
+      if (Object.keys(sketches).length > 0) {
+        tx.aggregate("insights_sketches", key, sketches, { shardBy });
+      }
+    }
+  }
+  tx.aggregate(
+    "insights_outcomes",
+    {
+      platform: event.platform,
+      channel: event.channel,
+      type: event.type,
+      bundle_ref:
+        event.type === "RECOVERED"
+          ? `from:${event.from_bundle_id}`
+          : `to:${event.to_bundle_id}`,
+      bucket_start_ms: hourOf(event.received_at_ms),
+    },
+    { events: 1 },
+    { shardBy },
+  );
+  // The global event list reads only the days this row counts: a gap costs
+  // it one empty day and one read of this row, not a read a day. One more
+  // blind increment per event, on the event's shard like the others.
+  tx.aggregate(
+    "insights_outcomes",
+    { ...DAILY_EVENTS, bucket_start_ms: dayOf(event.received_at_ms) },
+    { events: 1 },
+    { shardBy },
+  );
+};
+
+const isNewer = (event: Head, head: Head) =>
+  event.received_at_ms !== head.received_at_ms
+    ? event.received_at_ms > head.received_at_ms
+    : compareUtf8(event.id, head.id) > 0;
+
+/**
+ * Records one event in one transaction: one batch read of the event and its
+ * installation's head, one of the gauge and sketch rows it changes, then one
+ * write. A repeated id changes nothing, whichever installation sends it; an
+ * older event still counts in its own hour but never replaces the head.
+ */
+export const recordEvent = (
+  db: HotUpdaterDatabase<InsightsSchema>,
+  event: BundleEventRow,
+): Promise<void> => {
+  assertBundleEventRow(event);
+  return db.transaction(async (tx) => {
+    const [existing, previous] = await Promise.all([
+      tx.findOne("bundle_events", { id: event.id }),
+      tx.findOne("bundle_event_heads", { install_id: event.install_id }),
+    ]);
+    // The id is the report's idempotency key: a retry, or any report under
+    // an id already stored, changes nothing, as analytics ingestion drops
+    // duplicates.
+    if (existing !== null) return;
+    tx.create("bundle_events", event);
+    countEvent(tx, event);
+    const fields = headFields(event);
+    const head = { ...fields, install_id: event.install_id };
+    if (previous !== null && !isNewer(head, previous)) return;
+    if (previous === null) {
+      tx.create("bundle_event_heads", head);
+    } else {
+      countHead(tx, previous, -1);
+      tx.update("bundle_event_heads", previous, fields);
+    }
+    countHead(tx, head, 1);
+  });
+};