@hot-updater/server 1.0.0-rc.18 → 1.0.0-rc.21

package/src/plugins/insights/routes.ts

@@ -1,149 +0,0 @@
-import { isDatabaseBusyError } from "../../database/busy";
-import {
-  InsightsBadRequestError,
-  InsightsPayloadTooLargeError,
-} from "./errors";
-import { parseBundleEventRequest } from "./eventInput";
-import {
-  parseReportingOverviewInput,
-  parseEventPageInput,
-  parseUserInstallationPageInput,
-} from "./queryInput";
-import type { InsightsProvider } from "./types";
-
-/** Seconds a client waits before it resends a request the busy database refused. */
-const RETRY_AFTER_SECONDS = 5;
-
-const json = (
-  body: unknown,
-  status: number,
-  headers: Readonly<Record<string, string>> = {},
-): Response =>
-  Response.json(body, {
-    headers: { "cache-control": "private, no-store", ...headers },
-    status,
-  });
-
-const requireParam = (
-  params: Readonly<Record<string, string>>,
-  key: string,
-): string => {
-  const value = params[key];
-  if (value === undefined || value.length === 0) {
-    throw new InsightsBadRequestError(`Missing route parameter: ${key}`);
-  }
-  try {
-    return decodeURIComponent(value);
-  } catch {
-    throw new InsightsBadRequestError(`Invalid route parameter: ${key}`);
-  }
-};
-
-const run = async (operation: () => Promise<Response>): Promise<Response> => {
-  try {
-    return await operation();
-  } catch (error) {
-    if (error instanceof InsightsBadRequestError) {
-      return json({ error: error.message }, 400);
-    }
-    if (error instanceof InsightsPayloadTooLargeError) {
-      return json({ error: error.message }, 413);
-    }
-    // Back-pressure: a busy database has not failed the request, so the
-    // client may send it again after Retry-After. A 500 would read as a fault.
-    // The warning keeps the overload visible to whoever runs the server.
-    if (isDatabaseBusyError(error)) {
-      console.warn(
-        "[hot-updater] Insights answered 503: the database is busy.",
-        error,
-      );
-      return json({ error: "Service unavailable" }, 503, {
-        "retry-after": String(RETRY_AFTER_SECONDS),
-      });
-    }
-    throw error;
-  }
-};
-
-const query = (operation: () => Promise<unknown>): Promise<Response> =>
-  run(async () => json(await operation(), 200));
-
-/** Every Insights route: where it mounts, and the handler it runs. */
-export const INSIGHTS_ROUTES = [
-  {
-    access: "client",
-    method: "POST",
-    path: "/events",
-    handler: "appendBundleEvent",
-  },
-  { access: "admin", method: "GET", path: "/events", handler: "listEvents" },
-  {
-    access: "admin",
-    method: "GET",
-    path: "/overview",
-    handler: "getReportingOverview",
-  },
-  {
-    access: "admin",
-    method: "GET",
-    path: "/installations",
-    handler: "pageInstallationsByCurrentUserId",
-  },
-  {
-    access: "admin",
-    method: "GET",
-    path: "/installations/:installId/events",
-    handler: "listInstallationEvents",
-  },
-  {
-    access: "admin",
-    method: "GET",
-    path: "/installations/:installId",
-    handler: "getInstallation",
-  },
-] as const;
-
-export type InsightsRouteName = (typeof INSIGHTS_ROUTES)[number]["handler"];
-
-export type InsightsRouteHandler = (
-  params: Readonly<Record<string, string>>,
-  request: Request,
-) => Promise<Response>;
-
-export const createInsightsRouteHandlers = (
-  provider: InsightsProvider,
-): Record<InsightsRouteName, InsightsRouteHandler> => ({
-  appendBundleEvent: async (_params, request) =>
-    run(async () => {
-      await provider.appendBundleEvent(await parseBundleEventRequest(request));
-      return new Response(null, { status: 204 });
-    }),
-  getReportingOverview: (_params, request) =>
-    query(() =>
-      provider.getReportingOverview(parseReportingOverviewInput(request)),
-    ),
-  getInstallation: (params) =>
-    run(async () => {
-      const installation = await provider.getInstallation({
-        installId: requireParam(params, "installId"),
-      });
-      return installation === null
-        ? json({ error: "Installation not found" }, 404)
-        : json(installation, 200);
-    }),
-  listEvents: (_params, request) =>
-    query(() => provider.listEvents(parseEventPageInput(request))),
-  listInstallationEvents: (params, request) =>
-    query(() =>
-      provider.listInstallationEvents({
-        ...parseEventPageInput(request),
-        installId: requireParam(params, "installId"),
-      }),
-    ),
-  pageInstallationsByCurrentUserId: (_params, request) =>
-    query(() =>
-      provider.pageInstallationsByCurrentUserId(
-        parseUserInstallationPageInput(request),
-      ),
-    ),
-});

package/src/plugins/insights/schema.ts

@@ -1,409 +0,0 @@
-import { defineAggregate, defineTable } from "../../database/schema";
-
-export const HOUR_MS = 3_600_000;
-export const DAY_MS = 86_400_000;
-
-/** How long Insights keeps its rows, in whole days. Lifetime counters are kept. */
-export interface InsightsRetention {
-  /** Raw events and hourly rollups. */
-  readonly rawDays: number;
-  /** Daily rollups, and each installation's latest event and the gauges that count it. */
-  readonly dailyDays: number;
-}
-
-/**
- * The default periods (decision 59): 90 days, and 13 months as 400 days, so
- * the same month a year ago stays whole.
- */
-export const RAW_RETENTION_DAYS = 90;
-export const DAILY_RETENTION_DAYS = 400;
-export const DEFAULT_INSIGHTS_RETENTION: InsightsRetention = {
-  rawDays: RAW_RETENTION_DAYS,
-  dailyDays: DAILY_RETENTION_DAYS,
-};
-
-/** Rows keyed by a bucket that expire `days` after it starts. */
-interface BucketRetention {
-  readonly field: "bucket_start_ms";
-  readonly days: number;
-}
-const bucketRetention = (days: number): BucketRetention => ({
-  field: "bucket_start_ms",
-  days,
-});
-
-const MOVEMENTS = new Set([
-  "UPDATE_DOWNLOADED",
-  "UPDATE_APPLIED",
-  "RECOVERED",
-  "UPDATE_FAILED",
-]);
-
-/** A failed update check: it names no target bundle, so no bundle list shows it. */
-export const isFailedCheck = (row: {
-  readonly type: string;
-  readonly metadata: unknown;
-}): boolean =>
-  row.type === "UPDATE_FAILED" &&
-  (row.metadata as { failure?: { stage?: unknown } } | null)?.failure?.stage ===
-    "check";
-
-/**
- * Downloads, applies, recoveries, and update failures, whole. An UNCHANGED
- * report counts as a launch and moves its installation's head, but no list
- * shows it, so no row keeps it.
- */
-const bundleEvents = (days: number) =>
-  defineTable(
-    {
-      id: { type: "string", maxLength: 36 },
-      type: { type: "string", maxLength: 32 },
-      install_id: { type: "string", maxLength: 255 },
-      user_id: { type: "string", maxLength: 255, required: false },
-      from_release_id: { type: "string", maxLength: 36, required: false },
-      from_bundle_id: { type: "string", maxLength: 36, required: false },
-      to_release_id: { type: "string", maxLength: 36, required: false },
-      to_bundle_id: { type: "string", maxLength: 36 },
-      platform: { type: "string", maxLength: 16 },
-      app_version: { type: "string" },
-      channel: { type: "string" },
-      metadata: { type: "json" },
-      received_at_ms: { type: "integer" },
-    },
-    {
-      key: ["id"],
-      derived: {
-        /** The UTC day the event was received in. */
-        day: {
-          type: "integer",
-          compute: (row) => row.received_at_ms - (row.received_at_ms % DAY_MS),
-        },
-        /** Set only for events that belong in installation history. */
-        movement_install_id: {
-          type: "string",
-          compute: (row) => (MOVEMENTS.has(row.type) ? row.install_id : null),
-        },
-        /**
-         * The bundle a bundle filter matches: `from:<bundle>` for RECOVERED,
-         * none for a failed check, else `to:<bundle>`.
-         */
-        bundle_ref: {
-          type: "string",
-          multi: true,
-          compute: (row) =>
-            isFailedCheck(row)
-              ? []
-              : [
-                  row.type === "RECOVERED" && row.from_bundle_id !== null
-                    ? `from:${row.from_bundle_id}`
-                    : `to:${row.to_bundle_id}`,
-                ],
-        },
-      },
-      indexes: {
-        recent: {
-          eq: ["channel", "platform", "day"],
-          sort: ["received_at_ms"],
-        },
-        movementsByInstall: {
-          eq: ["movement_install_id"],
-          sort: ["received_at_ms"],
-        },
-        byBundle: {
-          eq: ["platform", "channel", "type", "bundle_ref", "day"],
-          sort: ["received_at_ms"],
-        },
-        /** Events of every scope by day, for the console's unfiltered event list. */
-        byDay: { eq: ["day"], sort: ["received_at_ms"] },
-      },
-      // A day's events go together, oldest days first, walking `byDay`.
-      retention: { field: "day", days },
-    },
-  );
-
-/**
- * Each installation's latest event, whole, so reading it is one point read.
- * Its gauges count it in the UTC day of that event. An update failure moves
- * nothing, so no head holds one.
- */
-const bundleEventHeads = (days: number) =>
-  defineTable(
-    {
-      install_id: { type: "string", maxLength: 255 },
-      id: { type: "string", maxLength: 36 },
-      type: { type: "string", maxLength: 32 },
-      user_id: { type: "string", maxLength: 255, required: false },
-      from_release_id: { type: "string", maxLength: 36, required: false },
-      from_bundle_id: { type: "string", maxLength: 36, required: false },
-      to_release_id: { type: "string", maxLength: 36, required: false },
-      to_bundle_id: { type: "string", maxLength: 36 },
-      platform: { type: "string", maxLength: 16 },
-      app_version: { type: "string" },
-      channel: { type: "string" },
-      metadata: { type: "json" },
-      received_at_ms: { type: "integer" },
-    },
-    {
-      key: ["install_id"],
-      indexes: { byUser: { eq: ["user_id"], sort: ["install_id"] } },
-      // An installation unseen for the daily period is gone, with its gauges' day.
-      retention: { field: "received_at_ms", days },
-    },
-  );
-
-/**
- * Shards are keyed by install id. Counters are blind increments and never
- * conflict, so they keep 8. Gauges and sketches are read, merged, and written
- * back. B3's rollout gate on PostgreSQL retried 2.4% of transactions at 8
- * shards and 1.3% at 16. DynamoDB's transactions take longer, so D8's gate
- * on DynamoDB Local retried 2–24% at 16, almost all on gauge rows, and 1–5%
- * with gauges at 32. Sketches stay at 16: each shard row carries 2 KB of
- * registers that every read merges.
- *
- * Every Insights aggregate is `batched`: on DynamoDB and Firestore, which
- * bill each write, an event's changes apply after it commits, merged with
- * other events' into one write per row, on one shard per writer.
- */
-const COUNTER_SHARDS = 8;
-const GAUGE_SHARDS = 32;
-const SKETCH_SHARDS = 16;
-
-/** Rows keyed by a hashed identity (scope and period) and a bucket. */
-const identityFields = {
-  identity: { type: "string", maxLength: 32 },
-  bucket_start_ms: { type: "integer" },
-} as const;
-const window = { eq: ["identity"], sort: ["bucket_start_ms"] } as const;
-
-/**
- * Counters of one period kind: hourly, daily, or lifetime at bucket 0. A
- * release or channel row counts downloads, launches, and failed launches,
- * and downloads a patch delivered and those that fell back from one. A
- * release's lifetime row counts its failed updates (`failed_updates`), since
- * no breakdown row outlives the raw period; windowed reads sum
- * `insights_failures` instead.
- */
-const counters = (retention?: BucketRetention) =>
-  defineAggregate(identityFields, {
-    key: ["identity", "bucket_start_ms"],
-    counters: [
-      "downloads",
-      "launches",
-      "failed_launches",
-      "failed_updates",
-      "patch_downloads",
-      "patch_fallbacks",
-    ],
-    shards: COUNTER_SHARDS,
-    batched: true,
-    indexes: { window },
-    ...(retention === undefined ? {} : { retention }),
-  });
-
-/**
- * Unique-installation sketches of one period kind. Rows of a failure
- * identity (`failure` and `check` scopes) keep only `failed_users`, so a
- * release's launch rows never grow by a failure's registers.
- */
-const sketches = (retention: BucketRetention) =>
-  defineAggregate(identityFields, {
-    key: ["identity", "bucket_start_ms"],
-    distinct: ["launch_users", "activity_users", "failed_users"],
-    shards: SKETCH_SHARDS,
-    batched: true,
-    indexes: { window },
-    retention,
-  });
-
-/** Installations whose update to a release failed, since its first failure: kept, at bucket 0. */
-const lifetimeSketches = () =>
-  defineAggregate(identityFields, {
-    key: ["identity", "bucket_start_ms"],
-    distinct: ["failed_users"],
-    shards: SKETCH_SHARDS,
-    batched: true,
-    indexes: { window },
-  });
-
-/**
- * Update failures by what failed, and recoveries by why the crashed process
- * exited, a row an hour. `release_id` is the failed update's target release,
- * or empty for a failed check or an unknown target, and the recovered
- * release for a recovery. A failure's `stage` and `reason` say where and why
- * it failed, and `detail` what else the client knew, as the JSON array
- * `[resource, httpStatus, originCode, transport]`. A recovery's stage is
- * `launch` and its reason the exit reason, with an empty detail. Every index
- * is in key order, so its counters stay blind increments.
- */
-const insightsFailures = (retention: BucketRetention) =>
-  defineAggregate(
-    {
-      platform: { type: "string", maxLength: 16 },
-      channel: { type: "string" },
-      bucket_start_ms: { type: "integer" },
-      release_id: { type: "string", maxLength: 36 },
-      stage: { type: "string", maxLength: 16 },
-      reason: { type: "string", maxLength: 64 },
-      detail: { type: "string", maxLength: 160 },
-    },
-    {
-      key: [
-        "platform",
-        "channel",
-        "bucket_start_ms",
-        "release_id",
-        "stage",
-        "reason",
-        "detail",
-      ],
-      counters: ["events"],
-      shards: COUNTER_SHARDS,
-      batched: true,
-      indexes: {
-        byScope: { eq: ["platform", "channel"], sort: ["bucket_start_ms"] },
-      },
-      retention,
-    },
-  );
-
-/** Installations counted in the UTC day of their latest event, so reads sum whole UTC days. */
-const insightsDistribution = (retention: BucketRetention) =>
-  defineAggregate(
-    {
-      channel: { type: "string" },
-      platform: { type: "string", maxLength: 16 },
-      app_version: { type: "string" },
-      release_id: { type: "string", maxLength: 36 },
-      bucket_start_ms: { type: "integer" },
-    },
-    {
-      key: [
-        "channel",
-        "platform",
-        "app_version",
-        "release_id",
-        "bucket_start_ms",
-      ],
-      gauges: ["latest_installations"],
-      shards: GAUGE_SHARDS,
-      batched: true,
-      indexes: {
-        byScope: { eq: ["channel", "platform"], sort: ["bucket_start_ms"] },
-        byVersion: {
-          eq: ["channel", "platform", "app_version"],
-          sort: ["bucket_start_ms"],
-        },
-      },
-      retention,
-    },
-  );
-
-/** Latest events by the bundle they came from or went to, bucketed like the distribution. */
-const insightsLatestByBundle = (retention: BucketRetention) =>
-  defineAggregate(
-    {
-      platform: { type: "string", maxLength: 16 },
-      channel: { type: "string" },
-      bundle_field: { type: "string", maxLength: 16 },
-      bundle_id: { type: "string", maxLength: 36 },
-      type: { type: "string", maxLength: 32 },
-      bucket_start_ms: { type: "integer" },
-    },
-    {
-      key: [
-        "platform",
-        "channel",
-        "bundle_field",
-        "bundle_id",
-        "type",
-        "bucket_start_ms",
-      ],
-      gauges: ["installations"],
-      shards: GAUGE_SHARDS,
-      batched: true,
-      indexes: {
-        byBundle: {
-          eq: ["platform", "channel", "bundle_field", "bundle_id", "type"],
-          sort: ["bucket_start_ms"],
-        },
-      },
-      retention,
-    },
-  );
-
-/**
- * Stored events by outcome and the bundle a bundle filter matches, a row an
- * hour, which a failed check has none of; and every stored event, a row a UTC
- * day (`DAILY_EVENTS`). Bundle counts sum the hourly rows, and event lists
- * read either to skip days without events.
- */
-const insightsOutcomes = (retention: BucketRetention) =>
-  defineAggregate(
-    {
-      platform: { type: "string", maxLength: 16 },
-      channel: { type: "string" },
-      type: { type: "string", maxLength: 32 },
-      bundle_ref: { type: "string", maxLength: 41 },
-      bucket_start_ms: { type: "integer" },
-    },
-    {
-      key: ["platform", "channel", "type", "bundle_ref", "bucket_start_ms"],
-      counters: ["events"],
-      shards: COUNTER_SHARDS,
-      batched: true,
-      indexes: {
-        byRef: {
-          eq: ["platform", "channel", "type", "bundle_ref"],
-          sort: ["bucket_start_ms"],
-        },
-      },
-      // The per-day rows too: they count raw events, which keep the raw period.
-      retention,
-    },
-  );
-
-/**
- * The `insights_outcomes` identity that counts every stored event, for the
- * global event list. Its bucket is a UTC day, not an hour. Every event's
- * platform is ios or android, so no outcome row of a bundle filter shares it.
- */
-export const DAILY_EVENTS = {
-  platform: "*",
-  channel: "",
-  type: "",
-  bundle_ref: "",
-} as const;
-
-/**
- * Insights' tables, each expiring after its period. The periods are values
- * the engine applies at runtime; the tables and indexes are the same for
- * any periods.
- */
-export const createInsightsSchema = ({
-  rawDays,
-  dailyDays,
-}: InsightsRetention = DEFAULT_INSIGHTS_RETENTION) => {
-  const hourly = bucketRetention(rawDays);
-  const daily = bucketRetention(dailyDays);
-  return {
-    bundle_events: bundleEvents(rawDays),
-    bundle_event_heads: bundleEventHeads(dailyDays),
-    /** Hourly counters and sketches: the raw period. */
-    insights_overview: counters(hourly),
-    insights_sketches: sketches(hourly),
-    /** Daily counters and sketches: the daily period. */
-    insights_overview_daily: counters(daily),
-    insights_sketches_daily: sketches(daily),
-    /** A release's counters since its first event, and its failed installations: kept. */
-    insights_overview_lifetime: counters(),
-    insights_sketches_lifetime: lifetimeSketches(),
-    insights_distribution: insightsDistribution(daily),
-    insights_latest_by_bundle: insightsLatestByBundle(daily),
-    insights_outcomes: insightsOutcomes(hourly),
-    insights_failures: insightsFailures(hourly),
-  } as const;
-};
-
-export const insightsSchema = createInsightsSchema();
-
-export type InsightsSchema = typeof insightsSchema;

package/src/plugins/insights/testing/expectInsightsIndex.spec.ts

@@ -1,42 +0,0 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
-
-import { expectInsightsIndex } from "./expectInsightsIndex";
-
-afterEach(() => vi.useRealTimers());
-
-describe("Insights index assertions", () => {
-  it("waits for acknowledged writes to become visible in a lagging index", async () => {
-    vi.useFakeTimers();
-    const query = vi
-      .fn()
-      .mockResolvedValueOnce([])
-      .mockResolvedValue(["report"]);
-    const result = expectInsightsIndex(query, ["report"]);
-    await vi.advanceTimersByTimeAsync(50);
-    await result;
-    expect(query).toHaveBeenCalledTimes(2);
-  });
-
-  it("propagates a native query failure even after an earlier incomplete result", async () => {
-    vi.useFakeTimers();
-    const failure = new Error("native index unavailable");
-    const query = vi.fn().mockResolvedValueOnce([]).mockRejectedValue(failure);
-    const result = expect(expectInsightsIndex(query, ["report"])).rejects.toBe(
-      failure,
-    );
-    await vi.advanceTimersByTimeAsync(50);
-    await result;
-    expect(query).toHaveBeenCalledTimes(2);
-  });
-
-  it("fails with the actual result when the bounded catch-up interval expires", async () => {
-    vi.useFakeTimers();
-    const query = vi.fn().mockResolvedValue([]);
-    const result = expect(
-      expectInsightsIndex(query, ["report"]),
-    ).rejects.toThrow();
-    await vi.advanceTimersByTimeAsync(5_000);
-    await result;
-    expect(query).toHaveBeenCalledTimes(101);
-  });
-});

package/src/plugins/insights/testing/expectInsightsIndex.ts

@@ -1,20 +0,0 @@
-import { expect } from "vitest";
-
-/** Allow committed secondary indexes to catch up; query failures still fail. */
-export const expectInsightsIndex = async <T>(
-  query: () => Promise<T>,
-  expected: T,
-): Promise<void> => {
-  const deadline = Date.now() + 5_000;
-  for (;;) {
-    // Keep native errors outside the assertion retry handler.
-    const actual = await query();
-    try {
-      expect(actual).toEqual(expected);
-      return;
-    } catch (error) {
-      if (Date.now() >= deadline) throw error;
-    }
-    await new Promise((resolve) => setTimeout(resolve, 50));
-  }
-};

package/src/plugins/insights/testing/fixtures.ts

@@ -1,30 +0,0 @@
-import type { BundleEventRow } from "../eventRow";
-
-const fixtureId = (suffix: string): string =>
-  `00000000-0000-7000-8000-${suffix.padStart(12, "0")}`;
-
-/** An applied update from install `suffix`, received at `receivedAtMs`. */
-export const createBundleEventRowFixture = (
-  suffix: string,
-  receivedAtMs: number,
-): Extract<BundleEventRow, { type: "UPDATE_APPLIED" | "RECOVERED" }> => ({
-  id: fixtureId(suffix),
-  type: "UPDATE_APPLIED",
-  install_id: `install-${suffix}`,
-  user_id: null,
-  metadata: {
-    cohort: "0",
-    update_strategy: "appVersion",
-    fingerprint_hash: null,
-    sdk_version: null,
-  },
-  from_bundle_id: fixtureId(`${Number(suffix) + 1000}`),
-  from_release_id: null,
-  to_bundle_id: fixtureId(`${Number(suffix) + 2000}`),
-  to_release_id: null,
-  platform: "ios",
-  app_version: "1.0.0",
-  channel: "production",
-
-  received_at_ms: receivedAtMs,
-});

package/src/plugins/insights/testing/index.ts

@@ -1,45 +0,0 @@
-import { describe } from "vitest";
-
-import type { InsightsModel } from "../modelTypes";
-import { registerInsightsModelTests } from "./insightsModelTests";
-import { setupInsightsHttpTestSuite } from "./setupInsightsHttpTestSuite";
-import type { HttpTestClient } from "./types";
-
-export { createBundleEventRowFixture } from "./fixtures";
-export { setupInsightsHttpTestSuite } from "./setupInsightsHttpTestSuite";
-export { setupInsightsModelTestSuite } from "./setupInsightsModelTestSuite";
-export type {
-  DatabaseTestLifecycle,
-  DatabaseTestState,
-  HttpTestClient,
-  HttpTestRequest,
-} from "./types";
-
-/**
- * The Insights plugin's tests, for `setupDatabaseTestSuite`'s `plugins`: its
- * HTTP routes on the provider's server, which must run `insights()`, and
- * with `createModel` its report contract on the provider's database, such as
- * `(database) => createInsightsModel(createDatabasePluginApis(database,
- * [insights()]).insights)`.
- */
-export const insightsTestSuite = <TDatabase = unknown>(
-  options: {
-    readonly createModel?: (database: TDatabase) => InsightsModel;
-  } = {},
-) => ({
-  name: "Insights plugin",
-  register: (context: {
-    readonly getClient: () => HttpTestClient;
-    readonly getDatabase: () => TDatabase;
-  }): void => {
-    setupInsightsHttpTestSuite({ getClient: context.getClient });
-    const { createModel } = options;
-    if (createModel !== undefined) {
-      describe("Insights model", () => {
-        registerInsightsModelTests({
-          getDatabase: () => createModel(context.getDatabase()),
-        });
-      });
-    }
-  },
-});