@hot-updater/server 1.0.0-rc.1 → 1.0.0-rc.16

package/src/plugins/insights/schema.ts

@@ -0,0 +1,229 @@
+import { defineAggregate, defineTable } from "../../database/schema";
+
+export const HOUR_MS = 3_600_000;
+export const DAY_MS = 86_400_000;
+
+const MOVEMENTS = new Set(["UPDATE_DOWNLOADED", "UPDATE_APPLIED", "RECOVERED"]);
+
+const bundleEvents = defineTable(
+  {
+    id: { type: "string", maxLength: 36 },
+    type: { type: "string", maxLength: 32 },
+    install_id: { type: "string", maxLength: 255 },
+    user_id: { type: "string", maxLength: 255, required: false },
+    from_release_id: { type: "string", maxLength: 36, required: false },
+    from_bundle_id: { type: "string", maxLength: 36, required: false },
+    to_release_id: { type: "string", maxLength: 36, required: false },
+    to_bundle_id: { type: "string", maxLength: 36 },
+    platform: { type: "string", maxLength: 16 },
+    app_version: { type: "string" },
+    channel: { type: "string" },
+    metadata: { type: "json" },
+    received_at_ms: { type: "integer" },
+  },
+  {
+    key: ["id"],
+    derived: {
+      /** The UTC day the event was received in. */
+      day: {
+        type: "integer",
+        compute: (row) => row.received_at_ms - (row.received_at_ms % DAY_MS),
+      },
+      /** Set only for events that belong in installation history. */
+      movement_install_id: {
+        type: "string",
+        compute: (row) => (MOVEMENTS.has(row.type) ? row.install_id : null),
+      },
+      /** The bundle a bundle filter matches: `from:<bundle>` for RECOVERED, else `to:<bundle>`. */
+      bundle_ref: {
+        type: "string",
+        multi: true,
+        compute: (row) => [
+          row.type === "RECOVERED" && row.from_bundle_id !== null
+            ? `from:${row.from_bundle_id}`
+            : `to:${row.to_bundle_id}`,
+        ],
+      },
+    },
+    indexes: {
+      recent: { eq: ["channel", "platform", "day"], sort: ["received_at_ms"] },
+      movementsByInstall: {
+        eq: ["movement_install_id"],
+        sort: ["received_at_ms"],
+      },
+      byBundle: {
+        eq: ["platform", "channel", "type", "bundle_ref", "day"],
+        sort: ["received_at_ms"],
+      },
+      /** Events of every scope by day, for the console's unfiltered event list. */
+      byDay: { eq: ["day"], sort: ["received_at_ms"] },
+    },
+  },
+);
+
+/** Each installation's latest event, whole, so reading it is one point read. */
+const bundleEventHeads = defineTable(
+  {
+    install_id: { type: "string", maxLength: 255 },
+    id: { type: "string", maxLength: 36 },
+    type: { type: "string", maxLength: 32 },
+    user_id: { type: "string", maxLength: 255, required: false },
+    from_release_id: { type: "string", maxLength: 36, required: false },
+    from_bundle_id: { type: "string", maxLength: 36, required: false },
+    to_release_id: { type: "string", maxLength: 36, required: false },
+    to_bundle_id: { type: "string", maxLength: 36 },
+    platform: { type: "string", maxLength: 16 },
+    app_version: { type: "string" },
+    channel: { type: "string" },
+    metadata: { type: "json" },
+    received_at_ms: { type: "integer" },
+    current_release_id: { type: "string", maxLength: 36, required: false },
+  },
+  {
+    key: ["install_id"],
+    indexes: { byUser: { eq: ["user_id"], sort: ["install_id"] } },
+  },
+);
+
+/**
+ * Shards are keyed by install id. Counters are blind increments and never
+ * conflict, so they keep 8. Gauges and sketches are read, merged, and written
+ * back. B3's rollout gate on PostgreSQL retried 2.4% of transactions at 8
+ * shards and 1.3% at 16. DynamoDB's transactions take longer, so D8's gate
+ * on DynamoDB Local retried 2–24% at 16, almost all on gauge rows, and 1–5%
+ * with gauges at 32. Sketches stay at 16: each shard row carries 2 KB of
+ * registers that every read merges.
+ */
+const COUNTER_SHARDS = 8;
+const GAUGE_SHARDS = 32;
+const SKETCH_SHARDS = 16;
+
+/** Rows keyed by a hashed identity (scope and period) and a bucket. */
+const identityFields = {
+  identity: { type: "string", maxLength: 32 },
+  bucket_start_ms: { type: "integer" },
+} as const;
+const window = { eq: ["identity"], sort: ["bucket_start_ms"] } as const;
+
+const insightsOverview = defineAggregate(identityFields, {
+  key: ["identity", "bucket_start_ms"],
+  counters: ["downloads", "launches", "failed_launches"],
+  shards: COUNTER_SHARDS,
+  indexes: { window },
+});
+
+const insightsSketches = defineAggregate(identityFields, {
+  key: ["identity", "bucket_start_ms"],
+  distinct: ["launch_users", "activity_users"],
+  shards: SKETCH_SHARDS,
+  indexes: { window },
+});
+
+/** Installations counted in the hour of their latest event. */
+const insightsDistribution = defineAggregate(
+  {
+    channel: { type: "string" },
+    platform: { type: "string", maxLength: 16 },
+    app_version: { type: "string" },
+    release_id: { type: "string", maxLength: 36 },
+    bucket_start_ms: { type: "integer" },
+  },
+  {
+    key: [
+      "channel",
+      "platform",
+      "app_version",
+      "release_id",
+      "bucket_start_ms",
+    ],
+    gauges: ["latest_installations"],
+    shards: GAUGE_SHARDS,
+    indexes: {
+      byScope: { eq: ["channel", "platform"], sort: ["bucket_start_ms"] },
+      byVersion: {
+        eq: ["channel", "platform", "app_version"],
+        sort: ["bucket_start_ms"],
+      },
+    },
+  },
+);
+
+/** Latest events by the bundle they came from or went to. */
+const insightsLatestByBundle = defineAggregate(
+  {
+    platform: { type: "string", maxLength: 16 },
+    channel: { type: "string" },
+    bundle_field: { type: "string", maxLength: 16 },
+    bundle_id: { type: "string", maxLength: 36 },
+    type: { type: "string", maxLength: 32 },
+    bucket_start_ms: { type: "integer" },
+  },
+  {
+    key: [
+      "platform",
+      "channel",
+      "bundle_field",
+      "bundle_id",
+      "type",
+      "bucket_start_ms",
+    ],
+    gauges: ["installations"],
+    shards: GAUGE_SHARDS,
+    indexes: {
+      byBundle: {
+        eq: ["platform", "channel", "bundle_field", "bundle_id", "type"],
+        sort: ["bucket_start_ms"],
+      },
+    },
+  },
+);
+
+/**
+ * Events by outcome and the bundle a bundle filter matches, a row an hour;
+ * and every event, a row a UTC day (`DAILY_EVENTS`). Bundle counts sum the
+ * hourly rows, and event lists read either to skip days without events.
+ */
+const insightsOutcomes = defineAggregate(
+  {
+    platform: { type: "string", maxLength: 16 },
+    channel: { type: "string" },
+    type: { type: "string", maxLength: 32 },
+    bundle_ref: { type: "string", maxLength: 41 },
+    bucket_start_ms: { type: "integer" },
+  },
+  {
+    key: ["platform", "channel", "type", "bundle_ref", "bucket_start_ms"],
+    counters: ["events"],
+    shards: COUNTER_SHARDS,
+    indexes: {
+      byRef: {
+        eq: ["platform", "channel", "type", "bundle_ref"],
+        sort: ["bucket_start_ms"],
+      },
+    },
+  },
+);
+
+/**
+ * The `insights_outcomes` identity that counts every event, for the global
+ * event list. Its bucket is a UTC day, not an hour. Every event's platform
+ * is ios or android, so no outcome row of a bundle filter shares it.
+ */
+export const DAILY_EVENTS = {
+  platform: "*",
+  channel: "",
+  type: "",
+  bundle_ref: "",
+} as const;
+
+export const insightsSchema = {
+  bundle_events: bundleEvents,
+  bundle_event_heads: bundleEventHeads,
+  insights_overview: insightsOverview,
+  insights_sketches: insightsSketches,
+  insights_distribution: insightsDistribution,
+  insights_latest_by_bundle: insightsLatestByBundle,
+  insights_outcomes: insightsOutcomes,
+} as const;
+
+export type InsightsSchema = typeof insightsSchema;

package/src/plugins/pluginTestHarness.spec.ts

@@ -0,0 +1,56 @@
+import { createPluginTestHarness } from "@hot-updater/test-utils";
+import { describe, expect, it } from "vitest";
+
+import * as engine from "../database";
+import { defineTable } from "../database/schema";
+import { definePlugin } from "./definePlugin";
+
+const notes = definePlugin({
+  id: "notes",
+  schemaVersion: "1",
+  schema: {
+    notes: defineTable(
+      {
+        id: { type: "string" },
+        text: { type: "string" },
+        at: { type: "integer" },
+      },
+      { key: ["id"], indexes: { byTime: { eq: [], sort: ["at"] } } },
+    ),
+  },
+  init: ({ db, now }) => ({
+    api: {
+      add: (id: string, text: string) =>
+        db.transaction(async (tx) => {
+          tx.create("notes", { id, text, at: now() });
+        }),
+      latest: () =>
+        db.findMany("notes", {
+          index: "byTime",
+          where: {},
+          order: "desc",
+          limit: 2,
+        }),
+    },
+  }),
+});
+
+describe("plugin test harness", () => {
+  it("runs a plugin on a namespaced memory database and measures its reads", async () => {
+    const harness = await createPluginTestHarness(notes, {
+      engine,
+      now: () => 1,
+    });
+    await harness.api.add("a", "first");
+    harness.setNow(() => 2);
+    await harness.api.add("b", "second");
+    harness.setNow(() => 3);
+    await harness.api.add("c", "third");
+
+    expect(harness.tables.map(({ name }) => name)).toEqual(["notes_notes"]);
+    const measured = await harness.measureReads(() => harness.api.latest());
+    expect(measured.result.rows.map(({ id }) => id)).toEqual(["c", "b"]);
+    expect(measured.adapter.rows).toBe(measured.engine.rows);
+    expect(measured.engine).toEqual({ calls: 1, rows: 2 });
+  });
+});

package/src/readBudgets.testFixtures.ts

@@ -0,0 +1,15 @@
+import type { ReadBudgetServer } from "@hot-updater/test-utils";
+
+import { createMeasuredDatabase } from "./assembly/databasePlugins";
+import { targetBaseCandidateKey } from "./core/baseCandidates";
+import { builtInSchema } from "./db/builtInDatabase";
+import { apiKeys } from "./plugins/api-keys";
+import { insights } from "./plugins/insights";
+
+/** The server parts `setupReadBudgetTestSuite` takes, for the server's own backends. */
+export const readBudgetServer = {
+  createMeasuredDatabase,
+  builtInSchema,
+  plugins: [insights(), apiKeys()],
+  targetBaseCandidateKey,
+} satisfies ReadBudgetServer;

package/src/releaseCatalogRoutes.integration.spec.ts

@@ -1,96 +1,62 @@
 import { encodeChannelKey } from "@hot-updater/core";
-import { commitReleaseCatalogMutation } from "@hot-updater/plugin-core";
+import type { EngineDatabase } from "@hot-updater/plugin-core";
 import { describe, expect, it, vi } from "vitest";
 
-import { createInMemoryDatabasePlugin } from "../../test-utils/test/inMemoryDatabasePlugin";
-import { registerApiKey } from "./apiKeys";
+import { createDatabaseCoreApi } from "./core/api";
 import { createHotUpdater } from "./index";
+import { apiKeys } from "./plugins/api-keys";
+import { createRuntimeDatabase } from "./runtime.testFixtures";
 
 const API_KEY = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE";
 
 const channelKey = encodeChannelKey("production");
 const scopeKey = `v1:app-version:ios:${channelKey}`;
 
+/** A database with one deployed release in production's app-version scope. */
 const createCatalogDatabase = async () => {
-  const database = createInMemoryDatabasePlugin();
-  const channel = { id: "channel-production", name: "production" };
-  await database.models.channels.insert({
-    onConflict: "returnExisting",
-    row: channel,
-  });
-  const bundleId = "00000000-0000-7001-8000-000000000001";
-  await database.commit({
-    changes: [
-      {
-        model: "bundles",
-        operation: "insert",
-        row: {
-          asset_base_storage_uri: null,
-          archive_byte_size: 3_000_000_001,
-          file_hash: "bundle-hash",
-          git_commit_hash: null,
-          id: bundleId,
-          manifest_file_hash: null,
-          manifest_storage_uri: null,
-          metadata: {},
-          platform: "ios",
-          storage_uri: "storage://bundle.zip",
-        },
+  const database = createRuntimeDatabase();
+  await createDatabaseCoreApi(database).deploy([
+    {
+      bundle: {
+        assetBaseStorageUri: "storage://assets",
+        gitCommitHash: null,
+        id: "00000000-0000-7001-8000-000000000001",
+        manifestFileHash: "manifest-hash",
+        manifestStorageUri: "storage://bundle/manifest.json",
+        metadata: {},
+        platform: "ios",
       },
-    ],
-  });
-  await commitReleaseCatalogMutation({
-    database,
-    mutation: {
-      operation: "insert",
-      row: {
-        bundle_id: bundleId,
-        channel_id: channel.id,
-        created_at_ms: 1,
+      release: {
+        channel: "production",
         enabled: true,
-        fingerprint_hash: null,
-        id: "00000000-0000-7000-8000-000000000001",
-        kind: "BUNDLE",
+        fingerprintHash: null,
         message: "Stable Release",
-        operation: "DEPLOY",
-        platform: "ios",
-        revision: 1,
-        rollout_cohort_count: 1000,
-        scope_key: scopeKey,
-        should_force_update: false,
-        source_release_id: null,
-        strategy: "APP_VERSION",
-        target_app_version: ">=1.0.0 <2.0.0",
-        target_cohorts: [],
-        updated_at_ms: 1,
+        shouldForceUpdate: false,
+        targetAppVersion: ">=1.0.0 <2.0.0",
       },
     },
-    scope: {
-      channelId: channel.id,
-      channelName: channel.name,
-      fingerprintHash: null,
-      platform: "ios",
-      scopeKey,
-      strategy: "APP_VERSION",
-    },
-    updatedAtMs: 1,
-  });
+  ]);
   return database;
 };
 
+/** Counts point reads of catalog rows, the update check's one read. */
+const countCatalogReads = (database: EngineDatabase) => {
+  const get = vi.spyOn(database.adapter, "get");
+  return () =>
+    get.mock.calls.filter(([table]) => table.name === "release_catalogs")
+      .length;
+};
+
 describe("Release catalog routes", () => {
   it("serves persisted Catalog identity without configuration and keeps it across server restarts", async () => {
     const database = await createCatalogDatabase();
     const storedCatalog =
-      await database.models.releaseCatalogs.findByScopeKey(scopeKey);
+      await createDatabaseCoreApi(database).getReleaseCatalogRow(scopeKey);
+    const catalogReads = countCatalogReads(database);
     const hotUpdater = createHotUpdater({
       database,
-      clientAccess: { type: "public" },
+      clientAccess: "public",
     });
-    const catalogRead = vi.spyOn(
-      database.models.releaseCatalogs,
-      "findByScopeKey",
-    );
     const url =
       `https://updates.example.com/release-catalogs/app-version/` +
       `ios/${channelKey}/1.5.0`;
@@ -132,7 +98,7 @@ describe("Release catalog routes", () => {
     );
     expect(revalidated.status).toBe(304);
     expect(await revalidated.text()).toBe("");
-    expect(catalogRead).toHaveBeenCalledOnce();
+    expect(catalogReads()).toBe(1);
 
     const nonCanonicalVersion = await hotUpdater.handlers.client(
       new Request(url.replace("/1.5.0", "/v1.5")),
@@ -141,13 +107,13 @@ describe("Release catalog routes", () => {
     expect(nonCanonicalVersion.headers.get("cache-control")).toBe(
       "private, no-store",
     );
-    expect(catalogRead).toHaveBeenCalledOnce();
+    expect(catalogReads()).toBe(1);
 
     const invalidPlatform = await hotUpdater.handlers.client(
       new Request(url.replace("/ios/", "/windows/")),
     );
     expect(invalidPlatform.status).toBe(400);
-    expect(catalogRead).toHaveBeenCalledOnce();
+    expect(catalogReads()).toBe(1);
 
     const otherChannel = await hotUpdater.handlers.client(
       new Request(
@@ -155,7 +121,7 @@ describe("Release catalog routes", () => {
       ),
     );
     expect(otherChannel.status).toBe(404);
-    expect(catalogRead).toHaveBeenCalledTimes(2);
+    expect(catalogReads()).toBe(2);
 
     const legacyAuthorityPath = await hotUpdater.handlers.client(
       new Request(
@@ -167,28 +133,25 @@ describe("Release catalog routes", () => {
     expect(legacyAuthorityPath.headers.get("cache-control")).toBe(
       "private, no-store",
     );
-    expect(catalogRead).toHaveBeenCalledTimes(2);
-    const commit = vi.spyOn(database, "commit");
+    expect(catalogReads()).toBe(2);
+    const write = vi.spyOn(database.adapter, "write");
     const restarted = createHotUpdater({
       database,
-      clientAccess: { type: "public" },
+      clientAccess: "public",
     });
     const relocated = await restarted.handlers.client(
       new Request(url.replace("updates.example.com", "new.example.com")),
     );
     expect(await relocated.json()).toEqual(body);
-    expect(commit).not.toHaveBeenCalled();
+    expect(write).not.toHaveBeenCalled();
   });
 
   it("singleflights concurrent cold requests into one exact catalog read", async () => {
     const database = await createCatalogDatabase();
-    const catalogRead = vi.spyOn(
-      database.models.releaseCatalogs,
-      "findByScopeKey",
-    );
+    const catalogReads = countCatalogReads(database);
     const hotUpdater = createHotUpdater({
       database,
-      clientAccess: { type: "public" },
+      clientAccess: "public",
     });
     const url =
       `https://updates.example.com/release-catalogs/app-version/` +
@@ -201,23 +164,15 @@ describe("Release catalog routes", () => {
     );
 
     expect(responses.every(({ status }) => status === 200)).toBe(true);
-    expect(catalogRead).toHaveBeenCalledOnce();
+    expect(catalogReads()).toBe(1);
   });
 
   it("varies authenticated catalog responses by the configured header", async () => {
-    const database = await createCatalogDatabase();
-    await registerApiKey({
-      apiKeys: database.models.apiKeys,
-      apiKey: API_KEY,
-      name: "App",
-    });
     const hotUpdater = createHotUpdater({
-      clientAccess: {
-        headerName: "X-Hot-Updater-Key",
-        type: "api-key",
-      },
-      database,
+      database: await createCatalogDatabase(),
+      plugins: [apiKeys({ headerName: "X-Hot-Updater-Key" })],
     });
+    await hotUpdater.api.apiKeys.register({ apiKey: API_KEY, name: "App" });
     const url =
       `https://updates.example.com/release-catalogs/app-version/` +
       `ios/${channelKey}/1.5.0`;