@hot-updater/plugin-insights 1.0.0-rc.20

package/dist/server/schema.mjs

@@ -0,0 +1,483 @@
+import { defineAggregate, defineTable } from "@hot-updater/plugin-core/server-plugin";
+//#region src/server/schema.ts
+const HOUR_MS = 36e5;
+const DAY_MS = 864e5;
+const DEFAULT_INSIGHTS_RETENTION = {
+	rawDays: 90,
+	dailyDays: 400
+};
+const bucketRetention = (days) => ({
+	field: "bucket_start_ms",
+	days
+});
+const MOVEMENTS = /* @__PURE__ */ new Set([
+	"UPDATE_DOWNLOADED",
+	"UPDATE_APPLIED",
+	"RECOVERED",
+	"UPDATE_FAILED"
+]);
+/** A failed update check: it names no target bundle, so no bundle list shows it. */
+const isFailedCheck = (row) => row.type === "UPDATE_FAILED" && row.metadata?.failure?.stage === "check";
+/**
+* Downloads, applies, recoveries, and update failures, whole. An UNCHANGED
+* report counts as a launch and moves its installation's head, but no list
+* shows it, so no row keeps it.
+*/
+const bundleEvents = (days) => defineTable({
+	id: {
+		type: "string",
+		maxLength: 36
+	},
+	type: {
+		type: "string",
+		maxLength: 32
+	},
+	install_id: {
+		type: "string",
+		maxLength: 255
+	},
+	user_id: {
+		type: "string",
+		maxLength: 255,
+		required: false
+	},
+	from_release_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	from_bundle_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	to_release_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	to_bundle_id: {
+		type: "string",
+		maxLength: 36
+	},
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	app_version: { type: "string" },
+	channel: { type: "string" },
+	metadata: { type: "json" },
+	received_at_ms: { type: "integer" }
+}, {
+	key: ["id"],
+	derived: {
+		/** The UTC day the event was received in. */
+		day: {
+			type: "integer",
+			compute: (row) => row.received_at_ms - row.received_at_ms % DAY_MS
+		},
+		/** Set only for events that belong in installation history. */
+		movement_install_id: {
+			type: "string",
+			compute: (row) => MOVEMENTS.has(row.type) ? row.install_id : null
+		},
+		/**
+		* The bundle a bundle filter matches: `from:<bundle>` for RECOVERED,
+		* none for a failed check, else `to:<bundle>`.
+		*/
+		bundle_ref: {
+			type: "string",
+			multi: true,
+			compute: (row) => isFailedCheck(row) ? [] : [row.type === "RECOVERED" && row.from_bundle_id !== null ? `from:${row.from_bundle_id}` : `to:${row.to_bundle_id}`]
+		}
+	},
+	indexes: {
+		recent: {
+			eq: [
+				"channel",
+				"platform",
+				"day"
+			],
+			sort: ["received_at_ms"]
+		},
+		movementsByInstall: {
+			eq: ["movement_install_id"],
+			sort: ["received_at_ms"]
+		},
+		byBundle: {
+			eq: [
+				"platform",
+				"channel",
+				"type",
+				"bundle_ref",
+				"day"
+			],
+			sort: ["received_at_ms"]
+		},
+		/** Events of every scope by day, for the console's unfiltered event list. */
+		byDay: {
+			eq: ["day"],
+			sort: ["received_at_ms"]
+		}
+	},
+	retention: {
+		field: "day",
+		days
+	}
+});
+/**
+* Each installation's latest event, whole, so reading it is one point read.
+* Its gauges count it in the UTC day of that event. An update failure moves
+* nothing, so no head holds one.
+*/
+const bundleEventHeads = (days) => defineTable({
+	install_id: {
+		type: "string",
+		maxLength: 255
+	},
+	id: {
+		type: "string",
+		maxLength: 36
+	},
+	type: {
+		type: "string",
+		maxLength: 32
+	},
+	user_id: {
+		type: "string",
+		maxLength: 255,
+		required: false
+	},
+	from_release_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	from_bundle_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	to_release_id: {
+		type: "string",
+		maxLength: 36,
+		required: false
+	},
+	to_bundle_id: {
+		type: "string",
+		maxLength: 36
+	},
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	app_version: { type: "string" },
+	channel: { type: "string" },
+	metadata: { type: "json" },
+	received_at_ms: { type: "integer" }
+}, {
+	key: ["install_id"],
+	indexes: { byUser: {
+		eq: ["user_id"],
+		sort: ["install_id"]
+	} },
+	retention: {
+		field: "received_at_ms",
+		days
+	}
+});
+/**
+* Shards are keyed by install id. Counters are blind increments and never
+* conflict, so they keep 8. Gauges and sketches are read, merged, and written
+* back. B3's rollout gate on PostgreSQL retried 2.4% of transactions at 8
+* shards and 1.3% at 16. DynamoDB's transactions take longer, so D8's gate
+* on DynamoDB Local retried 2–24% at 16, almost all on gauge rows, and 1–5%
+* with gauges at 32. Sketches stay at 16: each shard row carries 2 KB of
+* registers that every read merges.
+*
+* Every Insights aggregate is `batched`: on DynamoDB and Firestore, which
+* bill each write, an event's changes apply after it commits, merged with
+* other events' into one write per row, on one shard per writer.
+*/
+const COUNTER_SHARDS = 8;
+const GAUGE_SHARDS = 32;
+const SKETCH_SHARDS = 16;
+/** Rows keyed by a hashed identity (scope and period) and a bucket. */
+const identityFields = {
+	identity: {
+		type: "string",
+		maxLength: 32
+	},
+	bucket_start_ms: { type: "integer" }
+};
+const window = {
+	eq: ["identity"],
+	sort: ["bucket_start_ms"]
+};
+/**
+* Counters of one period kind: hourly, daily, or lifetime at bucket 0. A
+* release or channel row counts downloads, launches, and failed launches,
+* and downloads a patch delivered and those that fell back from one. A
+* release's lifetime row counts its failed updates (`failed_updates`), since
+* no breakdown row outlives the raw period; windowed reads sum
+* `insights_failures` instead.
+*/
+const counters = (retention) => defineAggregate(identityFields, {
+	key: ["identity", "bucket_start_ms"],
+	counters: [
+		"downloads",
+		"launches",
+		"failed_launches",
+		"failed_updates",
+		"patch_downloads",
+		"patch_fallbacks"
+	],
+	shards: COUNTER_SHARDS,
+	batched: true,
+	indexes: { window },
+	...retention === void 0 ? {} : { retention }
+});
+/**
+* Unique-installation sketches of one period kind. Rows of a failure
+* identity (`failure` and `check` scopes) keep only `failed_users`, so a
+* release's launch rows never grow by a failure's registers.
+*/
+const sketches = (retention) => defineAggregate(identityFields, {
+	key: ["identity", "bucket_start_ms"],
+	distinct: [
+		"launch_users",
+		"activity_users",
+		"failed_users"
+	],
+	shards: SKETCH_SHARDS,
+	batched: true,
+	indexes: { window },
+	retention
+});
+/** Installations whose update to a release failed, since its first failure: kept, at bucket 0. */
+const lifetimeSketches = () => defineAggregate(identityFields, {
+	key: ["identity", "bucket_start_ms"],
+	distinct: ["failed_users"],
+	shards: SKETCH_SHARDS,
+	batched: true,
+	indexes: { window }
+});
+/**
+* Update failures by what failed, and recoveries by why the crashed process
+* exited, a row an hour. `release_id` is the failed update's target release,
+* or empty for a failed check or an unknown target, and the recovered
+* release for a recovery. A failure's `stage` and `reason` say where and why
+* it failed, and `detail` what else the client knew, as the JSON array
+* `[resource, httpStatus, originCode, transport]`. A recovery's stage is
+* `launch` and its reason the exit reason, with an empty detail. Every index
+* is in key order, so its counters stay blind increments.
+*/
+const insightsFailures = (retention) => defineAggregate({
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	channel: { type: "string" },
+	bucket_start_ms: { type: "integer" },
+	release_id: {
+		type: "string",
+		maxLength: 36
+	},
+	stage: {
+		type: "string",
+		maxLength: 16
+	},
+	reason: {
+		type: "string",
+		maxLength: 64
+	},
+	detail: {
+		type: "string",
+		maxLength: 160
+	}
+}, {
+	key: [
+		"platform",
+		"channel",
+		"bucket_start_ms",
+		"release_id",
+		"stage",
+		"reason",
+		"detail"
+	],
+	counters: ["events"],
+	shards: COUNTER_SHARDS,
+	batched: true,
+	indexes: { byScope: {
+		eq: ["platform", "channel"],
+		sort: ["bucket_start_ms"]
+	} },
+	retention
+});
+/** Installations counted in the UTC day of their latest event, so reads sum whole UTC days. */
+const insightsDistribution = (retention) => defineAggregate({
+	channel: { type: "string" },
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	app_version: { type: "string" },
+	release_id: {
+		type: "string",
+		maxLength: 36
+	},
+	bucket_start_ms: { type: "integer" }
+}, {
+	key: [
+		"channel",
+		"platform",
+		"app_version",
+		"release_id",
+		"bucket_start_ms"
+	],
+	gauges: ["latest_installations"],
+	shards: GAUGE_SHARDS,
+	batched: true,
+	indexes: {
+		byScope: {
+			eq: ["channel", "platform"],
+			sort: ["bucket_start_ms"]
+		},
+		byVersion: {
+			eq: [
+				"channel",
+				"platform",
+				"app_version"
+			],
+			sort: ["bucket_start_ms"]
+		}
+	},
+	retention
+});
+/** Latest events by the bundle they came from or went to, bucketed like the distribution. */
+const insightsLatestByBundle = (retention) => defineAggregate({
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	channel: { type: "string" },
+	bundle_field: {
+		type: "string",
+		maxLength: 16
+	},
+	bundle_id: {
+		type: "string",
+		maxLength: 36
+	},
+	type: {
+		type: "string",
+		maxLength: 32
+	},
+	bucket_start_ms: { type: "integer" }
+}, {
+	key: [
+		"platform",
+		"channel",
+		"bundle_field",
+		"bundle_id",
+		"type",
+		"bucket_start_ms"
+	],
+	gauges: ["installations"],
+	shards: GAUGE_SHARDS,
+	batched: true,
+	indexes: { byBundle: {
+		eq: [
+			"platform",
+			"channel",
+			"bundle_field",
+			"bundle_id",
+			"type"
+		],
+		sort: ["bucket_start_ms"]
+	} },
+	retention
+});
+/**
+* Stored events by outcome and the bundle a bundle filter matches, a row an
+* hour, which a failed check has none of; and every stored event, a row a UTC
+* day (`DAILY_EVENTS`). Bundle counts sum the hourly rows, and event lists
+* read either to skip days without events.
+*/
+const insightsOutcomes = (retention) => defineAggregate({
+	platform: {
+		type: "string",
+		maxLength: 16
+	},
+	channel: { type: "string" },
+	type: {
+		type: "string",
+		maxLength: 32
+	},
+	bundle_ref: {
+		type: "string",
+		maxLength: 41
+	},
+	bucket_start_ms: { type: "integer" }
+}, {
+	key: [
+		"platform",
+		"channel",
+		"type",
+		"bundle_ref",
+		"bucket_start_ms"
+	],
+	counters: ["events"],
+	shards: COUNTER_SHARDS,
+	batched: true,
+	indexes: { byRef: {
+		eq: [
+			"platform",
+			"channel",
+			"type",
+			"bundle_ref"
+		],
+		sort: ["bucket_start_ms"]
+	} },
+	retention
+});
+/**
+* The `insights_outcomes` identity that counts every stored event, for the
+* global event list. Its bucket is a UTC day, not an hour. Every event's
+* platform is ios or android, so no outcome row of a bundle filter shares it.
+*/
+const DAILY_EVENTS = {
+	platform: "*",
+	channel: "",
+	type: "",
+	bundle_ref: ""
+};
+/**
+* Insights' tables, each expiring after its period. The periods are values
+* the engine applies at runtime; the tables and indexes are the same for
+* any periods.
+*/
+const createInsightsSchema = ({ rawDays, dailyDays } = DEFAULT_INSIGHTS_RETENTION) => {
+	const hourly = bucketRetention(rawDays);
+	const daily = bucketRetention(dailyDays);
+	return {
+		bundle_events: bundleEvents(rawDays),
+		bundle_event_heads: bundleEventHeads(dailyDays),
+		/** Hourly counters and sketches: the raw period. */
+		insights_overview: counters(hourly),
+		insights_sketches: sketches(hourly),
+		/** Daily counters and sketches: the daily period. */
+		insights_overview_daily: counters(daily),
+		insights_sketches_daily: sketches(daily),
+		/** A release's counters since its first event, and its failed installations: kept. */
+		insights_overview_lifetime: counters(),
+		insights_sketches_lifetime: lifetimeSketches(),
+		insights_distribution: insightsDistribution(daily),
+		insights_latest_by_bundle: insightsLatestByBundle(daily),
+		insights_outcomes: insightsOutcomes(hourly),
+		insights_failures: insightsFailures(hourly)
+	};
+};
+const insightsSchema = createInsightsSchema();
+//#endregion
+export { DAILY_EVENTS, DAY_MS, DEFAULT_INSIGHTS_RETENTION, HOUR_MS, createInsightsSchema, insightsSchema, isFailedCheck };

package/dist/server/testing/eventRow.d.mts

@@ -0,0 +1,64 @@
+import { DatabaseJsonObject, Platform } from "@hot-updater/plugin-core";
+//#region src/server/eventRow.d.ts
+/** Where an update failed, and what failed; open sets, `unknown` for any other value. */
+type BundleEventFailureStage = "check" | "download" | "install" | "unknown";
+type BundleEventFailureReason = "network" | "http" | "invalid_response" | "hash_mismatch" | "signature" | "patch" | "extract" | "storage" | "unknown";
+/** An `UPDATE_FAILED` report's failure, as stored. */
+type BundleEventFailure = DatabaseJsonObject & {
+  readonly stage: BundleEventFailureStage;
+  readonly reason: BundleEventFailureReason;
+  /** What the client was fetching or writing. */
+  readonly resource?: "catalog" | "artifact" | "manifest" | "file" | "patch" | "archive" | "unknown";
+  readonly http_status?: number;
+  /** How the connection failed, when it did. */
+  readonly transport?: "timeout" | "dns" | "tls" | "connection" | "offline" | "cancelled" | "unknown";
+  /** The storage origin's error code, such as `ExpiredToken`. */
+  readonly origin_code?: string;
+  /** Android's `ApplicationExitInfo` reason for the process before this one. */
+  readonly previous_process_exit?: string;
+};
+/** Ancillary report data; queryable identity and lifecycle fields stay on the row. */
+type DatabaseBundleEventMetadata = DatabaseJsonObject & {
+  readonly cohort: string;
+  readonly update_strategy: "fingerprint" | "appVersion" | null;
+  readonly fingerprint_hash: string | null;
+  readonly sdk_version: string | null;
+  /** `UPDATE_FAILED`: what failed. */
+  readonly failure?: BundleEventFailure;
+  /** `UPDATE_DOWNLOADED`: how the bundle arrived. */
+  readonly delivery?: "patch" | "manifest" | "archive" | "unknown";
+  /** `UPDATE_DOWNLOADED`: a patch failed and the full files came instead. */
+  readonly patch_fallback?: boolean;
+  /** `RECOVERED`: Android's `ApplicationExitInfo` reason for the crashed process. */
+  readonly previous_process_exit?: string;
+};
+type BundleEventRowBase = {
+  readonly id: string;
+  readonly install_id: string;
+  readonly user_id: string | null;
+  readonly from_release_id: string | null;
+  readonly to_release_id: string | null;
+  readonly to_bundle_id: string;
+  readonly platform: Platform;
+  readonly app_version: string;
+  readonly channel: string;
+  readonly metadata: DatabaseBundleEventMetadata;
+  readonly received_at_ms: number;
+};
+type BundleEventRow = BundleEventRowBase & ({
+  readonly type: "UPDATE_DOWNLOADED";
+  readonly from_bundle_id: string;
+} | {
+  readonly type: "UPDATE_APPLIED" | "RECOVERED";
+  readonly from_bundle_id: string;
+} |
+/** An update that failed: `from` is the running bundle, `to` the target (for a check, the running bundle). */
+{
+  readonly type: "UPDATE_FAILED";
+  readonly from_bundle_id: string;
+} | {
+  readonly type: "UNCHANGED";
+  readonly from_bundle_id: null;
+});
+//#endregion
+export { BundleEventFailure, BundleEventFailureReason, BundleEventFailureStage, BundleEventRow, BundleEventRowBase, DatabaseBundleEventMetadata };

package/dist/server/testing/expectInsightsIndex.mjs

@@ -0,0 +1,18 @@
+import { expect } from "vitest";
+//#region src/server/testing/expectInsightsIndex.ts
+/** Allow committed secondary indexes to catch up; query failures still fail. */
+const expectInsightsIndex = async (query, expected) => {
+	const deadline = Date.now() + 5e3;
+	for (;;) {
+		const actual = await query();
+		try {
+			expect(actual).toEqual(expected);
+			return;
+		} catch (error) {
+			if (Date.now() >= deadline) throw error;
+		}
+		await new Promise((resolve) => setTimeout(resolve, 50));
+	}
+};
+//#endregion
+export { expectInsightsIndex };

package/dist/server/testing/fixtures.d.mts

@@ -0,0 +1,8 @@
+import { BundleEventRow } from "./eventRow.mjs";
+//#region src/server/testing/fixtures.d.ts
+/** An applied update from install `suffix`, received at `receivedAtMs`. */
+declare const createBundleEventRowFixture: (suffix: string, receivedAtMs: number) => Extract<BundleEventRow, {
+  type: "UPDATE_APPLIED" | "RECOVERED";
+}>;
+//#endregion
+export { createBundleEventRowFixture };

package/dist/server/testing/fixtures.mjs

@@ -0,0 +1,25 @@
+//#region src/server/testing/fixtures.ts
+const fixtureId = (suffix) => `00000000-0000-7000-8000-${suffix.padStart(12, "0")}`;
+/** An applied update from install `suffix`, received at `receivedAtMs`. */
+const createBundleEventRowFixture = (suffix, receivedAtMs) => ({
+	id: fixtureId(suffix),
+	type: "UPDATE_APPLIED",
+	install_id: `install-${suffix}`,
+	user_id: null,
+	metadata: {
+		cohort: "0",
+		update_strategy: "appVersion",
+		fingerprint_hash: null,
+		sdk_version: null
+	},
+	from_bundle_id: fixtureId(`${Number(suffix) + 1e3}`),
+	from_release_id: null,
+	to_bundle_id: fixtureId(`${Number(suffix) + 2e3}`),
+	to_release_id: null,
+	platform: "ios",
+	app_version: "1.0.0",
+	channel: "production",
+	received_at_ms: receivedAtMs
+});
+//#endregion
+export { createBundleEventRowFixture };

package/dist/server/testing/index.d.mts

@@ -0,0 +1,24 @@
+import { InsightsModel } from "./modelTypes.mjs";
+import { DatabaseTestLifecycle, DatabaseTestState, HttpTestClient, HttpTestRequest } from "./types.mjs";
+import { createBundleEventRowFixture } from "./fixtures.mjs";
+import { setupInsightsHttpTestSuite } from "./setupInsightsHttpTestSuite.mjs";
+import { setupInsightsModelTestSuite } from "./setupInsightsModelTestSuite.mjs";
+//#region src/server/testing/index.d.ts
+/**
+ * The Insights plugin's tests, for `setupDatabaseTestSuite`'s `plugins`: its
+ * HTTP routes on the provider's server, which must run `insights()`, and
+ * with `createModel` its report contract on the provider's database, such as
+ * `(database) => createInsightsModel(createDatabasePluginApis(database,
+ * [insights()]).insights)`.
+ */
+declare const insightsTestSuite: <TDatabase = unknown>(options?: {
+  readonly createModel?: (database: TDatabase) => InsightsModel;
+}) => {
+  name: string;
+  register: (context: {
+    readonly getClient: () => HttpTestClient;
+    readonly getDatabase: () => TDatabase;
+  }) => void;
+};
+//#endregion
+export { type DatabaseTestLifecycle, type DatabaseTestState, type HttpTestClient, type HttpTestRequest, createBundleEventRowFixture, insightsTestSuite, setupInsightsHttpTestSuite, setupInsightsModelTestSuite };

package/dist/server/testing/index.mjs

@@ -0,0 +1,25 @@
+import { createBundleEventRowFixture } from "./fixtures.mjs";
+import { registerInsightsModelTests } from "./insightsModelTests.mjs";
+import { setupInsightsHttpTestSuite } from "./setupInsightsHttpTestSuite.mjs";
+import { setupInsightsModelTestSuite } from "./setupInsightsModelTestSuite.mjs";
+import { describe } from "vitest";
+//#region src/server/testing/index.ts
+/**
+* The Insights plugin's tests, for `setupDatabaseTestSuite`'s `plugins`: its
+* HTTP routes on the provider's server, which must run `insights()`, and
+* with `createModel` its report contract on the provider's database, such as
+* `(database) => createInsightsModel(createDatabasePluginApis(database,
+* [insights()]).insights)`.
+*/
+const insightsTestSuite = (options = {}) => ({
+	name: "Insights plugin",
+	register: (context) => {
+		setupInsightsHttpTestSuite({ getClient: context.getClient });
+		const { createModel } = options;
+		if (createModel !== void 0) describe("Insights model", () => {
+			registerInsightsModelTests({ getDatabase: () => createModel(context.getDatabase()) });
+		});
+	}
+});
+//#endregion
+export { createBundleEventRowFixture, insightsTestSuite, setupInsightsHttpTestSuite, setupInsightsModelTestSuite };