@hot-updater/server 1.0.0-rc.1 → 1.0.0-rc.16

package/src/insights/domain.ts

@@ -1,4 +1,16 @@
+/**
+ * The body of `POST /events`. Fields the server does not know are ignored, so
+ * a newer client's report still records on an older server.
+ */
 export type CreateBundleEventRequestBase = {
+  /**
+   * The report's idempotency key: a canonical lowercase UUIDv7 that the
+   * client creates once per report and repeats on every retry. The server
+   * stores the report under it, so a retried report counts once, and a
+   * report under an ID already stored changes nothing. Without it, the
+   * server creates the ID, and a retry counts as another report.
+   */
+  readonly eventId?: string;
   readonly installId: string;
   readonly toBundleId: string;
   readonly userId?: string;
@@ -15,17 +27,7 @@ export type CreateBundleEventRequestBase = {
 
 export type CreateBundleEventRequest =
   | (CreateBundleEventRequestBase & {
-      readonly type: "UPDATE_APPLIED";
-      readonly fromBundleId: string;
-      readonly updateStrategy: "fingerprint" | "appVersion";
-    })
-  | (CreateBundleEventRequestBase & {
-      readonly type: "RECOVERED";
-      readonly fromBundleId: string;
-      readonly updateStrategy: "fingerprint" | "appVersion";
-    })
-  | (CreateBundleEventRequestBase & {
-      readonly type: "RELEASE_ADOPTED";
+      readonly type: "UPDATE_DOWNLOADED" | "UPDATE_APPLIED" | "RECOVERED";
       readonly fromBundleId: string;
       readonly updateStrategy: "fingerprint" | "appVersion";
     })
@@ -35,28 +37,20 @@ export type CreateBundleEventRequest =
       readonly updateStrategy: null;
     });
 
-export type BundleEventSummary = {
-  readonly installed: number;
-  readonly recovered: number;
-};
-
-export type BundleEventSummaryByBundle = BundleEventSummary & {
-  readonly bundleId: string;
-};
-
-export type BundleEventInsightsWindow = "24h" | "7d" | "30d" | "all";
 export type ActiveInstallationWindow = "24h" | "7d" | "30d";
 
-export type InstallationSearchRow = {
+export type EventHistoryRow = {
+  readonly id: string;
   readonly installId: string;
-  readonly username: string | null;
-  readonly userId: string | null;
-  readonly lastKnownBundleId: string;
-  readonly latestStatus:
+  readonly type:
+    | "UPDATE_DOWNLOADED"
     | "UPDATE_APPLIED"
     | "RECOVERED"
-    | "RELEASE_ADOPTED"
     | "UNCHANGED";
+  readonly fromBundleId: string | null;
+  readonly toBundleId: string;
+  readonly username: string | null;
+  readonly userId: string | null;
   readonly platform: "ios" | "android";
   readonly appVersion: string;
   readonly channel: string;
@@ -64,13 +58,19 @@ export type InstallationSearchRow = {
   readonly receivedAtMs: number;
 };
 
-export type InstallationHistoryRow = {
-  readonly id: string;
-  readonly type: "UPDATE_APPLIED" | "RECOVERED";
+export type InstallationHistoryRow = EventHistoryRow & {
+  readonly type: "UPDATE_DOWNLOADED" | "UPDATE_APPLIED" | "RECOVERED";
   readonly fromBundleId: string;
-  readonly toBundleId: string;
+};
+
+export type InstallationRow = {
+  readonly installId: string;
   readonly username: string | null;
   readonly userId: string | null;
+  readonly lastKnownBundleId: string;
+  readonly pendingBundleId: string | null;
+  readonly pendingReleaseId: string | null;
+  readonly latestStatus: EventHistoryRow["type"];
   readonly platform: "ios" | "android";
   readonly appVersion: string;
   readonly channel: string;
@@ -78,62 +78,41 @@ export type InstallationHistoryRow = {
   readonly receivedAtMs: number;
 };
 
-export type OffsetPaginationResult<TData> = {
-  readonly data: readonly TData[];
-  readonly pagination: {
-    readonly total: number;
-    readonly limit: number;
-    readonly offset: number;
-  };
+export type CursorPage<T> = {
+  readonly data: readonly T[];
+  readonly nextCursor: string | null;
 };
 
-export type InsightsSeriesPoint = {
-  readonly bucketStartMs: number;
-  readonly value: number;
+export type EventCursorPage<T extends EventHistoryRow> = CursorPage<T> & {
+  readonly beforeReceivedAtMs: number;
 };
 
-export type InsightsCohortPoint = {
-  readonly cohort: string;
-  readonly value: number;
+export type InsightsScope = {
+  readonly platform: "ios" | "android";
+  readonly channel: string;
 };
 
-export type BundleEventInsightsResult = {
-  readonly summary: BundleEventSummary;
-  readonly series: {
-    readonly installed: readonly InsightsSeriesPoint[];
-    readonly recovered: readonly InsightsSeriesPoint[];
-  };
-  readonly cohorts: {
-    readonly installed: readonly InsightsCohortPoint[];
-    readonly recovered: readonly InsightsCohortPoint[];
-  };
-  readonly recentEvents: OffsetPaginationResult<InstallationHistoryRow>;
+export type InsightsBundleSelection = InsightsScope & {
+  readonly bundleId: string;
+  readonly outcome: "downloaded" | "applied" | "recovered" | "unchanged";
 };
 
-export type BundleEventOverview = {
-  readonly trackedInstallations: number;
-  readonly bundles: readonly {
-    readonly bundleId: string;
-    readonly installations: number;
-  }[];
+export type InsightsCountMeasurement = {
+  readonly count: number;
+  readonly measuredAtMs: number;
 };
 
-export type ActiveInstallationOverview = {
-  readonly asOfMs: number;
+export type ReportingOverview = InsightsScope & {
   readonly window: ActiveInstallationWindow;
-  readonly activeInstallations: number;
-  readonly series: readonly InsightsSeriesPoint[];
-  readonly bundleSeries: readonly {
-    readonly bundleId: string;
-    readonly series: readonly InsightsSeriesPoint[];
-  }[];
-  readonly bundles: readonly {
+  readonly sinceMs: number;
+  readonly beforeReceivedAtMs: number;
+  readonly reportingInstallations: InsightsCountMeasurement;
+  readonly bundle?: {
     readonly bundleId: string;
-    readonly installations: number;
-  }[];
-};
-
-export type ActiveInstallationInput = {
-  readonly window: ActiveInstallationWindow;
-  readonly userId?: string;
+    readonly reportingInstallations: InsightsCountMeasurement;
+    readonly downloadedReports: InsightsCountMeasurement;
+    readonly appliedReports: InsightsCountMeasurement;
+    readonly recoveredReports: InsightsCountMeasurement;
+    readonly unchangedReports: InsightsCountMeasurement;
+  };
 };

package/src/insights/errors.spec.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  DatabaseConflictError,
+  DatabaseTransactionError,
+} from "../database/errors";
+import { isDatabaseBusyError } from "./errors";
+
+const failure = (message: string, fields: object) =>
+  Object.assign(new Error(message), fields);
+
+describe("isDatabaseBusyError", () => {
+  it.each([
+    [
+      "a transaction out of retries",
+      new DatabaseConflictError("The transaction conflicted 8 times."),
+    ],
+    [
+      "a throttled DynamoDB read",
+      failure("Throughput exceeds the current capacity of your table.", {
+        name: "ProvisionedThroughputExceededException",
+      }),
+    ],
+    ["an HTTP 429", failure("Too Many Requests", { status: 429 })],
+    [
+      "an AWS SDK 429",
+      failure("Too Many Requests", { $metadata: { httpStatusCode: 429 } }),
+    ],
+    [
+      "Firestore's RESOURCE_EXHAUSTED",
+      failure("8 RESOURCE_EXHAUSTED: Quota exceeded.", { code: 8 }),
+    ],
+  ])("holds for %s", (_case, error) => {
+    expect(isDatabaseBusyError(error)).toBe(true);
+  });
+
+  it.each([
+    ["a gauge below zero", new DatabaseTransactionError("gauge below zero")],
+    ["MongoDB's unknown error, also code 8", failure("unknown", { code: 8 })],
+    [
+      "a refused connection",
+      failure("connect ECONNREFUSED", { code: "ECONNREFUSED" }),
+    ],
+    ["a thrown string", "busy"],
+  ])("does not hold for %s", (_case, error) => {
+    expect(isDatabaseBusyError(error)).toBe(false);
+  });
+});

package/src/insights/errors.ts

@@ -1,9 +1,4 @@
-export class InsightsScanLimitExceededError extends Error {
-  constructor(readonly limit: number) {
-    super(`Insights event scan exceeded ${limit} rows.`);
-    this.name = "InsightsScanLimitExceededError";
-  }
-}
+import { DatabaseConflictError } from "../database/errors";
 
 export class InsightsBadRequestError extends Error {
   readonly name = "InsightsBadRequestError";
@@ -16,3 +11,52 @@ export class InsightsPayloadTooLargeError extends Error {
     super(`Event payload exceeds ${maximumBytes} bytes`);
   }
 }
+
+/** Error names the AWS SDKs give a throttled request, as their retry strategy classifies them. */
+const THROTTLING_NAMES = new Set([
+  "ProvisionedThroughputExceededException",
+  "RequestLimitExceeded",
+  "RequestThrottled",
+  "RequestThrottledException",
+  "SlowDown",
+  "ThrottledException",
+  "Throttling",
+  "ThrottlingException",
+  "TooManyRequestsException",
+]);
+
+/** gRPC's RESOURCE_EXHAUSTED, the status Firestore throttles a request with. */
+const RESOURCE_EXHAUSTED = 8;
+
+interface ErrorFields {
+  readonly name?: unknown;
+  readonly code?: unknown;
+  readonly message?: unknown;
+  readonly status?: unknown;
+  readonly statusCode?: unknown;
+  readonly $metadata?: { readonly httpStatusCode?: unknown };
+}
+
+/**
+ * Whether the database is busy rather than broken, so the same request can
+ * succeed later: a transaction ran out of retries, or the backend throttled
+ * a request. Writes already retry transient failures, throttled ones
+ * included, until the budget runs out. A throttled read throws the backend's
+ * own error, so it is recognized by what the backends call it: a name the
+ * AWS SDKs retry as throttling, HTTP 429, or gRPC's RESOURCE_EXHAUSTED.
+ */
+export const isDatabaseBusyError = (error: unknown): boolean => {
+  if (error instanceof DatabaseConflictError) return true;
+  if (typeof error !== "object" || error === null) return false;
+  const { name, code, message, status, statusCode, $metadata } =
+    error as ErrorFields;
+  return (
+    (typeof name === "string" && THROTTLING_NAMES.has(name)) ||
+    [status, statusCode, $metadata?.httpStatusCode].includes(429) ||
+    // A gRPC error's message starts with its status name; other drivers use
+    // numeric codes too, such as MongoDB's 8 for an unknown error.
+    (code === RESOURCE_EXHAUSTED &&
+      typeof message === "string" &&
+      message.includes("RESOURCE_EXHAUSTED"))
+  );
+};

package/src/insights/eventInput.ts

@@ -1,3 +1,9 @@
+import {
+  createUUIDv7,
+  isUUIDv7,
+  type BundleEventRow,
+} from "@hot-updater/plugin-core";
+
 import type {
   CreateBundleEventRequest,
   CreateBundleEventRequestBase,
@@ -8,26 +14,9 @@ import {
 } from "./errors";
 
 const MAX_EVENT_STRING_LENGTH = 1_024;
+const MAX_IDENTITY_LENGTH = 255;
 export const EVENT_BODY_MAX_BYTES = 16 * 1_024;
 
-const eventKeys = new Set([
-  "type",
-  "installId",
-  "toBundleId",
-  "userId",
-  "username",
-  "platform",
-  "appVersion",
-  "channel",
-  "cohort",
-  "fingerprintHash",
-  "fromBundleId",
-  "fromReleaseId",
-  "toReleaseId",
-  "updateStrategy",
-  "sdkVersion",
-]);
-
 function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -40,7 +29,10 @@ function requireStringField(
   if (
     typeof value !== "string" ||
     value.length === 0 ||
-    value.length > MAX_EVENT_STRING_LENGTH
+    value.length > MAX_EVENT_STRING_LENGTH ||
+    new TextDecoder("utf-8", { ignoreBOM: true }).decode(
+      new TextEncoder().encode(value),
+    ) !== value
   ) {
     throw new InsightsBadRequestError(`Invalid event field: ${key}`);
   }
@@ -55,6 +47,32 @@ function requireNullableStringField(
   return requireStringField(payload, key);
 }
 
+function requireIdentityField(
+  payload: Readonly<Record<string, unknown>>,
+  key: "installId" | "userId",
+): string {
+  const value = requireStringField(payload, key);
+  if (value.length > MAX_IDENTITY_LENGTH) {
+    throw new InsightsBadRequestError(`Invalid event field: ${key}`);
+  }
+  return value;
+}
+
+/**
+ * The report's idempotency key, when the client sends one: it creates the
+ * UUIDv7 once and repeats it on every retry of that report.
+ */
+function readEventId(
+  payload: Readonly<Record<string, unknown>>,
+): string | undefined {
+  const value = payload.eventId;
+  if (value === undefined) return undefined;
+  if (!isUUIDv7(value)) {
+    throw new InsightsBadRequestError("Invalid event field: eventId");
+  }
+  return value;
+}
+
 async function readBoundedText(request: Request): Promise<string> {
   const contentLength = request.headers.get("content-length");
   const declaredByteLength = Number(contentLength);
@@ -96,22 +114,23 @@ async function parseJson(request: Request): Promise<unknown> {
 }
 
 function requireEvent(payload: unknown): CreateBundleEventRequest {
-  if (
-    !isRecord(payload) ||
-    Object.keys(payload).some((key) => !eventKeys.has(key))
-  ) {
+  // Fields this server does not know are ignored, not refused, so a report
+  // from a newer SDK that adds one still records on an older server.
+  if (!isRecord(payload)) {
     throw new InsightsBadRequestError("Invalid event payload");
   }
   const platform = requireStringField(payload, "platform");
   if (platform !== "ios" && platform !== "android") {
     throw new InsightsBadRequestError("Invalid event field: platform");
   }
+  const eventId = readEventId(payload);
   const base: CreateBundleEventRequestBase = {
-    installId: requireStringField(payload, "installId"),
+    ...(eventId === undefined ? {} : { eventId }),
+    installId: requireIdentityField(payload, "installId"),
     toBundleId: requireStringField(payload, "toBundleId"),
     ...(payload.userId === undefined
       ? {}
-      : { userId: requireStringField(payload, "userId") }),
+      : { userId: requireIdentityField(payload, "userId") }),
     ...(payload.username === undefined
       ? {}
       : { username: requireStringField(payload, "username") }),
@@ -129,9 +148,9 @@ function requireEvent(payload: unknown): CreateBundleEventRequest {
   };
   const type = requireStringField(payload, "type");
   switch (type) {
+    case "UPDATE_DOWNLOADED":
     case "UPDATE_APPLIED":
-    case "RECOVERED":
-    case "RELEASE_ADOPTED": {
+    case "RECOVERED": {
       const updateStrategy = requireStringField(payload, "updateStrategy");
       if (updateStrategy !== "fingerprint" && updateStrategy !== "appVersion") {
         throw new InsightsBadRequestError(
@@ -166,3 +185,37 @@ export async function parseBundleEventRequest(
   const payload = await parseJson(request);
   return requireEvent(payload);
 }
+
+export function createBundleEventRow(
+  input: CreateBundleEventRequest,
+): BundleEventRow {
+  input = requireEvent(input);
+  const base = {
+    app_version: input.appVersion,
+    channel: input.channel,
+    from_release_id: input.fromReleaseId,
+    // The client's ID makes a retried report the same row, which records once.
+    id: input.eventId ?? createUUIDv7(),
+    install_id: input.installId,
+    platform: input.platform,
+    received_at_ms: Date.now(),
+    to_bundle_id: input.toBundleId,
+    to_release_id: input.toReleaseId,
+    user_id: input.userId ?? null,
+    metadata: {
+      cohort: input.cohort,
+      fingerprint_hash: input.fingerprintHash,
+      sdk_version: input.sdkVersion ?? null,
+      username: input.username ?? null,
+      update_strategy: input.updateStrategy,
+    },
+  };
+  switch (input.type) {
+    case "UPDATE_DOWNLOADED":
+    case "UPDATE_APPLIED":
+    case "RECOVERED":
+      return { ...base, from_bundle_id: input.fromBundleId, type: input.type };
+    case "UNCHANGED":
+      return { ...base, from_bundle_id: null, type: input.type };
+  }
+}