@hot-updater/server 0.36.7 → 1.0.0-rc.1

package/src/insights/bounded/provider.ts

@@ -0,0 +1,229 @@
+import type {
+  BundleEventInsightsWindow,
+  InstallationHistoryRow,
+} from "../domain";
+import type {
+  InsightsPersistence,
+  BundleEventPersistenceRow,
+} from "../persistence";
+import type { InsightsProvider } from "../types";
+import { collectActiveInstallationOverview } from "./activeOverview";
+import { searchEventInstallations } from "./installationSearch";
+import { createBundleEventRow } from "./persistence";
+import {
+  INSIGHTS_SCAN_MAX_ROWS,
+  collectEventActivity,
+  compareCodePoints,
+  compareEventNewest,
+  materializeActiveRows,
+  materializeEventRows,
+  materializeRowsForWindow,
+} from "./scan";
+
+type TransitionEventRow = BundleEventPersistenceRow & {
+  readonly type: "UPDATE_APPLIED" | "RECOVERED";
+};
+
+const isTransitionEventRow = (
+  row: BundleEventPersistenceRow,
+): row is TransitionEventRow =>
+  row.type === "UPDATE_APPLIED" || row.type === "RECOVERED";
+
+const toHistoryRow = (row: TransitionEventRow): InstallationHistoryRow => ({
+  id: row.id,
+  type: row.type,
+  fromBundleId: row.from_bundle_id,
+  toBundleId: row.to_bundle_id,
+  username: row.username,
+  userId: row.user_id,
+  platform: row.platform,
+  appVersion: row.app_version,
+  channel: row.channel,
+  cohort: row.cohort,
+  receivedAtMs: row.received_at_ms,
+});
+
+const isInstalledForBundle = (
+  row: TransitionEventRow,
+  bundleId: string,
+): boolean => row.type === "UPDATE_APPLIED" && row.to_bundle_id === bundleId;
+
+const isRecoveredFromBundle = (
+  row: TransitionEventRow,
+  bundleId: string,
+): boolean => row.type === "RECOVERED" && row.from_bundle_id === bundleId;
+
+const countDistinctInstallations = (
+  rows: readonly BundleEventPersistenceRow[],
+): number => new Set(rows.map(({ install_id }) => install_id)).size;
+
+const getInsightsResult = async (
+  persistence: InsightsPersistence,
+  bundleId: string,
+  window: BundleEventInsightsWindow,
+  limit: number,
+  offset: number,
+) => {
+  const scope = { persistence, cutoffMs: Date.now() };
+  const rows = (await materializeRowsForWindow(scope, window)).filter(
+    isTransitionEventRow,
+  );
+  const installedRows = rows.filter((row) =>
+    isInstalledForBundle(row, bundleId),
+  );
+  const recoveredRows = rows.filter((row) =>
+    isRecoveredFromBundle(row, bundleId),
+  );
+  const installed = collectEventActivity({
+    rows: installedRows,
+    window,
+    cutoffMs: scope.cutoffMs,
+  });
+  const recovered = collectEventActivity({
+    rows: recoveredRows,
+    window,
+    cutoffMs: scope.cutoffMs,
+  });
+  const recentRows = [...installedRows, ...recoveredRows].sort(
+    compareEventNewest,
+  );
+  return {
+    summary: { installed: installed.summary, recovered: recovered.summary },
+    series: { installed: installed.series, recovered: recovered.series },
+    cohorts: { installed: installed.cohorts, recovered: recovered.cohorts },
+    recentEvents: {
+      data: recentRows.slice(offset, offset + limit).map(toHistoryRow),
+      pagination: { total: recentRows.length, limit, offset },
+    },
+  };
+};
+
+const getInsightsSummaries = async (
+  persistence: InsightsPersistence,
+  bundleIds: readonly string[],
+  window: BundleEventInsightsWindow,
+) => {
+  const normalizedBundleIds = [...new Set(bundleIds)];
+  if (normalizedBundleIds.length === 0) return [];
+
+  const requestedBundleIds = new Set(normalizedBundleIds);
+  const installedByBundleId = new Map<string, Set<string>>();
+  const recoveredByBundleId = new Map<string, Set<string>>();
+  const rows = (
+    await materializeRowsForWindow(
+      { persistence, cutoffMs: Date.now() },
+      window,
+    )
+  ).filter(isTransitionEventRow);
+
+  for (const row of rows) {
+    const bundleId =
+      row.type === "UPDATE_APPLIED" ? row.to_bundle_id : row.from_bundle_id;
+    if (!requestedBundleIds.has(bundleId)) continue;
+
+    const counts =
+      row.type === "UPDATE_APPLIED" ? installedByBundleId : recoveredByBundleId;
+    const installIds = counts.get(bundleId) ?? new Set<string>();
+    installIds.add(row.install_id);
+    counts.set(bundleId, installIds);
+  }
+
+  return normalizedBundleIds.map((bundleId) => ({
+    bundleId,
+    installed: installedByBundleId.get(bundleId)?.size ?? 0,
+    recovered: recoveredByBundleId.get(bundleId)?.size ?? 0,
+  }));
+};
+
+export const createInsightsProvider = (
+  persistence: InsightsPersistence,
+): InsightsProvider =>
+  Object.freeze({
+    mode: "bounded",
+    maxMatchingRows: INSIGHTS_SCAN_MAX_ROWS,
+    async appendBundleEvent(input) {
+      await persistence.append(createBundleEventRow(input));
+    },
+    async getBundleEventSummary(bundleId) {
+      const rows = (
+        await materializeEventRows({ persistence, cutoffMs: Date.now() })
+      ).filter(isTransitionEventRow);
+      return {
+        installed: countDistinctInstallations(
+          rows.filter((row) => isInstalledForBundle(row, bundleId)),
+        ),
+        recovered: countDistinctInstallations(
+          rows.filter((row) => isRecoveredFromBundle(row, bundleId)),
+        ),
+      };
+    },
+    getBundleEventSummaries(bundleIds, window) {
+      return getInsightsSummaries(persistence, bundleIds, window);
+    },
+    getBundleEventInsights(bundleId, window, limit, offset) {
+      return getInsightsResult(persistence, bundleId, window, limit, offset);
+    },
+    async getBundleEventOverview() {
+      const rows = await materializeEventRows({
+        persistence,
+        cutoffMs: Date.now(),
+      });
+      const latestByInstall = new Map<string, BundleEventPersistenceRow>();
+      for (const row of rows) {
+        const current = latestByInstall.get(row.install_id);
+        if (current === undefined || compareEventNewest(row, current) < 0) {
+          latestByInstall.set(row.install_id, row);
+        }
+      }
+      const counts = new Map<string, number>();
+      for (const row of latestByInstall.values()) {
+        counts.set(row.to_bundle_id, (counts.get(row.to_bundle_id) ?? 0) + 1);
+      }
+      return {
+        trackedInstallations: latestByInstall.size,
+        bundles: [...counts]
+          .map(([bundleId, installations]) => ({ bundleId, installations }))
+          .sort(
+            (left, right) =>
+              right.installations - left.installations ||
+              compareCodePoints(left.bundleId, right.bundleId),
+          ),
+      };
+    },
+    async getActiveInstallationOverview(input) {
+      const asOfMs = Date.now();
+      const rows = await materializeActiveRows(
+        { persistence, cutoffMs: asOfMs },
+        input.window,
+      );
+      return collectActiveInstallationOverview({
+        rows,
+        asOfMs,
+        window: input.window,
+        ...(input.userId === undefined ? {} : { userId: input.userId }),
+      });
+    },
+    async searchInstallations(query, limit, offset) {
+      const rows = await materializeEventRows({
+        persistence,
+        cutoffMs: Date.now(),
+      });
+      return searchEventInstallations({ rows, query, limit, offset });
+    },
+    async getInstallationHistory(installId, limit, offset) {
+      const rows = await materializeEventRows({
+        persistence,
+        cutoffMs: Date.now(),
+      });
+      const ordered = rows
+        .filter(
+          (row): row is TransitionEventRow =>
+            row.install_id === installId && isTransitionEventRow(row),
+        )
+        .toSorted(compareEventNewest);
+      return {
+        data: ordered.slice(offset, offset + limit).map(toHistoryRow),
+        pagination: { total: ordered.length, limit, offset },
+      };
+    },
+  } satisfies InsightsProvider);

package/src/insights/bounded/scan.ts

@@ -0,0 +1,235 @@
+import type {
+  ActiveInstallationWindow,
+  BundleEventInsightsWindow,
+} from "../domain";
+import { InsightsScanLimitExceededError } from "../errors";
+import type {
+  InsightsPersistence,
+  BundleEventPersistenceRow,
+} from "../persistence";
+import {
+  ACTIVE_BUNDLE_EVENT_TYPES,
+  getActiveWindowDefinition,
+} from "./activeOverview";
+
+export const INSIGHTS_SCAN_MAX_ROWS = 50_000;
+export const INSIGHTS_MATERIALIZATION_LIMIT = INSIGHTS_SCAN_MAX_ROWS + 1;
+export const INSIGHTS_SCAN_PAGE_SIZE = 1_000;
+const INSIGHTS_LOWER_BOUND_ID = "00000000-0000-0000-0000-000000000000";
+
+export type InsightsScanScope = {
+  readonly persistence: InsightsPersistence;
+  readonly cutoffMs: number;
+  readonly lowerBoundMs?: number;
+};
+
+type EventActivityRequest = {
+  readonly rows: readonly BundleEventPersistenceRow[];
+  readonly window: BundleEventInsightsWindow;
+  readonly cutoffMs: number;
+};
+
+export const compareCodePoints = (left: string, right: string): number => {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+};
+
+export const compareEventNewest = (
+  left: BundleEventPersistenceRow,
+  right: BundleEventPersistenceRow,
+): number =>
+  right.received_at_ms - left.received_at_ms ||
+  compareCodePoints(right.id, left.id);
+
+const compareEventOldest = (
+  left: { readonly received_at_ms: number; readonly id: string },
+  right: { readonly received_at_ms: number; readonly id: string },
+): number =>
+  left.received_at_ms - right.received_at_ms ||
+  compareCodePoints(left.id, right.id);
+
+export const materializeEventRows = async (
+  scope: InsightsScanScope,
+): Promise<readonly BundleEventPersistenceRow[]> => {
+  const rows: BundleEventPersistenceRow[] = [];
+  const seenIds = new Set<string>();
+  let after =
+    scope.lowerBoundMs === undefined
+      ? undefined
+      : { receivedAtMs: scope.lowerBoundMs, id: INSIGHTS_LOWER_BOUND_ID };
+  while (rows.length < INSIGHTS_MATERIALIZATION_LIMIT) {
+    const limit = Math.min(
+      INSIGHTS_SCAN_PAGE_SIZE,
+      INSIGHTS_MATERIALIZATION_LIMIT - rows.length,
+    );
+    const page = await scope.persistence.scan({
+      beforeReceivedAtMs: scope.cutoffMs,
+      ...(after === undefined ? {} : { after }),
+      limit,
+    });
+    if (page.length === 0) break;
+    if (page.length > limit) throw new InsightsPersistenceOrderError();
+    let previous =
+      after === undefined
+        ? undefined
+        : { received_at_ms: after.receivedAtMs, id: after.id };
+    for (const row of page) {
+      if (
+        row.received_at_ms >= scope.cutoffMs ||
+        (previous !== undefined && compareEventOldest(previous, row) >= 0) ||
+        seenIds.has(row.id)
+      ) {
+        throw new InsightsPersistenceOrderError();
+      }
+      seenIds.add(row.id);
+      rows.push(row);
+      previous = row;
+    }
+    const last = page.at(-1);
+    if (last !== undefined) {
+      after = { receivedAtMs: last.received_at_ms, id: last.id };
+    }
+    if (rows.length > INSIGHTS_SCAN_MAX_ROWS) {
+      throw new InsightsScanLimitExceededError(INSIGHTS_SCAN_MAX_ROWS);
+    }
+  }
+  return rows;
+};
+
+export class InsightsPersistenceOrderError extends Error {
+  readonly name = "InsightsPersistenceOrderError";
+
+  constructor() {
+    super("Insights persistence did not advance its scan cursor.");
+  }
+}
+
+export const materializeActiveRows = async (
+  scope: InsightsScanScope,
+  window: ActiveInstallationWindow,
+): Promise<readonly BundleEventPersistenceRow[]> => {
+  const definition = getActiveWindowDefinition(window);
+  const durationMs = definition.bucketCount * definition.bucketSizeMs;
+  return (
+    await materializeEventRows({
+      ...scope,
+      lowerBoundMs: scope.cutoffMs - durationMs,
+    })
+  ).filter(
+    (row) =>
+      row.received_at_ms >= scope.cutoffMs - durationMs &&
+      ACTIVE_BUNDLE_EVENT_TYPES.includes(row.type),
+  );
+};
+
+const startOfUtcHour = (value: number): number => {
+  const date = new Date(value);
+  return Date.UTC(
+    date.getUTCFullYear(),
+    date.getUTCMonth(),
+    date.getUTCDate(),
+    date.getUTCHours(),
+  );
+};
+
+const startOfUtcDay = (value: number): number => {
+  const date = new Date(value);
+  return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
+};
+
+export const getWindowRange = (
+  window: Exclude<BundleEventInsightsWindow, "all">,
+  now: number,
+): { readonly sizeMs: number; readonly rangeStart: number } => {
+  if (window === "24h") {
+    return {
+      sizeMs: 60 * 60 * 1_000,
+      rangeStart: startOfUtcHour(now) - 23 * 60 * 60 * 1_000,
+    };
+  }
+  const days = window === "7d" ? 7 : 30;
+  return {
+    sizeMs: 24 * 60 * 60 * 1_000,
+    rangeStart: startOfUtcDay(now) - (days - 1) * 24 * 60 * 60 * 1_000,
+  };
+};
+
+export const materializeRowsForWindow = async (
+  scope: InsightsScanScope,
+  window: BundleEventInsightsWindow,
+): Promise<readonly BundleEventPersistenceRow[]> => {
+  if (window === "all") return materializeEventRows(scope);
+  const range = getWindowRange(window, scope.cutoffMs);
+  const rows = await materializeEventRows({
+    ...scope,
+    lowerBoundMs: range.rangeStart,
+  });
+  return rows.filter(
+    ({ received_at_ms }) => received_at_ms >= range.rangeStart,
+  );
+};
+
+const bucketStart = (receivedAtMs: number, sizeMs: number): number =>
+  sizeMs === 60 * 60 * 1_000
+    ? startOfUtcHour(receivedAtMs)
+    : startOfUtcDay(receivedAtMs);
+
+const createSeries = (request: EventActivityRequest) => {
+  const range =
+    request.window === "all"
+      ? undefined
+      : getWindowRange(request.window, request.cutoffMs);
+  const sizeMs = range?.sizeMs ?? 24 * 60 * 60 * 1_000;
+  const installIdsByBucket = new Map<number, Set<string>>();
+  let oldestMs = request.cutoffMs;
+  for (const row of request.rows) {
+    oldestMs = Math.min(oldestMs, row.received_at_ms);
+    const start = bucketStart(row.received_at_ms, sizeMs);
+    const installIds = installIdsByBucket.get(start) ?? new Set<string>();
+    installIds.add(row.install_id);
+    installIdsByBucket.set(start, installIds);
+  }
+  const first = range?.rangeStart ?? startOfUtcDay(oldestMs);
+  const last = bucketStart(request.cutoffMs, sizeMs);
+  return Array.from(
+    { length: Math.floor((last - first) / sizeMs) + 1 },
+    (_, index) => {
+      const start = first + index * sizeMs;
+      return {
+        bucketStartMs: start,
+        value: installIdsByBucket.get(start)?.size ?? 0,
+      };
+    },
+  );
+};
+
+export const collectEventActivity = (request: EventActivityRequest) => {
+  const range =
+    request.window === "all"
+      ? undefined
+      : getWindowRange(request.window, request.cutoffMs);
+  const rows =
+    range === undefined
+      ? request.rows
+      : request.rows.filter(
+          ({ received_at_ms }) =>
+            received_at_ms >= range.rangeStart &&
+            received_at_ms < request.cutoffMs,
+        );
+  const installs = new Set<string>();
+  const installsByCohort = new Map<string, Set<string>>();
+  for (const row of rows) {
+    installs.add(row.install_id);
+    const cohort = installsByCohort.get(row.cohort) ?? new Set<string>();
+    cohort.add(row.install_id);
+    installsByCohort.set(row.cohort, cohort);
+  }
+  return {
+    summary: installs.size,
+    cohorts: [...installsByCohort]
+      .sort(([left], [right]) => compareCodePoints(left, right))
+      .map(([cohort, installIds]) => ({ cohort, value: installIds.size })),
+    series: createSeries({ ...request, rows }),
+  };
+};

package/src/insights/domain.ts

@@ -0,0 +1,139 @@
+export type CreateBundleEventRequestBase = {
+  readonly installId: string;
+  readonly toBundleId: string;
+  readonly userId?: string;
+  readonly username?: string;
+  readonly platform: "ios" | "android";
+  readonly appVersion: string;
+  readonly channel: string;
+  readonly cohort: string;
+  readonly fingerprintHash: string | null;
+  readonly sdkVersion?: string | null;
+  readonly fromReleaseId: string | null;
+  readonly toReleaseId: string | null;
+};
+
+export type CreateBundleEventRequest =
+  | (CreateBundleEventRequestBase & {
+      readonly type: "UPDATE_APPLIED";
+      readonly fromBundleId: string;
+      readonly updateStrategy: "fingerprint" | "appVersion";
+    })
+  | (CreateBundleEventRequestBase & {
+      readonly type: "RECOVERED";
+      readonly fromBundleId: string;
+      readonly updateStrategy: "fingerprint" | "appVersion";
+    })
+  | (CreateBundleEventRequestBase & {
+      readonly type: "RELEASE_ADOPTED";
+      readonly fromBundleId: string;
+      readonly updateStrategy: "fingerprint" | "appVersion";
+    })
+  | (CreateBundleEventRequestBase & {
+      readonly type: "UNCHANGED";
+      readonly fromBundleId: null;
+      readonly updateStrategy: null;
+    });
+
+export type BundleEventSummary = {
+  readonly installed: number;
+  readonly recovered: number;
+};
+
+export type BundleEventSummaryByBundle = BundleEventSummary & {
+  readonly bundleId: string;
+};
+
+export type BundleEventInsightsWindow = "24h" | "7d" | "30d" | "all";
+export type ActiveInstallationWindow = "24h" | "7d" | "30d";
+
+export type InstallationSearchRow = {
+  readonly installId: string;
+  readonly username: string | null;
+  readonly userId: string | null;
+  readonly lastKnownBundleId: string;
+  readonly latestStatus:
+    | "UPDATE_APPLIED"
+    | "RECOVERED"
+    | "RELEASE_ADOPTED"
+    | "UNCHANGED";
+  readonly platform: "ios" | "android";
+  readonly appVersion: string;
+  readonly channel: string;
+  readonly cohort: string;
+  readonly receivedAtMs: number;
+};
+
+export type InstallationHistoryRow = {
+  readonly id: string;
+  readonly type: "UPDATE_APPLIED" | "RECOVERED";
+  readonly fromBundleId: string;
+  readonly toBundleId: string;
+  readonly username: string | null;
+  readonly userId: string | null;
+  readonly platform: "ios" | "android";
+  readonly appVersion: string;
+  readonly channel: string;
+  readonly cohort: string;
+  readonly receivedAtMs: number;
+};
+
+export type OffsetPaginationResult<TData> = {
+  readonly data: readonly TData[];
+  readonly pagination: {
+    readonly total: number;
+    readonly limit: number;
+    readonly offset: number;
+  };
+};
+
+export type InsightsSeriesPoint = {
+  readonly bucketStartMs: number;
+  readonly value: number;
+};
+
+export type InsightsCohortPoint = {
+  readonly cohort: string;
+  readonly value: number;
+};
+
+export type BundleEventInsightsResult = {
+  readonly summary: BundleEventSummary;
+  readonly series: {
+    readonly installed: readonly InsightsSeriesPoint[];
+    readonly recovered: readonly InsightsSeriesPoint[];
+  };
+  readonly cohorts: {
+    readonly installed: readonly InsightsCohortPoint[];
+    readonly recovered: readonly InsightsCohortPoint[];
+  };
+  readonly recentEvents: OffsetPaginationResult<InstallationHistoryRow>;
+};
+
+export type BundleEventOverview = {
+  readonly trackedInstallations: number;
+  readonly bundles: readonly {
+    readonly bundleId: string;
+    readonly installations: number;
+  }[];
+};
+
+export type ActiveInstallationOverview = {
+  readonly asOfMs: number;
+  readonly window: ActiveInstallationWindow;
+  readonly activeInstallations: number;
+  readonly series: readonly InsightsSeriesPoint[];
+  readonly bundleSeries: readonly {
+    readonly bundleId: string;
+    readonly series: readonly InsightsSeriesPoint[];
+  }[];
+  readonly bundles: readonly {
+    readonly bundleId: string;
+    readonly installations: number;
+  }[];
+};
+
+export type ActiveInstallationInput = {
+  readonly window: ActiveInstallationWindow;
+  readonly userId?: string;
+};

package/src/insights/errors.ts

@@ -0,0 +1,18 @@
+export class InsightsScanLimitExceededError extends Error {
+  constructor(readonly limit: number) {
+    super(`Insights event scan exceeded ${limit} rows.`);
+    this.name = "InsightsScanLimitExceededError";
+  }
+}
+
+export class InsightsBadRequestError extends Error {
+  readonly name = "InsightsBadRequestError";
+}
+
+export class InsightsPayloadTooLargeError extends Error {
+  readonly name = "InsightsPayloadTooLargeError";
+
+  constructor(readonly maximumBytes: number) {
+    super(`Event payload exceeds ${maximumBytes} bytes`);
+  }
+}