@hot-updater/server 1.0.0-rc.18 → 1.0.0-rc.21

package/src/database/retention.spec.ts

@@ -1,314 +0,0 @@
-import {
-  createMemoryAdapter,
-  DatabaseSchemaError,
-  expiresAt,
-  type PhysicalTable,
-} from "@hot-updater/plugin-core/internal";
-import { describe, expect, it, vi } from "vitest";
-
-import { createDatabaseEngine } from "./database";
-import { NegativeGaugeError } from "./engineAggregates";
-import { resolveSchema, type SchemaModule } from "./resolveSchema";
-import { pruneDuringWrites } from "./retention";
-import { defineAggregate, defineTable, type ModuleSchema } from "./schema";
-
-const DAY = 86_400_000;
-
-const events = defineTable(
-  {
-    id: { type: "string", maxLength: 36 },
-    received_at_ms: { type: "integer" },
-  },
-  {
-    key: ["id"],
-    derived: {
-      day: {
-        type: "integer",
-        compute: (row) => row.received_at_ms - (row.received_at_ms % DAY),
-      },
-    },
-    indexes: { byDay: { eq: ["day"], sort: ["received_at_ms"] } },
-    retention: { field: "day", days: 90 },
-  },
-);
-
-/** Installations counted in the day of their latest event. */
-const latest = defineAggregate(
-  { release: { type: "string", maxLength: 16 }, bucket: { type: "integer" } },
-  {
-    key: ["release", "bucket"],
-    gauges: ["installations"],
-    shards: 4,
-    indexes: { byRelease: { eq: ["release"], sort: ["bucket"] } },
-    retention: { field: "bucket", days: 400 },
-  },
-);
-
-const kept = defineTable(
-  { id: { type: "string", maxLength: 36 } },
-  { key: ["id"] },
-);
-
-const module = <S extends ModuleSchema>(schema: S) => ({
-  id: "retention",
-  schema,
-});
-
-const invalid = (schema: SchemaModule["schema"]) => () =>
-  resolveSchema([{ id: "retention", schema }]);
-
-describe("retention", () => {
-  it("maps a model's retention onto its physical table", () => {
-    const { models } = resolveSchema([module({ events, latest, kept })]);
-
-    expect(models.get("events")!.table.retention).toEqual({
-      column: "day",
-      ms: 90 * DAY,
-    });
-    expect(models.get("latest")!.table.retention).toEqual({
-      column: "bucket",
-      ms: 400 * DAY,
-    });
-    expect(models.get("kept")!.table).not.toHaveProperty("retention");
-    expect(expiresAt(models.get("events")!.table, { day: 10 * DAY })).toBe(
-      100 * DAY,
-    );
-    expect(expiresAt(models.get("events")!.table, { day: null })).toBe(
-      undefined,
-    );
-  });
-
-  it("rejects retention on a field that holds no integer, or over no whole days", () => {
-    const table = (retention: { field: string; days: number }) =>
-      ({ ...kept, retention }) as unknown as typeof kept;
-
-    expect(invalid({ kept: table({ field: "id", days: 1 }) })).toThrow(
-      'retention.kept: retention field "id" is not an integer field',
-    );
-    expect(invalid({ kept: table({ field: "missing", days: 1 }) })).toThrow(
-      'retention field "missing" is not an integer field',
-    );
-    expect(
-      invalid({
-        events: { ...events, retention: { field: "day", days: 0.5 } },
-      }),
-    ).toThrow("retention days must be a whole number of at least 1");
-    expect(
-      invalid({ events: { ...events, retention: { field: "day", days: 0 } } }),
-    ).toThrow(DatabaseSchemaError);
-  });
-
-  it("rejects retention on a model in a reference or a rooted index", () => {
-    const parent = defineTable(
-      {
-        id: { type: "string", maxLength: 36 },
-        at: { type: "integer" },
-      },
-      { key: ["id"], retention: { field: "at", days: 1 } },
-    );
-    const child = defineTable(
-      {
-        id: { type: "string", maxLength: 36 },
-        parent: {
-          type: "string",
-          maxLength: 36,
-          references: { model: "parent", onDelete: "cascade" },
-        },
-      },
-      {
-        key: ["id"],
-        indexes: {
-          byParent: {
-            eq: ["parent"],
-            sort: [],
-            root: { model: "parent" },
-          },
-        },
-      },
-    );
-
-    expect(invalid({ parent, child })).toThrow(
-      "retention.parent: a model with retention is in no reference or rooted index",
-    );
-  });
-
-  const lease: PhysicalTable = {
-    name: "settings",
-    columns: [
-      { name: "key", type: "string", nullable: false, maxLength: 255 },
-      { name: "value", type: "string", nullable: false, maxLength: 255 },
-      { name: "_v", type: "integer", nullable: false, default: 0 },
-    ],
-    key: ["key"],
-    indexes: [],
-  };
-
-  it("prunes after writes only where rows expire and the backend keeps them", () => {
-    const adapter = createMemoryAdapter();
-    const { prune: _prune, ...native } = adapter;
-    const options = { leaseTable: lease, now: () => 0 };
-
-    const expiring = resolveSchema([module({ events })]).tables;
-    expect(pruneDuringWrites(native, expiring, options)).toBe(native);
-    const kepts = resolveSchema([module({ kept })]).tables;
-    expect(pruneDuringWrites(adapter, kepts, options)).toBe(adapter);
-  });
-
-  it("prunes again in a minute while a pass finds a full batch, else in an hour", async () => {
-    const { tables } = resolveSchema([module({ events })]);
-    const memory = createMemoryAdapter();
-    const prune = vi.fn(async () => 500);
-    let now = 100 * DAY;
-    const adapter = pruneDuringWrites({ ...memory, prune }, tables, {
-      leaseTable: lease,
-      now: () => now,
-    });
-    const writeAt = async (ms: number) => {
-      now += ms;
-      expect(await adapter.write([])).toEqual({ ok: true });
-    };
-
-    await writeAt(0);
-    await writeAt(30_000);
-    expect(prune).toHaveBeenCalledTimes(1);
-    expect(prune.mock.calls[0]).toEqual([tables[0], 10 * DAY, 500]);
-    await writeAt(31_000);
-    expect(prune).toHaveBeenCalledTimes(2);
-
-    prune.mockResolvedValue(3);
-    await writeAt(61_000);
-    await writeAt(30 * 60_000);
-    expect(prune).toHaveBeenCalledTimes(3);
-    await writeAt(31 * 60_000);
-    expect(prune).toHaveBeenCalledTimes(4);
-    const [row] = await memory.get(lease, [["retention.nextPassAt"]]);
-    expect(row).toMatchObject({ value: String(now + 3_600_000) });
-  });
-
-  it("writes even when its pass fails", async () => {
-    const { tables } = resolveSchema([module({ events })]);
-    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const adapter = pruneDuringWrites(
-      {
-        ...createMemoryAdapter(),
-        prune: async () => Promise.reject(new Error("down")),
-      },
-      tables,
-      { leaseTable: lease, now: () => 100 * DAY },
-    );
-
-    await expect(adapter.write([])).resolves.toEqual({ ok: true });
-    expect(warn).toHaveBeenCalledWith(
-      "[hot-updater] Pruning expired rows failed.",
-      expect.objectContaining({ message: "down" }),
-    );
-    warn.mockRestore();
-  });
-
-  it("prunes each table with retention as of the write, and nothing else", async () => {
-    const { tables } = resolveSchema([module({ events, latest, kept })]);
-    const prune = vi.fn(
-      async (_table: PhysicalTable, _before: number, _limit: number) => 2,
-    );
-    const adapter = pruneDuringWrites(
-      { ...createMemoryAdapter(), prune },
-      tables,
-      { leaseTable: lease, now: () => 1_000 * DAY },
-    );
-
-    await adapter.write([]);
-
-    expect(
-      prune.mock.calls.map(([table, before, limit]) => [
-        table.name,
-        before,
-        limit,
-      ]),
-    ).toEqual([
-      ["events", 910 * DAY, 500],
-      ["latest", 600 * DAY, 500],
-    ]);
-  });
-
-  it("deletes expired rows on the reference adapter, and keeps the rest", async () => {
-    const schema = resolveSchema([module({ events, kept })]);
-    let now = 100 * DAY;
-    const adapter = pruneDuringWrites(createMemoryAdapter(), schema.tables, {
-      leaseTable: lease,
-      now: () => now,
-    });
-    const db = createDatabaseEngine({ adapter, schema }).database(
-      module({ events, kept }),
-    );
-
-    // The write that takes the lease prunes before it stores its rows.
-    await db.transaction(async (tx) => {
-      tx.create("events", { id: "old", received_at_ms: 1 * DAY + 5 });
-      tx.create("events", { id: "new", received_at_ms: 95 * DAY });
-      tx.create("kept", { id: "kept" });
-    });
-    expect(await db.findOne("events", { id: "old" })).not.toBeNull();
-
-    // Day 1 expires at day 91; day 95 at day 185. The next pass is due in an hour.
-    now += 3_600_000;
-    await db.transaction(async (tx) => {
-      tx.create("kept", { id: "later" });
-    });
-    expect(await db.findOne("events", { id: "old" })).toBeNull();
-    expect(await db.findOne("events", { id: "new" })).not.toBeNull();
-    expect(await db.findOne("kept", { id: "kept" })).not.toBeNull();
-  });
-
-  it("drops a gauge decrement of a row retention already deleted", async () => {
-    const schema = resolveSchema([module({ latest })]);
-    const adapter = createMemoryAdapter();
-    const db = createDatabaseEngine({ adapter, schema }).database(
-      module({ latest }),
-    );
-    const move = (bucket: number, installations: number) =>
-      db.transaction(async (tx) => {
-        tx.aggregate(
-          "latest",
-          { release: "r1", bucket },
-          { installations },
-          { shardBy: "install-1" },
-        );
-      });
-    await move(1, 1);
-    await adapter.prune!(schema.tables[0]!, 600 * DAY, 10);
-
-    // The installation's day was pruned; its move out of it changes nothing.
-    await move(1, -1);
-    await move(900 * DAY, 1);
-
-    const rows = await db.findAggregates("latest", {
-      index: "byRelease",
-      where: { release: "r1" },
-      limit: 10,
-    });
-    expect(rows.rows).toEqual([
-      { release: "r1", bucket: 900 * DAY, installations: 1 },
-    ]);
-  });
-
-  it("still refuses a gauge below zero on a model without retention", async () => {
-    const { retention: _retention, ...forever } = latest;
-    const schema = resolveSchema([module({ latest: forever })]);
-    const db = createDatabaseEngine({
-      adapter: createMemoryAdapter(),
-      schema,
-      retry: { attempts: 2, baseDelayMs: 0 },
-    }).database(module({ latest: forever }));
-
-    await expect(
-      db.transaction(async (tx) => {
-        tx.aggregate(
-          "latest",
-          { release: "r1", bucket: 1 },
-          { installations: -1 },
-          { shardBy: "install-1" },
-        );
-      }),
-    ).rejects.toThrow(NegativeGaugeError);
-  });
-});

package/src/database/retention.ts

@@ -1,129 +0,0 @@
-import type {
-  DatabaseAdapter,
-  PhysicalRetention,
-  PhysicalTable,
-  StoredRow,
-  WriteOp,
-} from "@hot-updater/plugin-core/internal";
-
-import type { ModelShape, SchemaShape } from "./definitions";
-
-const MINUTE_MS = 60_000;
-const HOUR_MS = 60 * MINUTE_MS;
-const DAY_MS = 24 * HOUR_MS;
-
-/** A model's retention as its physical table carries it, which each adapter maps natively. */
-export const physicalRetention = ({
-  retention,
-}: ModelShape): { readonly retention?: PhysicalRetention } =>
-  retention === undefined
-    ? {}
-    : { retention: { column: retention.field, ms: retention.days * DAY_MS } };
-
-/** Models in a reference or a rooted index, which an expired row's delete would break. */
-const linkedModels = (schema: SchemaShape) =>
-  Object.entries(schema).flatMap(([name, { fields, indexes }]) =>
-    [
-      ...Object.values(fields).map(({ references }) => references),
-      ...Object.values(indexes).map(({ root }) => root),
-    ].flatMap((link) => (link ? [name, link.model] : [])),
-  );
-
-/**
- * What is wrong with a model's retention: it counts from an integer field,
- * over whole days, and expired rows are deleted without cascades, counter
- * moves, or root bumps, so the model is in no reference or rooted index.
- */
-export const retentionProblems = (
-  schema: SchemaShape,
-  model: string,
-): string[] => {
-  const definition = schema[model]!;
-  if (definition.retention === undefined) return [];
-  const { field, days } = definition.retention;
-  const declared =
-    definition.fields[field] ??
-    (definition.kind === "table" ? definition.derived[field] : undefined);
-  return [
-    ...(declared?.type !== "integer" ||
-    ("multi" in declared && declared.multi === true)
-      ? [`retention field "${field}" is not an integer field`]
-      : []),
-    ...(Number.isSafeInteger(days) && days >= 1
-      ? []
-      : ["retention days must be a whole number of at least 1"]),
-    ...(linkedModels(schema).includes(model)
-      ? ["a model with retention is in no reference or rooted index"]
-      : []),
-  ];
-};
-
-/**
- * Rows a pass deletes from each table: one statement or two a table, so a
- * pass stays well inside D1's 50 queries an invocation.
- */
-const PASS_LIMIT = 500;
-/** The lease row: when the next pass is due, in epoch ms. */
-const LEASE_KEY = "retention.nextPassAt";
-
-/**
- * The adapter, pruning during its writes with no scheduler: once the lease
- * row in `leaseTable` is due, the write that takes it first runs one bounded
- * pass over the tables with retention, and other writers skip. The next pass is due in an hour, or in a minute while a pass still
- * found a full batch of expired rows in some table. A backend that expires
- * rows natively, or a schema without retention, gets its adapter back.
- */
-export const pruneDuringWrites = (
-  adapter: DatabaseAdapter,
-  tables: readonly PhysicalTable[],
-  { leaseTable: table, now }: { leaseTable: PhysicalTable; now: () => number },
-): DatabaseAdapter => {
-  const expiring = tables.filter(({ retention }) => retention !== undefined);
-  if (adapter.prune === undefined || expiring.length === 0) return adapter;
-  const key = [LEASE_KEY];
-  let checkAt = 0;
-  // Moves the lease to `due` from `row`: the row it wrote, or null when
-  // another writer moved the lease first.
-  const lease = async (row: StoredRow | null, due: number) => {
-    const value = String(due);
-    const guard = { v: row === null ? -1 : Number(row._v) };
-    const next = { key: LEASE_KEY, value, _v: guard.v + 1 };
-    const op: WriteOp =
-      row === null
-        ? { type: "insert", table, row: next }
-        : { type: "patch", table, key, set: { value }, guard, previous: row };
-    return (await adapter.write([op])).ok ? next : null;
-  };
-  const pass = async () => {
-    const at = now();
-    if (at < checkAt) return;
-    checkAt = at + MINUTE_MS;
-    const [row = null] = await adapter.get(table, [key]);
-    // Due, or missing: this process reads it again if another takes it first.
-    checkAt = row === null ? at : Number(row.value);
-    if (checkAt > at) return;
-    const held = await lease(row, at + HOUR_MS);
-    if (held === null) return;
-    checkAt = at + HOUR_MS;
-    const deleted: number[] = [];
-    for (const t of expiring) {
-      deleted.push(await adapter.prune!(t, at - t.retention!.ms, PASS_LIMIT));
-    }
-    const backlog = deleted.includes(PASS_LIMIT);
-    if (backlog && (await lease(held, at + MINUTE_MS)) !== null) {
-      checkAt = at + MINUTE_MS;
-    }
-  };
-  const warn = (error: unknown) =>
-    console.warn("[hot-updater] Pruning expired rows failed.", error);
-  // Inherits every other member, so it reads the adapter as it is at each
-  // call, a proxy's included.
-  return Object.assign(Object.create(adapter) as DatabaseAdapter, {
-    // Before the write, so a pass never deletes what that write stores:
-    // rows written already expired go at the next pass.
-    async write(ops: readonly WriteOp[]) {
-      await pass().catch(warn);
-      return adapter.write(ops);
-    },
-  });
-};

package/src/database/schema.ts

@@ -1,213 +0,0 @@
-import type { DatabaseJson } from "@hot-updater/plugin-core/internal";
-
-import type {
-  AggregateShape,
-  FieldDefinition,
-  FieldType,
-  IndexDefinition,
-  TableShape,
-} from "./definitions";
-
-export type {
-  FieldDefinition,
-  FieldReference,
-  FieldType,
-  IndexDefinition,
-  ReferenceAction,
-} from "./definitions";
-
-/**
- * How long a model keeps its rows: `days` past the epoch milliseconds in
- * `field`, an integer field (a table's derived one too, where null keeps
- * the row). Expired rows are deleted without cascades, so a model with
- * retention is in no reference and no rooted index.
- */
-export interface RetentionDefinition<TField extends string = string> {
-  readonly field: TField;
-  readonly days: number;
-}
-
-export type FieldValue<TType extends FieldType> = TType extends "string"
-  ? string
-  : TType extends "integer" | "number"
-    ? number
-    : TType extends "boolean"
-      ? boolean
-      : DatabaseJson;
-
-type Fields = Readonly<Record<string, FieldDefinition>>;
-
-export type RowOf<TFields extends Fields> = {
-  -readonly [K in keyof TFields]: TFields[K] extends FieldDefinition<
-    infer TType
-  >
-    ? TFields[K] extends { readonly required: false }
-      ? FieldValue<TType> | null
-      : FieldValue<TType>
-    : never;
-};
-
-/** Computed on every write and stored as a real column. */
-export interface DerivedDefinition<
-  TRow = never,
-  TType extends FieldType = FieldType,
-> {
-  readonly type: TType;
-  /** Up to 16 values; the row is indexed once per value. */
-  readonly multi?: true;
-  /** Null, or an empty list, leaves the row out of indexes on this field. */
-  compute(row: TRow): FieldValue<TType> | null | readonly FieldValue<TType>[];
-}
-
-type DerivedTypes = Readonly<Record<string, FieldType>>;
-
-/** Derived fields over a table's row, by name. */
-export type DerivedFields<
-  TFields extends Fields,
-  TDerived extends DerivedTypes,
-> = {
-  readonly [K in keyof TDerived]: DerivedDefinition<
-    RowOf<TFields>,
-    TDerived[K]
-  >;
-};
-
-type Indexes = Readonly<Record<string, IndexDefinition>>;
-
-/** The integer fields, declared or derived, that retention can count from. */
-type IntegerFields<TFields extends Fields, TDerived extends DerivedTypes> =
-  | {
-      [K in keyof TFields]: TFields[K] extends { readonly type: "integer" }
-        ? K
-        : never;
-    }[keyof TFields]
-  | {
-      [K in keyof TDerived]: TDerived[K] extends "integer" ? K : never;
-    }[keyof TDerived];
-
-type MissingFields<TIndex, TNames extends string> = TIndex extends {
-  readonly eq: readonly (infer TEq)[];
-  readonly sort: readonly (infer TSort)[];
-}
-  ? Exclude<TEq | TSort, TNames>
-  : never;
-
-/** Maps an index to an error message when it names an undeclared field. */
-export type CheckIndex<
-  TName extends PropertyKey,
-  TIndex,
-  TNames extends string,
-> = [MissingFields<TIndex, TNames>] extends [never]
-  ? TIndex
-  : `Index "${TName & string}" names undeclared field "${MissingFields<TIndex, TNames> & string}"`;
-
-type CheckIndexes<TIndexes, TNames extends string> = {
-  readonly [K in keyof TIndexes]: CheckIndex<K, TIndexes[K], TNames>;
-};
-
-export interface TableDefinition<
-  TFields extends Fields = Fields,
-  TDerived extends DerivedTypes = DerivedTypes,
-  TIndexes extends Indexes = Indexes,
-  TKey extends readonly (keyof TFields & string)[] = readonly (keyof TFields &
-    string)[],
-> extends TableShape {
-  readonly kind: "table";
-  readonly fields: TFields;
-  readonly key: TKey;
-  readonly derived: DerivedFields<TFields, TDerived>;
-  readonly indexes: TIndexes;
-}
-
-export const defineTable = <
-  const TFields extends Fields,
-  const TKey extends readonly (keyof TFields & string)[],
-  const TDerived extends DerivedTypes = {},
-  const TIndexes extends Indexes = {},
->(
-  fields: TFields,
-  options: {
-    readonly key: TKey;
-    readonly derived?: DerivedFields<TFields, TDerived>;
-    readonly indexes?: TIndexes &
-      CheckIndexes<TIndexes, (keyof TFields | keyof TDerived) & string>;
-    /** Rows expire `days` after the epoch milliseconds in `field`. */
-    readonly retention?: RetentionDefinition<
-      IntegerFields<TFields, TDerived> & string
-    >;
-  },
-): TableDefinition<TFields, TDerived, TIndexes, TKey> => ({
-  kind: "table",
-  fields,
-  key: options.key,
-  derived: options.derived ?? ({} as DerivedFields<TFields, TDerived>),
-  indexes: options.indexes ?? ({} as TIndexes),
-  ...(options.retention === undefined ? {} : { retention: options.retention }),
-});
-
-export interface AggregateDefinition<
-  TFields extends Fields = Fields,
-  TMetric extends string = string,
-  TIndexes extends Indexes = Indexes,
-  TSketch extends string = string,
-  TKey extends readonly (keyof TFields & string)[] = readonly (keyof TFields &
-    string)[],
-> extends AggregateShape {
-  readonly kind: "aggregate";
-  readonly fields: TFields;
-  readonly key: TKey;
-  readonly counters: readonly TMetric[];
-  readonly gauges: readonly TMetric[];
-  readonly distinct: readonly TSketch[];
-  readonly indexes: TIndexes;
-  /**
-   * On a database that batches aggregates, changes apply after their
-   * transaction commits, in one write per row with other transactions'
-   * changes, so reads may lag them. Elsewhere they commit with it.
-   */
-  readonly batched?: true;
-}
-
-export const defineAggregate = <
-  const TFields extends Fields,
-  const TKey extends readonly (keyof TFields & string)[],
-  const TMetric extends string = never,
-  const TIndexes extends Indexes = {},
-  const TSketch extends string = never,
->(
-  fields: TFields,
-  options: {
-    readonly key: TKey;
-    readonly counters?: readonly TMetric[];
-    readonly gauges?: readonly TMetric[];
-    readonly distinct?: readonly TSketch[];
-    readonly shards?: number;
-    readonly indexes?: TIndexes &
-      CheckIndexes<TIndexes, keyof TFields & string>;
-    readonly batched?: true;
-    /**
-     * Rows expire `days` after the epoch milliseconds in `field`, an
-     * identity field such as a bucket's start; a change to a row already
-     * past it is dropped, so a late gauge decrement never goes below zero.
-     */
-    readonly retention?: RetentionDefinition<
-      IntegerFields<TFields, {}> & string
-    >;
-  },
-): AggregateDefinition<TFields, TMetric, TIndexes, TSketch, TKey> => ({
-  kind: "aggregate",
-  fields,
-  key: options.key,
-  counters: options.counters ?? [],
-  gauges: options.gauges ?? [],
-  distinct: options.distinct ?? [],
-  shards: options.shards ?? 1,
-  indexes: options.indexes ?? ({} as TIndexes),
-  ...(options.batched ? { batched: true } : {}),
-  ...(options.retention === undefined ? {} : { retention: options.retention }),
-});
-
-export type ModelDefinition = TableDefinition | AggregateDefinition;
-
-/** A module's models by name: core, or one plugin. */
-export type ModuleSchema = Readonly<Record<string, ModelDefinition>>;