@hot-updater/server 1.0.0-rc.1 → 1.0.0-rc.16

package/src/database/resolveSchema.spec.ts

@@ -0,0 +1,230 @@
+import { DatabaseSchemaError } from "@hot-updater/plugin-core/internal";
+import { describe, expect, it } from "vitest";
+
+import { resolveSchema, validateSchema } from "./resolveSchema";
+import { defineAggregate, defineTable, type ModuleSchema } from "./schema";
+
+const parents = defineTable(
+  {
+    id: { type: "string", maxLength: 36 },
+    name: { type: "string", unique: true },
+  },
+  { key: ["id"], indexes: { all: { eq: [], sort: ["id"] } } },
+);
+
+const children = defineTable(
+  {
+    id: { type: "string", maxLength: 36 },
+    parent_id: {
+      type: "string",
+      maxLength: 36,
+      references: { model: "parents", onDelete: "cascade" },
+    },
+    owner_id: {
+      type: "string",
+      required: false,
+      references: { model: "parents", onDelete: "restrict" },
+    },
+    position: { type: "integer" },
+    received_at_ms: { type: "integer" },
+  },
+  {
+    key: ["id"],
+    derived: {
+      day: {
+        type: "integer",
+        compute: (row) =>
+          row.received_at_ms - (row.received_at_ms % 86_400_000),
+      },
+      refs: {
+        type: "string",
+        multi: true,
+        compute: (row) => [`parent:${row.parent_id}`],
+      },
+    },
+    indexes: {
+      byParent: {
+        eq: ["parent_id"],
+        sort: ["position"],
+        root: { model: "parents" },
+      },
+      byRef: { eq: ["refs", "day"], sort: ["received_at_ms"] },
+    },
+  },
+);
+
+const totals = defineAggregate(
+  { scope: { type: "string", maxLength: 64 }, bucket: { type: "integer" } },
+  {
+    key: ["scope", "bucket"],
+    counters: ["hits"],
+    shards: 4,
+    indexes: { byScope: { eq: ["scope"], sort: ["bucket"] } },
+  },
+);
+
+const core: ModuleSchema = { parents, children, totals };
+
+describe("resolveSchema", () => {
+  it("adds engine columns, unique-field indexes, reference counters, and roots", () => {
+    const schema = resolveSchema([{ id: "core", schema: core }]);
+    const parent = schema.models.get("parents")!;
+
+    expect(parent.table.columns.map(({ name }) => name)).toEqual([
+      "id",
+      "name",
+      "_refs_children_parent_id",
+      "_refs_children_owner_id",
+      "_v",
+    ]);
+    expect(parent.table.indexes).toEqual([
+      { name: "all", eq: [], sort: ["id"] },
+      { name: "name", eq: ["name"], sort: [], unique: true },
+    ]);
+    expect(
+      parent.referencedBy.map(({ field, onDelete }) => [field, onDelete]),
+    ).toEqual([
+      ["parent_id", "cascade"],
+      ["owner_id", "restrict"],
+    ]);
+
+    const child = schema.models.get("children")!;
+    expect(child.roots).toEqual(new Map([["byParent", "parents"]]));
+    expect(child.table.columns.find(({ name }) => name === "refs")).toEqual({
+      name: "refs",
+      type: "string",
+      nullable: true,
+      multi: true,
+    });
+    expect(
+      child.table.columns.find(({ name }) => name === "owner_id"),
+    ).toMatchObject({
+      nullable: true,
+    });
+  });
+
+  it("keys aggregates by identity then shard, with metrics as integers", () => {
+    const aggregate = resolveSchema([{ id: "core", schema: core }]).models.get(
+      "totals",
+    )!;
+
+    expect(aggregate.table.key).toEqual(["scope", "bucket", "_shard"]);
+    expect(
+      aggregate.table.columns.map(({ name, type }) => [name, type]),
+    ).toEqual([
+      ["scope", "string"],
+      ["bucket", "integer"],
+      ["_shard", "integer"],
+      ["hits", "integer"],
+      ["_v", "integer"],
+    ]);
+  });
+
+  it("namespaces a third-party module's tables", () => {
+    const schema = resolveSchema([
+      { id: "core", schema: core },
+      { id: "audit", namespace: "audit", schema: { parents, children } },
+    ]);
+
+    expect([...schema.models.keys()]).toEqual([
+      "parents",
+      "children",
+      "totals",
+      "audit_parents",
+      "audit_children",
+    ]);
+    expect(schema.models.get("audit_children")!.references[0]).toMatchObject({
+      target: "audit_parents",
+      counter: "_refs_audit_children_parent_id",
+    });
+  });
+});
+
+describe("validateSchema", () => {
+  const problems = (schema: ModuleSchema) => {
+    try {
+      validateSchema([{ id: "core", schema }]);
+    } catch (error) {
+      expect(error).toBeInstanceOf(DatabaseSchemaError);
+      return (error as Error).message;
+    }
+    return "";
+  };
+
+  it("rejects nullable, json, and undeclared key fields", () => {
+    const message = problems({
+      rows: defineTable(
+        { id: { type: "string", required: false }, meta: { type: "json" } },
+        { key: ["id", "meta"] },
+      ),
+    });
+
+    expect(message).toContain('core.rows: key field "id" is nullable');
+    expect(message).toContain('core.rows: key field "meta" is json');
+  });
+
+  it("rejects a misordered aggregate key and mixed sketches", () => {
+    const message = problems({
+      totals: defineAggregate(
+        { scope: { type: "string" }, bucket: { type: "integer" } },
+        { key: ["bucket", "scope"], counters: ["hits"], distinct: ["users"] },
+      ),
+    });
+
+    expect(message).toContain(
+      "core.totals: key must list the identity fields in declaration order: scope, bucket",
+    );
+    expect(message).toContain(
+      "core.totals: distinct sketches need an aggregate of their own",
+    );
+  });
+
+  it("rejects bad roots, references, cascades without an index, and multi-valued sorts", () => {
+    const message = problems({
+      parents,
+      children: defineTable(
+        {
+          id: { type: "string" },
+          parent_id: {
+            type: "integer",
+            references: { model: "parents", onDelete: "cascade" },
+          },
+        },
+        {
+          key: ["id"],
+          derived: {
+            tags: { type: "string", multi: true, compute: () => ["a"] },
+          },
+          indexes: {
+            byTag: { eq: [], sort: ["tags"] },
+            rooted: { eq: ["id"], sort: [], root: { model: "missing" } },
+          },
+        },
+      ),
+    });
+
+    expect(message).toContain(
+      'index "byTag" sorts by multi-valued field "tags"',
+    );
+    expect(message).toContain(
+      'index "rooted" is rooted at unknown table "missing"',
+    );
+    expect(message).toContain(
+      'field "parent_id" and the key of "parents" differ in type',
+    );
+    expect(message).toContain(
+      'cascade on "parent_id" needs an index whose eq is exactly [parent_id]',
+    );
+  });
+
+  it("rejects duplicate table names across modules", () => {
+    expect(() =>
+      validateSchema([
+        { id: "core", schema: { parents } },
+        { id: "plugin", schema: { parents } },
+      ]),
+    ).toThrow(
+      'plugin.parents: table "parents" is also declared by core.parents',
+    );
+  });
+});

package/src/database/resolveSchema.ts

@@ -0,0 +1,372 @@
+import {
+  DATABASE_VERSION_COLUMN,
+  DatabaseSchemaError,
+  type PhysicalColumn,
+  type PhysicalIndex,
+  type PhysicalTable,
+} from "@hot-updater/plugin-core/internal";
+
+import type {
+  AggregateShape,
+  ModelShape,
+  ReferenceAction,
+  SchemaShape,
+  TableShape,
+} from "./definitions";
+
+export const SHARD_COLUMN = "_shard";
+export const MAX_SHARDS = 64;
+
+export interface SchemaModule {
+  readonly id: string;
+  readonly schema: SchemaShape;
+  /** Prefixes this module's table names; third-party plugins use their id. */
+  readonly namespace?: string;
+}
+
+export interface ResolvedReference {
+  /** The child table and the field that holds the parent key. */
+  readonly table: string;
+  readonly field: string;
+  readonly target: string;
+  readonly onDelete: ReferenceAction;
+  /** The counter column on the parent; absent for `none`. */
+  readonly counter?: string;
+}
+
+export interface ResolvedModel {
+  readonly module: string;
+  /** The model name as its module declares it. */
+  readonly model: string;
+  readonly table: PhysicalTable;
+  readonly definition: ModelShape;
+  readonly references: readonly ResolvedReference[];
+  readonly referencedBy: readonly ResolvedReference[];
+  /** Index name to the physical name of its root table. */
+  readonly roots: ReadonlyMap<string, string>;
+}
+
+export interface ResolvedSchema {
+  /** By physical table name. */
+  readonly models: ReadonlyMap<string, ResolvedModel>;
+  readonly tables: readonly PhysicalTable[];
+}
+
+const NAME = /^[a-z][a-z0-9_]*$/u;
+const INDEX_NAME = /^[A-Za-z][A-Za-z0-9_]*$/u;
+
+const tableName = (module: SchemaModule, model: string) =>
+  module.namespace ? `${module.namespace}_${model}` : model;
+
+const integer = (name: string, defaulted = false): PhysicalColumn => ({
+  name,
+  type: "integer",
+  nullable: false,
+  ...(defaulted ? { default: 0 } : {}),
+});
+
+/** A declared or derived field's type, and whether it holds several values. */
+const fieldType = (model: ModelShape, name: string) => {
+  const field =
+    model.fields[name] ??
+    (model.kind === "table" ? model.derived[name] : undefined);
+  return (
+    field && { type: field.type, multi: "multi" in field && !!field.multi }
+  );
+};
+
+const metricsOf = (model: AggregateShape) => [
+  ...model.counters,
+  ...model.gauges,
+  ...model.distinct,
+];
+
+type Add = (message: string) => void;
+/** A table of the model's own module, by model name. */
+type TableOf = (name: string) => TableShape | undefined;
+
+/** Field names, the key, and each field's options. */
+const fieldProblems = (definition: ModelShape, add: Add) => {
+  const names = [
+    ...Object.keys(definition.fields),
+    ...(definition.kind === "table"
+      ? Object.keys(definition.derived)
+      : metricsOf(definition)),
+  ];
+  for (const name of new Set(names)) {
+    if (!NAME.test(name)) add(`field "${name}" must match ${NAME.source}`);
+    if (names.indexOf(name) !== names.lastIndexOf(name)) {
+      add(`field "${name}" is declared twice`);
+    }
+  }
+  const { key } = definition;
+  if (key.length === 0) add("key is empty");
+  if (new Set(key).size !== key.length) add("key repeats a field");
+  for (const name of key) {
+    const field = definition.fields[name];
+    if (field === undefined) add(`key field "${name}" is not a declared field`);
+    else if (field.required === false) add(`key field "${name}" is nullable`);
+    else if (field.type === "json") add(`key field "${name}" is json`);
+  }
+  for (const [name, { type, maxLength, ascii }] of Object.entries(
+    definition.fields,
+  )) {
+    if (
+      maxLength !== undefined &&
+      (type !== "string" || !Number.isSafeInteger(maxLength) || maxLength < 1)
+    ) {
+      add(`field "${name}" has an invalid maxLength`);
+    }
+    if (ascii && type !== "string") {
+      add(`field "${name}" is ascii but not a string`);
+    }
+  }
+};
+
+/** A table's derived fields and references. */
+const tableProblems = (definition: TableShape, add: Add, table: TableOf) => {
+  for (const [name, derived] of Object.entries(definition.derived)) {
+    if (typeof derived.compute !== "function") {
+      add(`derived field "${name}" has no compute function`);
+    }
+    if (derived.multi && derived.type === "json") {
+      add(`derived field "${name}" cannot be multi-valued json`);
+    }
+  }
+  for (const [name, { type, references }] of Object.entries(
+    definition.fields,
+  )) {
+    if (references === undefined) continue;
+    const target = table(references.model);
+    const targetKey =
+      target?.key.length === 1 ? target.fields[target.key[0]!] : undefined;
+    if (targetKey === undefined) {
+      add(
+        `field "${name}" references "${references.model}", which is not a single-key table of this module`,
+      );
+    } else if (targetKey.type !== type) {
+      add(
+        `field "${name}" and the key of "${references.model}" differ in type`,
+      );
+    }
+    if (
+      references.onDelete === "cascade" &&
+      !Object.values(definition.indexes).some(
+        ({ eq }) => eq.length === 1 && eq[0] === name,
+      )
+    ) {
+      add(`cascade on "${name}" needs an index whose eq is exactly [${name}]`);
+    }
+  }
+};
+
+/** An aggregate's identity, metrics, and shards. */
+const aggregateProblems = (definition: AggregateShape, add: Add) => {
+  const identity = Object.keys(definition.fields);
+  if (definition.key.join("\u0000") !== identity.join("\u0000")) {
+    add(
+      `key must list the identity fields in declaration order: ${identity.join(", ")}`,
+    );
+  }
+  if (metricsOf(definition).length === 0) add("aggregate has no metrics");
+  if (
+    definition.distinct.length > 0 &&
+    definition.counters.length + definition.gauges.length > 0
+  ) {
+    add("distinct sketches need an aggregate of their own");
+  }
+  const { shards } = definition;
+  if (!Number.isSafeInteger(shards) || shards < 1 || shards > MAX_SHARDS) {
+    add(`shards must be 1–${MAX_SHARDS}`);
+  }
+};
+
+/** Every problem in one model's declaration, its roots and references included. */
+const problemsOf = (
+  module: SchemaModule,
+  model: string,
+  definition: ModelShape,
+): string[] => {
+  const problems: string[] = [];
+  const add = (message: string) =>
+    problems.push(`${module.id}.${model}: ${message}`);
+  const table = (name: string) => {
+    const target = module.schema[name];
+    return target?.kind === "table" ? target : undefined;
+  };
+  if (!NAME.test(model)) {
+    add("model names use lowercase letters, digits, and _");
+  }
+  fieldProblems(definition, add);
+  for (const [name, index] of Object.entries(definition.indexes)) {
+    const columns = [...index.eq, ...index.sort];
+    if (!INDEX_NAME.test(name)) {
+      add(`index "${name}" must match ${INDEX_NAME.source}`);
+    }
+    if (columns.length === 0) add(`index "${name}" has no fields`);
+    if (index.unique && index.sort.length > 0) {
+      add(`unique index "${name}" cannot sort`);
+    }
+    const multi = columns.filter((column) => {
+      const type = fieldType(definition, column);
+      if (type === undefined) {
+        add(`index "${name}" names undeclared field "${column}"`);
+      } else if (type.type === "json") {
+        add(`index "${name}" names json field "${column}"`);
+      } else if (type.multi && index.sort.includes(column)) {
+        add(`index "${name}" sorts by multi-valued field "${column}"`);
+      }
+      return type?.type !== "json" && type?.multi;
+    });
+    if (multi.length > 1) {
+      add(`index "${name}" has more than one multi-valued field`);
+    }
+    if (index.root === undefined) continue;
+    const root = table(index.root.model);
+    if (root === undefined) {
+      add(`index "${name}" is rooted at unknown table "${index.root.model}"`);
+    } else if (
+      root.key.length > index.eq.length ||
+      root.key.some(
+        (field, position) =>
+          fieldType(definition, index.eq[position]!)?.type !==
+          root.fields[field]!.type,
+      )
+    ) {
+      add(
+        `index "${name}" eq must start with the key of "${index.root.model}"`,
+      );
+    }
+  }
+  if (definition.kind === "table") tableProblems(definition, add, table);
+  else aggregateProblems(definition, add);
+  return problems;
+};
+
+/** Rejects invalid module schemas at startup, listing every problem. */
+export const validateSchema = (modules: readonly SchemaModule[]): void => {
+  const problems: string[] = [];
+  const owners = new Map<string, string>();
+  for (const module of modules) {
+    for (const [model, definition] of Object.entries(module.schema)) {
+      const name = tableName(module, model);
+      const owner = owners.get(name);
+      if (owner !== undefined) {
+        problems.push(
+          `${module.id}.${model}: table "${name}" is also declared by ${owner}`,
+        );
+      }
+      owners.set(name, `${module.id}.${model}`);
+      problems.push(...problemsOf(module, model, definition));
+    }
+  }
+  if (problems.length > 0) {
+    throw new DatabaseSchemaError(
+      `Invalid database schema:\n- ${problems.join("\n- ")}`,
+    );
+  }
+};
+
+const indexesOf = (definition: ModelShape): PhysicalIndex[] => [
+  ...Object.entries(definition.indexes).map(([name, index]) => ({
+    name,
+    eq: [...index.eq],
+    sort: [...index.sort],
+    ...(index.unique ? { unique: true as const } : {}),
+  })),
+  ...Object.entries(definition.fields).flatMap(([name, field]) =>
+    field.unique ? [{ name, eq: [name], sort: [], unique: true as const }] : [],
+  ),
+];
+
+/** Validates the modules, then resolves them into physical tables with engine columns. */
+export const resolveSchema = (
+  modules: readonly SchemaModule[],
+): ResolvedSchema => {
+  validateSchema(modules);
+  const references: ResolvedReference[] = [];
+  for (const module of modules) {
+    for (const [model, definition] of Object.entries(module.schema)) {
+      if (definition.kind !== "table") continue;
+      const table = tableName(module, model);
+      for (const [field, { references: to }] of Object.entries(
+        definition.fields,
+      )) {
+        if (to === undefined) continue;
+        const { onDelete } = to;
+        const target = tableName(module, to.model);
+        const counter =
+          onDelete === "none" ? {} : { counter: `_refs_${table}_${field}` };
+        references.push({ table, field, target, onDelete, ...counter });
+      }
+    }
+  }
+  const models = new Map<string, ResolvedModel>();
+  for (const module of modules) {
+    for (const [model, definition] of Object.entries(module.schema)) {
+      const name = tableName(module, model);
+      const referencedBy = references.filter(({ target }) => target === name);
+      const columns: PhysicalColumn[] = Object.entries(definition.fields).map(
+        ([field, { type, required, maxLength, ascii }]) => ({
+          name: field,
+          type,
+          nullable: required === false,
+          ...(maxLength === undefined ? {} : { maxLength }),
+          ...(ascii ? { ascii } : {}),
+        }),
+      );
+      if (definition.kind === "table") {
+        for (const [field, { type, multi }] of Object.entries(
+          definition.derived,
+        )) {
+          columns.push({
+            name: field,
+            type,
+            nullable: true,
+            ...(multi ? { multi } : {}),
+          });
+        }
+      } else {
+        columns.push(
+          integer(SHARD_COLUMN),
+          ...[...definition.counters, ...definition.gauges].map((metric) =>
+            integer(metric),
+          ),
+          ...definition.distinct.map((metric) => ({
+            name: metric,
+            type: "string" as const,
+            nullable: true,
+          })),
+        );
+      }
+      for (const { counter } of referencedBy) {
+        if (counter !== undefined) columns.push(integer(counter, true));
+      }
+      columns.push(integer(DATABASE_VERSION_COLUMN, true));
+      models.set(name, {
+        module: module.id,
+        model,
+        definition,
+        table: {
+          name,
+          columns,
+          key:
+            definition.kind === "aggregate"
+              ? [...definition.key, SHARD_COLUMN]
+              : [...definition.key],
+          indexes: indexesOf(definition),
+        },
+        references: references.filter(({ table }) => table === name),
+        referencedBy,
+        roots: new Map(
+          Object.entries(definition.indexes).flatMap(([index, { root }]) =>
+            root === undefined
+              ? []
+              : [[index, tableName(module, root.model)] as const],
+          ),
+        ),
+      });
+    }
+  }
+  return { models, tables: [...models.values()].map(({ table }) => table) };
+};