akanjs 3.1.0-rc.2 → 3.1.0-rc.21

package/server/migration/migrationChangeLog.ts

@@ -0,0 +1,86 @@
+import { type AkanSqlClient, MigrationTables } from "akanjs/service";
+import type { DocSlot } from "./docPath";
+
+/** `delete` is written by a rollback alone, removing a row an apply inserted. */
+export type MigrationChangeOp = "update" | "insert" | "delete";
+
+/**
+ * One row write. `before` and `after` map each changed `_doc` path to its slot; `$removedAt` is the row's removal time
+ * and `$doc` a whole `_doc` an insert or delete wrote or took away.
+ */
+export interface MigrationChange {
+  runId: string;
+  seq: number;
+  table: string;
+  id: string;
+  op: MigrationChangeOp;
+  before: Record<string, DocSlot>;
+  after: Record<string, DocSlot>;
+}
+
+interface MigrationChangeRow {
+  runId: string;
+  seq: number | string;
+  tableName: string;
+  docId: string;
+  op: MigrationChangeOp;
+  before: string;
+  after: string;
+}
+
+export class MigrationChangeLog {
+  readonly runId: string;
+  readonly #connection: () => AkanSqlClient;
+  #seq = 0;
+
+  constructor(runId: string, connection: () => AkanSqlClient) {
+    this.runId = runId;
+    this.#connection = connection;
+  }
+
+  /** Written on the caller's connection, so inside its transaction the entry commits or rolls back with the row. */
+  async append(change: Omit<MigrationChange, "runId" | "seq">) {
+    const seq = this.#seq + 1;
+    await this.#connection()
+      .prepare(
+        `INSERT INTO "${MigrationTables.change}" ("runId", "seq", "tableName", "docId", "op", "before", "after") VALUES (?, ?, ?, ?, ?, ?, ?)`,
+      )
+      .run(
+        this.runId,
+        seq,
+        change.table,
+        change.id,
+        change.op,
+        JSON.stringify(change.before),
+        JSON.stringify(change.after),
+      );
+    this.#seq = seq;
+  }
+
+  static async count(connection: AkanSqlClient, runId: string) {
+    const row = await connection
+      .prepare(`SELECT COUNT(*) AS "count" FROM "${MigrationTables.change}" WHERE "runId" = ?`)
+      .get<{ count: number | string }>(runId);
+    return Number(row?.count ?? 0);
+  }
+
+  /** Newest first, the order a rollback undoes them in: up to `limit` changes, from the newest or under seq `below`. */
+  static async page(connection: AkanSqlClient, runId: string, below: number | null, limit: number) {
+    const rows = await connection
+      .prepare(
+        `SELECT "runId", "seq", "tableName", "docId", "op", "before", "after" FROM "${MigrationTables.change}" WHERE "runId" = ?${below === null ? "" : ` AND "seq" < ?`} ORDER BY "seq" DESC LIMIT ${Math.trunc(limit)}`,
+      )
+      .all<MigrationChangeRow>(...(below === null ? [runId] : [runId, below]));
+    return rows.map(
+      (row): MigrationChange => ({
+        runId: row.runId,
+        seq: Number(row.seq),
+        table: row.tableName,
+        id: row.docId,
+        op: row.op,
+        before: JSON.parse(row.before) as Record<string, DocSlot>,
+        after: JSON.parse(row.after) as Record<string, DocSlot>,
+      }),
+    );
+  }
+}

package/server/migration/migrationChanges.ts

@@ -0,0 +1,70 @@
+import type { MigrationChange } from "./migrationChangeLog";
+import type { MigrationRevertOutcome, MigrationTable } from "./migrationTable";
+
+export interface MigrationSkip {
+  table: string;
+  id: string;
+  seq: number;
+  reason: string;
+}
+
+/**
+ * What a rollback walks with `for await`: the run's changes newest first, read a page at a time, each reverted or
+ * skipped with a reason the report shows.
+ */
+export class MigrationChanges implements AsyncIterable<MigrationChange> {
+  static readonly pageSize = 500;
+  readonly skipped: MigrationSkip[] = [];
+  reverted = 0;
+  readonly #page: (below: number | null, limit: number) => Promise<MigrationChange[]>;
+  readonly #table: (name: string) => MigrationTable;
+  readonly #from: number | null;
+  #reached: number | null;
+
+  /** `from`: a seq a rollback cut short got down to, so walking resumes under it. */
+  constructor(
+    page: (below: number | null, limit: number) => Promise<MigrationChange[]>,
+    table: (name: string) => MigrationTable,
+    from: number | null = null,
+  ) {
+    this.#page = page;
+    this.#table = table;
+    this.#from = from;
+    this.#reached = from;
+  }
+
+  async *[Symbol.asyncIterator]() {
+    let below = this.#from;
+    for (;;) {
+      const page = await this.#page(below, MigrationChanges.pageSize);
+      yield* page;
+      const last = page.at(-1);
+      if (!last || page.length < MigrationChanges.pageSize) return;
+      below = last.seq;
+    }
+  }
+
+  /** The oldest change `revertAll` has finished with; every newer one is reverted or skipped. */
+  get reached() {
+    return this.#reached;
+  }
+
+  /** Undone only if the row still holds what the change wrote; a row edited since is skipped and reported. */
+  async revert(change: MigrationChange): Promise<MigrationRevertOutcome> {
+    const outcome = await this.#table(change.table).revert(change);
+    if (outcome === "reverted") this.reverted++;
+    else this.skip(change, outcome === "missing" ? "row no longer exists" : "row changed since the migration");
+    return outcome;
+  }
+
+  skip(change: MigrationChange, reason: string) {
+    this.skipped.push({ table: change.table, id: change.id, seq: change.seq, reason });
+  }
+
+  async revertAll() {
+    for await (const change of this) {
+      await this.revert(change);
+      this.#reached = change.seq;
+    }
+  }
+}

package/server/migration/migrationCommand.ts

@@ -0,0 +1,179 @@
+import { parseArgs } from "node:util";
+import { MigrationError, type MigrationErrorCode } from "./migrationError";
+import type { MigrationListItem, MigrationRunner } from "./migrationRunner";
+
+export const migrationActions = ["list", "plan", "apply", "verify", "rollback"] as const;
+export type MigrationAction = (typeof migrationActions)[number];
+const migrationOptions = ["param", "expect", "actor", "items"] as const;
+type MigrationOption = (typeof migrationOptions)[number];
+
+export interface MigrationArgs {
+  action: MigrationAction;
+  target: string;
+  params: Record<string, string>;
+  expect: string | null;
+  actor: string | null;
+  items: number | undefined;
+  json: boolean;
+}
+
+/** The last line `ops migrate --json` prints: what akasys records, keyed by `ok` and `code`. */
+export interface MigrationCommandResult {
+  ok: boolean;
+  action: MigrationAction | null;
+  code?: MigrationErrorCode;
+  error?: string;
+  runId?: string | null;
+  [key: string]: unknown;
+}
+
+export class MigrationCommand {
+  static readonly #usageCodes = new Set<MigrationErrorCode>(["usage", "invalidParam", "notFound"]);
+  static readonly #optionsOf: { [action in MigrationAction]: readonly MigrationOption[] } = {
+    list: [],
+    plan: ["param", "items", "actor"],
+    apply: ["param", "expect", "actor"],
+    verify: ["param", "actor"],
+    rollback: ["actor"],
+  };
+
+  static usageOf(invokedAs = "ops migrate") {
+    return [
+      "Usage:",
+      `  ${invokedAs} list [--json]`,
+      `  ${invokedAs} plan <id> [--param <name>=<value>]... [--items <n>] [--actor <id>] [--json]`,
+      `  ${invokedAs} apply <id> [--param <name>=<value>]... [--expect <planHash>] [--actor <id>] [--json]`,
+      `  ${invokedAs} verify <id> [--param <name>=<value>]... [--actor <id>] [--json]`,
+      `  ${invokedAs} rollback <applyRunId|id> [--actor <id>] [--json]`,
+    ].join("\n");
+  }
+
+  static jsonOf(argv: string[]) {
+    return argv.some((arg) => arg === "--json" || arg.startsWith("--json="));
+  }
+
+  static parse(argv: string[], invokedAs?: string): MigrationArgs {
+    const usage = MigrationCommand.usageOf(invokedAs);
+    const { values, positionals } = MigrationCommand.#parseArgs(argv, usage);
+    const [name = "list", target = "", ...extra] = positionals;
+    if (!migrationActions.includes(name as MigrationAction))
+      throw new MigrationError("usage", `Unknown action "${name}"\n${usage}`);
+    const action = name as MigrationAction;
+    if (action === "list" ? target : !target)
+      throw new MigrationError("usage", `${action} ${target ? "takes no id" : "needs an id"}\n${usage}`);
+    if (extra.length) throw new MigrationError("usage", `Unexpected arguments: ${extra.join(" ")}`);
+    const refused = migrationOptions.find(
+      (option) => values[option] !== undefined && !MigrationCommand.#optionsOf[action].includes(option),
+    );
+    if (refused) throw new MigrationError("usage", `${action} takes no --${refused}\n${usage}`);
+    if (values.items !== undefined && !/^\d+$/.test(values.items))
+      throw new MigrationError("usage", "--items takes a whole number");
+    return {
+      action,
+      target,
+      params: MigrationCommand.#params(values.param ?? []),
+      expect: values.expect ?? null,
+      actor: values.actor ?? null,
+      items: values.items === undefined ? undefined : Number(values.items),
+      json: values.json,
+    };
+  }
+
+  static async run(runner: MigrationRunner, args: MigrationArgs): Promise<MigrationCommandResult> {
+    const { action, target, params, expect, actor, items } = args;
+    try {
+      if (action === "list") return { ok: true, action, migrations: await runner.list() };
+      if (action === "plan") return { ok: true, action, ...(await runner.plan(target, params, { actor, items })) };
+      if (action === "rollback") return { ok: true, action, ...(await runner.rollback(target, { actor })) };
+      const result =
+        action === "apply"
+          ? await runner.apply(target, params, { expect, actor })
+          : await runner.verify(target, params, { actor });
+      if (!result.problems.length) return { ok: true, action, ...result };
+      return {
+        ok: false,
+        action,
+        code: "verifyFailed",
+        error: MigrationError.remaining(result.problems.length),
+        ...result,
+      };
+    } catch (error) {
+      return MigrationCommand.failure(action, error);
+    }
+  }
+
+  static failure(action: MigrationAction | null, error: unknown): MigrationCommandResult {
+    return {
+      ok: false,
+      action,
+      code: MigrationError.codeOf(error),
+      error: MigrationError.messageOf(error),
+      runId: error instanceof MigrationError ? error.runId : null,
+    };
+  }
+
+  static exitCode(result: MigrationCommandResult) {
+    if (result.ok) return 0;
+    if (result.code === "verifyFailed") return 3;
+    return result.code && MigrationCommand.#usageCodes.has(result.code) ? 2 : 1;
+  }
+
+  static async print(result: MigrationCommandResult, json: boolean) {
+    const text = `${MigrationCommand.render(result, json)}\n`;
+    await new Promise<void>((resolve) => process.stdout.write(text, () => resolve()));
+    return MigrationCommand.exitCode(result);
+  }
+
+  static render(result: MigrationCommandResult, json: boolean) {
+    if (json) return JSON.stringify(result);
+    if (result.ok && result.action === "list")
+      return MigrationCommand.#renderList(result.migrations as MigrationListItem[]);
+    return JSON.stringify(result, null, 2);
+  }
+
+  static #renderList(items: MigrationListItem[]) {
+    if (!items.length) return "No migration in script/migration/";
+    return items
+      .map((item) => {
+        const note = item.checksumMismatch ? "  (applied from another version of the script)" : "";
+        const head = `${item.status.padEnd(8)} ${item.id}  ${item.title ?? `cannot load: ${item.error}`}${note}`;
+        const params = item.params.map(
+          (param) =>
+            `--param ${param.name}=<${param.type}>${param.required ? "" : "default" in param ? ` (default ${JSON.stringify(param.default)})` : " (optional)"}`,
+        );
+        return [head, ...params.map((line) => `         ${line}`)].join("\n");
+      })
+      .join("\n");
+  }
+
+  static #parseArgs(argv: string[], usage: string) {
+    try {
+      return parseArgs({
+        args: argv,
+        allowPositionals: true,
+        options: {
+          param: { type: "string", multiple: true },
+          expect: { type: "string" },
+          actor: { type: "string" },
+          items: { type: "string" },
+          json: { type: "boolean", default: false },
+        },
+      });
+    } catch (error) {
+      throw new MigrationError("usage", `${MigrationError.messageOf(error)}\n${usage}`);
+    }
+  }
+
+  static #params(pairs: string[]) {
+    const params: Record<string, string> = {};
+    for (const pair of pairs) {
+      const at = pair.indexOf("=");
+      const name = at > 0 ? pair.slice(0, at) : "";
+      if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(name))
+        throw new MigrationError("usage", `--param takes <name>=<value>, not "${pair}"`);
+      if (Object.hasOwn(params, name)) throw new MigrationError("usage", `--param ${name} is given twice`);
+      params[name] = pair.slice(at + 1);
+    }
+    return params;
+  }
+}

package/server/migration/migrationContext.ts

@@ -0,0 +1,53 @@
+import type { Logger } from "akanjs/common";
+import type { AkanSqlClient, SqlDialect } from "akanjs/service";
+import type { AkanServer } from "../akanServer";
+import type { MigrationScope } from "./migrationScope";
+import { MigrationTable } from "./migrationTable";
+
+export type MigrationGet = AkanServer["get"];
+
+export interface MigrationContextOption<Params> {
+  id: string;
+  params: Params;
+  logger: Logger;
+  get: MigrationGet;
+  scope: MigrationScope;
+}
+
+export class MigrationContext<Params = Record<string, unknown>> {
+  readonly id: string;
+  readonly params: Params;
+  readonly logger: Logger;
+  /** A service, signal or adaptor of the booted server, as `server.get` resolves it. */
+  readonly get: MigrationGet;
+  readonly #scope: MigrationScope;
+  readonly #tables = new Map<string, MigrationTable>();
+
+  constructor({ id, params, logger, get, scope }: MigrationContextOption<Params>) {
+    this.id = id;
+    this.params = params;
+    this.logger = logger;
+    this.get = get;
+    this.#scope = scope;
+  }
+
+  /** Rows of the model table `refName` as stored, whatever its model declares now. */
+  readonly table = (refName: string) => {
+    const existing = this.#tables.get(refName);
+    if (existing) return existing;
+    const table = new MigrationTable(refName, this.#scope);
+    this.#tables.set(refName, table);
+    return table;
+  };
+
+  readonly createId = () => this.#scope.createId();
+
+  /** The connection the run writes through, inside its transaction when it has one. */
+  get sql(): AkanSqlClient {
+    return this.#scope.connection();
+  }
+
+  get dialect(): SqlDialect {
+    return this.#scope.dialect;
+  }
+}

package/server/migration/migrationDefinition.ts

@@ -0,0 +1,87 @@
+import type { PromiseOrObject } from "akanjs/base";
+import type { FieldToValue, ParamFieldType } from "akanjs/constant";
+import type { MigrationChanges } from "./migrationChanges";
+import type { MigrationContext } from "./migrationContext";
+import { type MigrationParamOption, MigrationParams, type MigrationParamValue } from "./migrationParams";
+import { MigrationScope } from "./migrationScope";
+
+export interface MigrationPlan<Item = unknown> {
+  /** What a person approving the run reads before it writes anything. */
+  summary: Record<string, unknown>;
+  /** The decisions `apply` carries out; hashed with the params, so a changed one refuses an approved apply. */
+  items?: Item[];
+}
+
+export interface MigrationSettings {
+  /** `false` commits as it goes, each `rewrite` batch with where it got to: for runs too long for one transaction. */
+  transaction: boolean;
+  batchSize: number;
+}
+
+type MigrationStage<Params, Result> = (ctx: MigrationContext<Params>) => PromiseOrObject<Result>;
+
+export class MigrationDefinition<Params = Record<never, never>, Plan extends MigrationPlan = MigrationPlan> {
+  
+  static readonly #marker = Symbol.for("akan.migrationDefinition");
+  static readonly defaultSettings: MigrationSettings = { transaction: true, batchSize: 500 };
+  readonly title: string;
+  readonly params = new MigrationParams();
+  settings: MigrationSettings = { ...MigrationDefinition.defaultSettings };
+  planStage: MigrationStage<Params, Plan> | null = null;
+  applyStage: ((ctx: MigrationContext<Params>, plan: Plan) => PromiseOrObject<void>) | null = null;
+  verifyStage: MigrationStage<Params, string[]> | null = null;
+  rollbackStage: ((ctx: MigrationContext<Params>, changes: MigrationChanges) => PromiseOrObject<void>) | null = null;
+
+  constructor(title: string) {
+    this.title = title;
+    Object.defineProperty(this, MigrationDefinition.#marker, { value: true });
+  }
+
+  static is(value: unknown): value is MigrationDefinition<Record<string, unknown>> {
+    return (
+      !!value && typeof value === "object" && (value as Record<symbol, unknown>)[MigrationDefinition.#marker] === true
+    );
+  }
+
+  param<
+    Name extends string,
+    Type extends ParamFieldType,
+    const Option extends MigrationParamOption<FieldToValue<Type>> = Record<never, never>,
+  >(name: Name, type: Type, option?: Option) {
+    this.params.declare({ ...option, name, type });
+    return this as unknown as MigrationDefinition<Params & { [K in Name]: MigrationParamValue<Type, Option> }, Plan>;
+  }
+
+  /** Reads only, and returns the report a person approves; `apply` recomputes it inside its own transaction. */
+  plan<NextPlan extends MigrationPlan>(fn: MigrationStage<Params, NextPlan>) {
+    const next = this as unknown as MigrationDefinition<Params, NextPlan>;
+    next.planStage = fn;
+    return next;
+  }
+
+  apply(fn: (ctx: MigrationContext<Params>, plan: Plan) => PromiseOrObject<void>) {
+    this.applyStage = fn;
+    return this;
+  }
+
+  /** Runs right after apply commits; each string is one problem, and none means the data is as intended. */
+  verify(fn: MigrationStage<Params, string[]>) {
+    this.verifyStage = fn;
+    return this;
+  }
+
+  /** Replaces the default rollback, which reverts every logged change whose row still holds what it wrote. */
+  rollback(fn: (ctx: MigrationContext<Params>, changes: MigrationChanges) => PromiseOrObject<void>) {
+    this.rollbackStage = fn;
+    return this;
+  }
+
+  config(settings: Partial<MigrationSettings>) {
+    if (settings.batchSize !== undefined) MigrationScope.assertBatchSize(settings.batchSize);
+    this.settings = { ...this.settings, ...settings };
+    return this;
+  }
+}
+
+/** A data migration in `script/migration/<yyyyMMddHHmm>_<name>.ts`, exported as the file's default. */
+export const migration = (title: string) => new MigrationDefinition(title);

package/server/migration/migrationError.ts

@@ -0,0 +1,45 @@
+export type MigrationErrorCode =
+  | "usage"
+  | "notFound"
+  | "invalidParam"
+  | "locked"
+  | "alreadyApplied"
+  | "notApplied"
+  | "planChanged"
+  | "verifyFailed"
+  | "failed";
+
+/** A refusal or failure whose `code` the JSON result carries, so a caller decides by it rather than by the message. */
+export class MigrationError extends Error {
+  readonly code: MigrationErrorCode;
+  /** The ledger run the failure was recorded on, when it got that far. */
+  runId: string | null = null;
+
+  constructor(code: MigrationErrorCode, message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.name = "MigrationError";
+    this.code = code;
+  }
+
+  static codeOf(error: unknown): MigrationErrorCode {
+    return error instanceof MigrationError ? error.code : "failed";
+  }
+
+  static messageOf(error: unknown) {
+    return error instanceof Error ? error.message : String(error);
+  }
+
+  /** What a verify that found `problemNum` problems reports. */
+  static remaining(problemNum: number) {
+    return problemNum === 1 ? "1 problem remains" : `${problemNum} problems remain`;
+  }
+
+  static recorded(error: unknown, runId: string) {
+    const recorded =
+      error instanceof MigrationError
+        ? error
+        : new MigrationError("failed", MigrationError.messageOf(error), { cause: error });
+    recorded.runId = runId;
+    return recorded;
+  }
+}