@kedge/cli 0.0.0-stage → 0.9.0

package/src/ops.ts

@@ -0,0 +1,355 @@
+// The operations database (docs/proposals/p0.8-cli-agent-kit.md §3.1): a
+// second D1 database per store, `<store>-ops`, that no Worker binds. It holds
+// what a lost CI runner can't lose and a restore of the store can't rewind:
+// who is running what on the store's production resources and on each preview
+// (one operation per scope), every control-plane call before it's sent and
+// its outcome, the deployment manifests, the hashes of applied migrations,
+// the store's own resources, and the preview registry.
+//
+// Exclusion is a primary key and compare-and-set updates, read back with
+// RETURNING, so of two callers one wins and a paused one finds its claim gone
+// at its next step. Nothing here takes over on a timer: elapsed time doesn't
+// prove a paused job is gone.
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
+import type { Row } from "./cf";
+
+export type Query = (sql: string, params?: readonly unknown[]) => Promise<Row[]>;
+
+export const OPS_SCHEMA = [
+  `CREATE TABLE IF NOT EXISTS kedge_ops_meta (id INTEGER PRIMARY KEY CHECK (id = 1), store TEXT NOT NULL, created_at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS operation (
+    scope TEXT PRIMARY KEY, op_id TEXT NOT NULL UNIQUE, kind TEXT NOT NULL, plan TEXT NOT NULL, step TEXT NOT NULL,
+    progress TEXT NOT NULL DEFAULT '{}',
+    executor TEXT, executor_label TEXT, heartbeat_at INTEGER NOT NULL, started_at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS operation_log (
+    op_id TEXT PRIMARY KEY, scope TEXT NOT NULL, kind TEXT NOT NULL, plans TEXT NOT NULL, outcome TEXT NOT NULL,
+    started_at INTEGER NOT NULL, ended_at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS intent (
+    id TEXT PRIMARY KEY, op_id TEXT NOT NULL, scope TEXT NOT NULL, step TEXT NOT NULL, kind TEXT NOT NULL,
+    tag TEXT NOT NULL, detail TEXT NOT NULL,
+    state TEXT NOT NULL CHECK (state IN ('open', 'done', 'failed', 'unknown', 'not_executed')),
+    created_at INTEGER NOT NULL, closed_at INTEGER, resolved_by TEXT)`,
+  `CREATE INDEX IF NOT EXISTS intent_scope_state ON intent (scope, state)`,
+  `CREATE TABLE IF NOT EXISTS deployment (
+    id TEXT PRIMARY KEY, seq INTEGER NOT NULL UNIQUE, op_id TEXT NOT NULL, mode TEXT NOT NULL,
+    before_half TEXT NOT NULL, after_half TEXT, created_at INTEGER NOT NULL, completed_at INTEGER)`,
+  `CREATE TABLE IF NOT EXISTS applied (file TEXT PRIMARY KEY, sha256 TEXT NOT NULL, applied_at INTEGER NOT NULL)`,
+  `CREATE TABLE IF NOT EXISTS resource (
+    kind TEXT NOT NULL, name TEXT NOT NULL, id TEXT, state TEXT NOT NULL, created_at INTEGER NOT NULL,
+    PRIMARY KEY (kind, name))`,
+  `CREATE TABLE IF NOT EXISTS preview (
+    identity TEXT PRIMARY KEY, nonce TEXT NOT NULL, worker TEXT NOT NULL, database_name TEXT NOT NULL,
+    database_id TEXT, queue_name TEXT NOT NULL, queue_id TEXT, classes TEXT NOT NULL DEFAULT '[]', consumer_id TEXT,
+    state TEXT NOT NULL CHECK (state IN ('creating', 'bootstrapped', 'destroying', 'destroyed')),
+    credentials TEXT, applied TEXT NOT NULL DEFAULT '{}', lost_create INTEGER NOT NULL DEFAULT 0,
+    created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL)`,
+  // A destroyed generation whose create lost its answer, kept by its own names when the identity
+  // is reused, so the sweep still finds what a late create leaves for its 7 days (§3.1, codex code r2).
+  `CREATE TABLE IF NOT EXISTS preview_tombstone (
+    worker TEXT PRIMARY KEY, identity TEXT NOT NULL, database_name TEXT NOT NULL, queue_name TEXT NOT NULL,
+    classes TEXT NOT NULL DEFAULT '[]', destroyed_at INTEGER NOT NULL)`,
+];
+
+export class OperationHeld extends Error {
+  constructor(readonly held: OperationRow) {
+    super(
+      `${held.scope} is held by operation ${held.op_id} (${held.kind}, at "${held.step}", ${held.executor ? `run by ${held.executor_label ?? held.executor}` : "stopped"}, since ${new Date(held.started_at).toISOString()})`,
+    );
+    this.name = "OperationHeld";
+  }
+}
+
+export class LostOwnership extends Error {
+  constructor(scope: string, step: string) {
+    super(
+      `${scope}: this process no longer holds its operation (at "${step}"); another took it over or it ended. Stopping.`,
+    );
+    this.name = "LostOwnership";
+  }
+}
+
+export interface OperationRow {
+  scope: string;
+  op_id: string;
+  kind: string;
+  plan: string;
+  step: string;
+  /** What the operation has learned so far (ids, versions), so a resume continues from it. */
+  progress: string;
+  executor: string | null;
+  executor_label: string | null;
+  heartbeat_at: number;
+  started_at: number;
+}
+
+export interface IntentRow {
+  id: string;
+  op_id: string;
+  scope: string;
+  step: string;
+  kind: string;
+  tag: string;
+  detail: string;
+  state: "open" | "done" | "failed" | "unknown" | "not_executed";
+  created_at: number;
+  closed_at: number | null;
+  resolved_by: string | null;
+}
+
+export interface DeploymentRow {
+  id: string;
+  seq: number;
+  op_id: string;
+  mode: string;
+  before_half: string;
+  after_half: string | null;
+  created_at: number;
+  completed_at: number | null;
+}
+
+/** Who this process is, for a refusal to name: the host and, in CI, the run. */
+export function executorLabel(env: NodeJS.ProcessEnv = process.env): string {
+  const run = env.GITHUB_RUN_ID
+    ? ` (GitHub run ${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/${env.GITHUB_RUN_ATTEMPT}` : ""})`
+    : "";
+  return `${hostname()}:${process.pid}${run}`;
+}
+
+export class OpsStore {
+  /** This process's identity: a random id, never reused. */
+  readonly executor = `ex_${randomUUID().replaceAll("-", "")}`;
+  constructor(
+    private readonly q: Query,
+    private readonly now: () => number = Date.now,
+    readonly label: string = executorLabel(),
+  ) {}
+
+  async init(store: string): Promise<void> {
+    for (const s of OPS_SCHEMA) await this.q(s);
+    await this.q("INSERT INTO kedge_ops_meta (id, store, created_at) VALUES (1, ?, ?) ON CONFLICT (id) DO NOTHING", [
+      store,
+      this.now(),
+    ]);
+    const [meta] = await this.q("SELECT store FROM kedge_ops_meta WHERE id = 1");
+    if (meta?.store !== store)
+      throw new Error(`this operations database belongs to ${String(meta?.store)}, not ${store}`);
+  }
+
+  // ---- operations --------------------------------------------------------------------------------------
+  async current(scope: string): Promise<OperationRow | null> {
+    return ((await this.q("SELECT * FROM operation WHERE scope = ?", [scope]))[0] as OperationRow | undefined) ?? null;
+  }
+
+  /** Starts an operation, or refuses naming the one that holds the scope. */
+  async start(scope: string, kind: string, plan: unknown, step: string): Promise<string> {
+    const opId = `op_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
+    const t = this.now();
+    const won = await this.q(
+      `INSERT INTO operation (scope, op_id, kind, plan, step, executor, executor_label, heartbeat_at, started_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (scope) DO NOTHING RETURNING op_id`,
+      [scope, opId, kind, JSON.stringify(plan), step, this.executor, this.label, t, t],
+    );
+    if (won.length === 1) return opId;
+    const held = await this.current(scope);
+    if (!held) throw new Error(`${scope}: the operation vanished while starting; run again`);
+    throw new OperationHeld(held);
+  }
+
+  /** Records the step about to run, and what's been learned; refuses unless this process still holds the operation. */
+  async step(scope: string, opId: string, step: string, progress?: unknown): Promise<void> {
+    const ok = await this.q(
+      `UPDATE operation SET step = ?, heartbeat_at = ?${progress === undefined ? "" : ", progress = ?"}
+       WHERE scope = ? AND op_id = ? AND executor = ? RETURNING op_id`,
+      [step, this.now(), ...(progress === undefined ? [] : [JSON.stringify(progress)]), scope, opId, this.executor],
+    );
+    if (ok.length !== 1) throw new LostOwnership(scope, step);
+  }
+
+  /** A clean stop: the operation stays, with nobody running it, for a resume. */
+  async stop(scope: string, opId: string): Promise<void> {
+    await this.q("UPDATE intent SET state = 'unknown' WHERE scope = ? AND op_id = ? AND state = 'open'", [scope, opId]);
+    await this.q(
+      "UPDATE operation SET executor = NULL, executor_label = NULL WHERE scope = ? AND op_id = ? AND executor = ?",
+      [scope, opId, this.executor],
+    );
+  }
+
+  /** Claims a stopped operation: of two resumes one wins, and a running one can't be resumed. */
+  async resume(scope: string, opId: string): Promise<OperationRow> {
+    const ok = await this.q(
+      "UPDATE operation SET executor = ?, executor_label = ?, heartbeat_at = ? WHERE scope = ? AND op_id = ? AND executor IS NULL RETURNING *",
+      [this.executor, this.label, this.now(), scope, opId],
+    );
+    if (ok.length === 1) return ok[0] as unknown as OperationRow;
+    const held = await this.current(scope);
+    if (!held || held.op_id !== opId) throw new Error(`${scope} has no operation ${opId}`);
+    throw new OperationHeld(held);
+  }
+
+  /**
+   * Takes over an operation whose executor is set, on someone's word that it
+   * was cancelled (--confirm-stopped). Its open intents become unknown: a call
+   * it had sent may still land, so the caller reconciles before acting.
+   */
+  async takeOver(scope: string, opId: string, observed: string | null): Promise<OperationRow> {
+    // From the executor the caller saw, and only from it: two runners recovering the same dead
+    // one can't both win, and the loser stops before it sends anything (codex code r8).
+    const ok = await this.q(
+      `UPDATE operation SET executor = ?, executor_label = ?, heartbeat_at = ?
+         WHERE scope = ? AND op_id = ? AND executor IS ? RETURNING *`,
+      [this.executor, this.label, this.now(), scope, opId, observed],
+    );
+    if (ok.length !== 1) {
+      const held = await this.current(scope);
+      if (!held || held.op_id !== opId) throw new Error(`${scope} has no operation ${opId}`);
+      throw new OperationHeld(held);
+    }
+    await this.q("UPDATE intent SET state = 'unknown' WHERE scope = ? AND op_id = ? AND state = 'open'", [scope, opId]);
+    return ok[0] as unknown as OperationRow;
+  }
+
+  /** A new plan under the same operation; the old one stays in its history (§6.2's --supersede). */
+  async supersede(scope: string, opId: string, plan: unknown, progress?: unknown): Promise<void> {
+    // The plan and what its progress may keep, in one statement: a crash between two writes would
+    // pair a new plan with the old plan's uploaded version (codex code r5).
+    const ok = await this.q(
+      `UPDATE operation SET plan = ?, heartbeat_at = ?${progress === undefined ? "" : ", progress = ?"}
+         WHERE scope = ? AND op_id = ? AND executor = ? RETURNING plan`,
+      [
+        JSON.stringify(plan),
+        this.now(),
+        ...(progress === undefined ? [] : [JSON.stringify(progress)]),
+        scope,
+        opId,
+        this.executor,
+      ],
+    );
+    if (ok.length !== 1) throw new LostOwnership(scope, "supersede");
+  }
+
+  /**
+   * Ends the operation, logged with every plan it ran under. Two requests, so each is safe to
+   * repeat: the log row is written only by the owner and only once, and a finish whose release
+   * was lost after its log landed releases on the next try instead of failing on the log's key
+   * and holding the store for good (codex code r1).
+   */
+  async finish(scope: string, opId: string, outcome: string, plans: readonly unknown[]): Promise<void> {
+    const row = await this.current(scope);
+    if (!row || row.op_id !== opId || row.executor !== this.executor) throw new LostOwnership(scope, "finish");
+    await this.q(
+      `INSERT INTO operation_log (op_id, scope, kind, plans, outcome, started_at, ended_at)
+         SELECT op_id, scope, kind, ?, ?, started_at, ? FROM operation WHERE scope = ? AND op_id = ? AND executor = ?
+         ON CONFLICT (op_id) DO NOTHING`,
+      [JSON.stringify(plans), outcome, this.now(), scope, opId, this.executor],
+    );
+    await this.q("DELETE FROM operation WHERE scope = ? AND op_id = ? AND executor = ?", [scope, opId, this.executor]);
+  }
+
+  // ---- intents (§3.1) ------------------------------------------------------------------------------------
+  async openIntent(
+    scope: string,
+    opId: string,
+    step: string,
+    kind: string,
+    tag: string,
+    detail: unknown,
+  ): Promise<string> {
+    const id = `int_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
+    // Recorded only by the operation's current executor: a runner that lost it sends nothing.
+    const ok = await this.q(
+      `INSERT INTO intent (id, op_id, scope, step, kind, tag, detail, state, created_at)
+         SELECT ?, ?, ?, ?, ?, ?, ?, 'open', ? WHERE EXISTS
+           (SELECT 1 FROM operation WHERE scope = ? AND op_id = ? AND executor = ?) RETURNING id`,
+      [id, opId, scope, step, kind, tag, JSON.stringify(detail), this.now(), scope, opId, this.executor],
+    );
+    if (ok.length !== 1) throw new LostOwnership(scope, step);
+    return id;
+  }
+
+  async closeIntent(
+    id: string,
+    state: "done" | "failed" | "unknown" | "not_executed",
+    resolvedBy: string,
+  ): Promise<void> {
+    await this.q("UPDATE intent SET state = ?, closed_at = ?, resolved_by = ? WHERE id = ?", [
+      state,
+      this.now(),
+      resolvedBy,
+      id,
+    ]);
+  }
+
+  /** Calls whose outcome nobody knows: nothing in the scope runs while one exists. */
+  async unresolved(scope: string): Promise<IntentRow[]> {
+    return (await this.q("SELECT * FROM intent WHERE scope = ? AND state IN ('open', 'unknown') ORDER BY created_at", [
+      scope,
+    ])) as unknown as IntentRow[];
+  }
+
+  async intents(scope: string, opId?: string): Promise<IntentRow[]> {
+    return (await this.q(`SELECT * FROM intent WHERE scope = ?${opId ? " AND op_id = ?" : ""} ORDER BY created_at`, [
+      scope,
+      ...(opId ? [opId] : []),
+    ])) as unknown as IntentRow[];
+  }
+
+  async intent(id: string): Promise<IntentRow | null> {
+    return ((await this.q("SELECT * FROM intent WHERE id = ?", [id]))[0] as IntentRow | undefined) ?? null;
+  }
+
+  // ---- deployments (§6.5) ----------------------------------------------------------------------------------
+  async deployments(): Promise<DeploymentRow[]> {
+    return (await this.q("SELECT * FROM deployment ORDER BY seq DESC")) as unknown as DeploymentRow[];
+  }
+
+  async addDeployment(opId: string, mode: string, before: unknown): Promise<string> {
+    const id = `dep_${randomUUID().replaceAll("-", "").slice(0, 20)}`;
+    await this.q(
+      `INSERT INTO deployment (id, seq, op_id, mode, before_half, created_at)
+       VALUES (?, (SELECT COALESCE(MAX(seq), 0) + 1 FROM deployment), ?, ?, ?, ?)`,
+      [id, opId, mode, JSON.stringify(before), this.now()],
+    );
+    return id;
+  }
+
+  async completeDeployment(id: string, after: unknown): Promise<void> {
+    await this.q("UPDATE deployment SET after_half = ?, completed_at = ? WHERE id = ?", [
+      JSON.stringify(after),
+      this.now(),
+      id,
+    ]);
+  }
+
+  // ---- applied migrations' hashes (§6.1) --------------------------------------------------------------------
+  async appliedHashes(): Promise<Map<string, string>> {
+    return new Map((await this.q("SELECT file, sha256 FROM applied")).map((r) => [String(r.file), String(r.sha256)]));
+  }
+
+  async recordApplied(file: string, sha256: string): Promise<void> {
+    await this.q("INSERT INTO applied (file, sha256, applied_at) VALUES (?, ?, ?) ON CONFLICT (file) DO NOTHING", [
+      file,
+      sha256,
+      this.now(),
+    ]);
+  }
+
+  // ---- the store's own resources (§6.1) ------------------------------------------------------------------
+  async resource(kind: string, name: string): Promise<{ id: string | null; state: string } | null> {
+    const [r] = await this.q("SELECT id, state FROM resource WHERE kind = ? AND name = ?", [kind, name]);
+    return r ? { id: (r.id as string | null) ?? null, state: String(r.state) } : null;
+  }
+
+  /** Recorded before the resource is used (a Worker before its first upload). */
+  async recordResource(kind: string, name: string, id: string | null, state: string): Promise<void> {
+    await this.q(
+      `INSERT INTO resource (kind, name, id, state, created_at) VALUES (?, ?, ?, ?, ?)
+       ON CONFLICT (kind, name) DO UPDATE SET id = COALESCE(excluded.id, resource.id), state = excluded.state`,
+      [kind, name, id, state, this.now()],
+    );
+  }
+
+  query(sql: string, params?: readonly unknown[]): Promise<Row[]> {
+    return this.q(sql, params);
+  }
+}

package/src/order.ts

@@ -0,0 +1,7 @@
+/**
+ * Strings in code-point order. `localeCompare` follows the machine's ICU locale,
+ * which orders `_` differently under C.UTF-8 than under en_US, so a generated
+ * file or a hash sorted with it differs between two machines and `generate
+ * --check` fails on the one that didn't write it.
+ */
+export const byCodePoint = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);

package/src/plugin-lint.ts

@@ -0,0 +1,145 @@
+// Lints everything a plugin will run as SQL, not only its migration files:
+// acquire bodies, checkout reads and statements, and release handlers. The
+// functions that build runtime SQL are called with a representative input,
+// since the linter needs their text, not their values.
+import type { PluginDefinition, ScheduleDescriptor, SqlStatement } from "@kedge/core";
+import {
+  entityTable,
+  entityWriteProblems,
+  errorSources,
+  isTimerDescriptor,
+  type LintIssue,
+  type LintOptions,
+  lintSql,
+  type Owner,
+  trackSql,
+} from "@kedge/core";
+
+type Context = LintOptions["context"];
+
+const sample = {
+  cartId: "cart_lint",
+  email: "lint@example.com",
+  customerId: "cus_lint",
+  currency: "USD",
+  variantIds: ["var_lint"],
+};
+
+/** The SQL a plugin builds at runtime, from sample inputs, with where each came from. */
+export function collect<O>(def: PluginDefinition<O>, options: O): { sql: string; context: Context; where: string }[] {
+  const out: { sql: string; context: Context; where: string }[] = [];
+  // Timer descriptors beside a contribution's SQL are core's to write (p0.6a §3.2): nothing to lint.
+  const push = (where: string, context: Context, ss: readonly (SqlStatement | ScheduleDescriptor)[] | undefined) => {
+    for (const s of ss ?? []) if (!isTimerDescriptor(s)) out.push({ sql: s.sql, context, where });
+  };
+  for (const [key, body] of Object.entries(def.acquire ?? {}))
+    out.push({ sql: body as string, context: "acquire", where: `acquire ${key}` });
+  out.push(...trackSql(def, options));
+  const c = def.checkout;
+  if (c?.reads) push("checkout.reads", "read", Object.values(c.reads(sample, options)));
+  if (c?.statements) {
+    // Cite every declared fact under a sample key, so conditional statements are produced.
+    const facts = Object.fromEntries(
+      Object.keys(c.facts ?? {}).map((n) => [n, new Proxy({}, { get: () => "cfa_lint" })]),
+    );
+    push(
+      "checkout.statements",
+      "contribution",
+      c.statements({
+        orderId: "ord_lint",
+        email: sample.email,
+        customerId: sample.customerId,
+        currency: sample.currency,
+        options,
+        claim: {},
+        lines: [
+          {
+            id: "ordl_lint",
+            variantId: "var_lint",
+            sku: "S",
+            title: "T",
+            quantity: 1,
+            unitAmount: 100,
+            lineTotal: 100,
+            kind: "item",
+            parentId: null,
+            chargeClass: "revenue",
+            taxable: true,
+            discountable: true,
+            periodStart: null,
+            periodEnd: null,
+            custom: {},
+          },
+        ],
+        facts,
+        fees: [{ code: "lint", amount: 1 }],
+        adjustments: [{ code: "lint", amount: -1 }],
+      }),
+    );
+  }
+  for (const [key, handler] of Object.entries(def.release ?? {})) {
+    if (!handler) continue;
+    push(
+      `release ${key}`,
+      "statement",
+      handler(
+        {
+          uid: "ord_lint:expired",
+          aggregate: "order",
+          aggregateId: "ord_lint",
+          fromState: "pending_payment",
+          toState: "expired",
+        },
+        options,
+      ),
+    );
+  }
+  return out;
+}
+
+export function lintPlugin<O>(def: PluginDefinition<O>, options: O): LintIssue[] {
+  const owner: Owner = { kind: "plugin", name: def.name };
+  const issues: LintIssue[] = [];
+  const texts: string[] = [];
+  for (const m of def.migrations ?? []) {
+    texts.push(m.sql);
+    for (const i of lintSql(owner, m.sql, { context: "migration" })) issues.push({ ...i, at: `${m.id}: ${i.at}` });
+  }
+  let runtime: ReturnType<typeof collect>;
+  try {
+    runtime = collect(def, options);
+  } catch (e) {
+    issues.push({
+      rule: "unlintable",
+      message: `building the plugin's SQL with sample input threw: ${(e as Error).message}`,
+      at: def.name,
+    });
+    runtime = [];
+  }
+  for (const r of runtime) {
+    texts.push(r.sql);
+    const factTables = Object.values(def.checkout?.facts ?? {}).map((f) => f.table);
+    for (const i of lintSql(owner, r.sql, { context: r.context, factTables }))
+      issues.push({ ...i, at: `${r.where}: ${i.at}` });
+  }
+  // Only core writes an entity table (§4.2), from every SQL source a plugin has:
+  // a migration's trigger, an acquire body core installs as a trigger, a
+  // consequence, a release (codex code r6).
+  const entityTables = Object.keys(def.entities ?? {}).map((e) => entityTable(def.name, e));
+  const sources = [
+    ...(def.migrations ?? []).map((m) => ({ where: `migration ${m.id}`, sql: m.sql })),
+    ...runtime.map((r) => ({ where: r.where, sql: r.sql })),
+  ];
+  for (const s of sources)
+    for (const message of entityWriteProblems(s.sql, entityTables))
+      issues.push({ rule: "entity-write", message, at: s.where });
+
+  // The error-map check: every named CHECK and RAISE text maps to a code.
+  const mapped = new Set(Object.keys(def.errors ?? {}));
+  const { checks, raises } = errorSources(texts.join(";\n"));
+  for (const key of [...checks, ...raises]) {
+    if (!mapped.has(key))
+      issues.push({ rule: "error-map", message: `"${key}" has no entry in the plugin's error map`, at: def.name });
+  }
+  return issues;
+}