@intentius/chant 0.91.0 → 0.94.0

package/src/op/operator.ts

@@ -38,9 +38,12 @@
 import type { ActivityFn, ActivityProfile } from "./activity-registry";
 import { discoverOps, type DiscoveredOp } from "./discover";
 import { runOpLocally, OpRunFailure, type OpRunResult } from "./local-executor";
-import { acquireLease, stillHoldsLease, currentHolderId, DEFAULT_LEASE_TTL_MS, type LeaseRecord, type AcquireLeaseResult } from "../lifecycle/lease";
-import { runEnvOf } from "../lifecycle/run-ledger";
-import { stewardLeaseName, type StewardDeclaration } from "./steward";
+import { acquireLease, releaseLease, stillHoldsLease, currentHolderId, DEFAULT_LEASE_TTL_MS, type LeaseRecord, type AcquireLeaseResult } from "../lifecycle/lease";
+import { randomUUID } from "node:crypto";
+import { readRunLedger, runEnvOf } from "../lifecycle/run-ledger";
+import { readGateLedger, latestResolutionSince } from "../lifecycle/gate-ledger";
+import { enterStewardTurn } from "./steward-turn";
+import { stewardLeaseName, stewardTurnLeaseName, type StewardDeclaration } from "./steward";
 import { stewardWorkHolder } from "./work-lease-run";
 import { StaleLockError } from "../lifecycle/git";
 import { cronMatches, cronDueBetween } from "./cron";
@@ -79,7 +82,36 @@ export async function discoverConvergeOps(
 
 /** One discovered ConvergeOp's outcome for one round — what `chant operator`'s one-line-per-tick log (and its tests) key off of. */
 export type OperatorTickEvent =
-  | { kind: "ticked"; op: string; env: string; result: OpRunResult }
+  /**
+   * The Op ran. `resumed` names the decision point question it was waiting on
+   * (#2749) when a steward ran it because that question is now answered,
+   * rather than because its cron fired.
+   */
+  | {
+      kind: "ticked";
+      op: string;
+      env: string;
+      result: OpRunResult;
+      resumed?: string;
+      /**
+       * The gate the Op's last run stopped at (#2779), when a steward ran it
+       * because that gate is now approved rather than because its cron fired.
+       */
+      approved?: { op: string; gate: string };
+    }
+  /**
+   * A steward's Op whose last run stopped on an open decision point (#2749),
+   * and the question is still open, so the Op is not run this round. A person
+   * answers it through hud or `points answer`; the steward never does.
+   */
+  | { kind: "waiting-on-point"; op: string; env: string; point: string; question: string; state: string }
+  /**
+   * A steward's Op whose last run stopped at a gate (#2779), and nobody has
+   * approved it since, so the Op is not run this round. `gateOp` is the op
+   * the gate is recorded under: the Op's own, or a command's such as
+   * `workspace-upgrade`.
+   */
+  | { kind: "waiting-on-gate"; op: string; env: string; gateOp: string; gate: string }
   | { kind: "skipped-lease-held"; op: string; env: string; heldBy?: string }
   /** The Op declares its own `schedule.cron` (#2120) and this round did not land on a firing minute — the lease was never touched. An Op without a cron is never reported this way: it ticks every round, on `--interval`. */
   | { kind: "skipped-not-due"; op: string; env: string; cron: string }
@@ -104,7 +136,17 @@ export type OperatorTickEvent =
    * steward. Nothing ticked. `op` is the steward's lease name and `env` is
    * `-`, so a log line keyed on either still reads.
    */
-  | { kind: "steward-busy"; op: string; env: string; steward: string; heldBy?: string };
+  | { kind: "steward-busy"; op: string; env: string; steward: string; heldBy?: string }
+  /**
+   * A local steward's round (#2750) found its turn lease
+   * (`stewardTurnLeaseName`) held by someone else already mid-run — a `chant
+   * run <op>` typed by hand, or another round somehow still finishing one.
+   * Nothing ticked: the round stops here rather than trying its remaining
+   * ops, since every one of them would fail the same check (the turn is one
+   * lease for the whole steward, not one per op). The next round tries
+   * again.
+   */
+  | { kind: "turn-busy"; op: string; env: string; steward: string; heldBy?: string };
 
 export interface OperatorRoundOptions {
   cwd?: string;
@@ -137,6 +179,98 @@ export interface OperatorRoundOptions {
    * lease, and ticks nothing when another holder has it.
    */
   steward?: StewardDeclaration;
+  /**
+   * How a steward's round reads the state of the questions its waiting runs
+   * stopped on (#2749): question id to state, or null when they can't be
+   * read. Defaults to the workspace's `points` read; a test injects one.
+   */
+  readQuestions?: (cwd: string) => Promise<Map<string, string> | null>;
+}
+
+/** A steward's Op whose last run waits on a decision point (#2749). */
+interface WaitState {
+  /** The answer record's id. */
+  question: string;
+  point: string;
+  /** The question's state now, or the one the run recorded when it can't be read. */
+  state: string;
+  /** The question is answered, or its record is gone, so the Op runs again. */
+  answered: boolean;
+}
+
+/**
+ * The steward's Ops whose newest run is the steward's own and stopped on a
+ * decision point, with that question's state now. The questions are read
+ * from the workspace the way `points` reads them; a workspace that can't be
+ * read leaves every run waiting.
+ */
+async function stewardWaits(steward: StewardDeclaration, opts: OperatorRoundOptions): Promise<Map<string, WaitState>> {
+  const out = new Map<string, WaitState>();
+  const runs: { op: string; question: string; point: string; state: string }[] = [];
+  for (const op of steward.ops) {
+    try {
+      const newest = (await readRunLedger(runEnvOf(op), op.name, { cwd: opts.cwd })).records.at(-1);
+      if (newest?.status === "waiting" && newest.point && newest.steward === steward.name) {
+        runs.push({ op: op.name, question: newest.point.id, point: newest.point.point, state: newest.point.state });
+      }
+    } catch {
+      // A ledger that can't be read has no waiting run to resume.
+    }
+  }
+  if (runs.length === 0) return out;
+  const states = await (opts.readQuestions ?? readQuestionStates)(opts.cwd ?? process.cwd());
+  for (const r of runs) {
+    const now = states?.get(r.question);
+    const answered = states !== null && (now === undefined || now === "answered");
+    out.set(r.op, { question: r.question, point: r.point, state: now ?? r.state, answered });
+  }
+  return out;
+}
+
+/** A steward's Op whose last run stopped at a gate (#2779). */
+interface GateWaitState {
+  /** The op the gate is recorded under. */
+  gateOp: string;
+  gate: string;
+  /** Someone approved the gate since the run stopped there, so the Op runs again. */
+  approved: boolean;
+}
+
+/**
+ * The steward's Ops whose newest run is the steward's own and stopped at a
+ * gate, and whether the gate has been approved since. A gate ledger that
+ * can't be read leaves the run where it stopped.
+ */
+async function stewardGates(steward: StewardDeclaration, opts: OperatorRoundOptions): Promise<Map<string, GateWaitState>> {
+  const out = new Map<string, GateWaitState>();
+  for (const op of steward.ops) {
+    try {
+      const newest = (await readRunLedger(runEnvOf(op), op.name, { cwd: opts.cwd })).records.at(-1);
+      if (newest?.status !== "gated" || !newest.gate || newest.steward !== steward.name) continue;
+      const gateOp = newest.gate.op ?? op.name;
+      const { resolutions } = await readGateLedger(gateOp, { cwd: opts.cwd });
+      const approved = latestResolutionSince(resolutions, newest.gate.name, newest.gate.since) !== undefined;
+      out.set(op.name, { gateOp, gate: newest.gate.name, approved });
+    } catch {
+      // A ledger that can't be read has no gated run to resume.
+    }
+  }
+  return out;
+}
+
+/** Every question's state in the workspace, by id, or null when the workspace's points can't be read. */
+async function readQuestionStates(cwd: string): Promise<Map<string, string> | null> {
+  try {
+    const { workspacePoints } = await import("../workspace/points-cli");
+    const doc = await workspacePoints({ cwd });
+    if (!("questions" in doc)) return null;
+    // An answer kind that could not be read might hold the question: say
+    // nothing rather than resume a run whose question may still be open.
+    if (doc.sources.some((s) => s.reason !== null)) return null;
+    return new Map(doc.questions.map((q) => [q.id, q.state]));
+  } catch {
+    return null;
+  }
 }
 
 /** Take or renew a local steward's own lease (#2731). */
@@ -148,9 +282,66 @@ export async function acquireStewardLease(
   return acquireLease(stewardLeaseName(steward), holder, { ...opts, ttlMs: opts.ttlMs ?? DEFAULT_LEASE_TTL_MS });
 }
 
-/** The Ops one round considers: a steward's scheduled ones, or every ConvergeOp. */
+/**
+ * How long a caller contending for a steward's turn lease
+ * (`acquireStewardTurn`) retries before giving up (#2750). Long enough to
+ * ride out a race with a turn that is just ending; short enough that a
+ * genuinely busy steward is reported back promptly. A round never waits
+ * (`waitMs` omitted there): it already runs on its own timer and simply
+ * tries the turn again next round, exactly as it already does for a busy
+ * per-op lease. `chant run <op>` (`../cli/handlers/run.ts`) is the caller
+ * that waits, since a human is standing at the terminal for the answer.
+ */
+export const STEWARD_TURN_WAIT_MS = 2_000;
+
+/** How often {@link acquireStewardTurn} retries while it waits. */
+const STEWARD_TURN_POLL_MS = 150;
+
+/**
+ * Take the steward's turn lease (#2750): the lease every run of one of its
+ * Ops — a round's scheduled tick, or `chant run <op>` typed by hand — holds
+ * for exactly as long as that one run takes, so the two are never beside each
+ * other. Distinct from {@link acquireStewardLease}, which one `chant operator
+ * --steward` process holds for its whole life: this one is held per-run, so a
+ * hand run succeeds the moment a round's tick ends, not only once the daemon
+ * itself stops.
+ *
+ * Contention never queues past `waitMs` (0 by default): consistent with a
+ * round's own leases, which skip and report rather than wait (see this
+ * module's doc), and with the fountain form, whose busy teammate is likewise
+ * refused rather than retried. A caller that wants to ride out a race with a
+ * turn about to end passes `waitMs`, and this polls every
+ * {@link STEWARD_TURN_POLL_MS} until it acquires or the wait runs out.
+ */
+export async function acquireStewardTurn(
+  steward: string,
+  holder: string,
+  opts: { cwd?: string; ttlMs?: number; now?: () => Date; waitMs?: number } = {},
+): Promise<AcquireLeaseResult> {
+  const deadline = Date.now() + (opts.waitMs ?? 0);
+  for (;;) {
+    const result = await acquireLease(stewardTurnLeaseName(steward), holder, {
+      cwd: opts.cwd,
+      ttlMs: opts.ttlMs ?? DEFAULT_LEASE_TTL_MS,
+      now: opts.now,
+    });
+    if (result.acquired || result.reason !== "held" || Date.now() >= deadline) return result;
+    await new Promise((r) => setTimeout(r, STEWARD_TURN_POLL_MS));
+  }
+}
+
+/** Release a steward's turn lease, best-effort — a run whose turn was never actually acquired (or already lost) has nothing to release. */
+async function releaseStewardTurn(steward: string, holder: string, token: string, opts: { cwd?: string } = {}): Promise<void> {
+  await releaseLease(stewardTurnLeaseName(steward), holder, token, opts).catch(() => false);
+}
+
+/**
+ * The Ops one round considers: a steward's Ops, or every ConvergeOp. A
+ * steward's Op with no schedule is considered only to resume a run of the
+ * steward's that waits on a decision point.
+ */
 async function roundOps(opts: OperatorRoundOptions): Promise<DiscoveredOp["config"][]> {
-  if (opts.steward) return opts.steward.ops.filter((op) => op.schedule !== undefined);
+  if (opts.steward) return [...opts.steward.ops];
   const { ops } = await discoverConvergeOps({ cwd: opts.cwd, env: opts.env });
   return ops.map((d) => d.config);
 }
@@ -197,8 +388,32 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
 
   if (!(await holdSteward())) return events;
 
+  // A steward's Ops whose last run waits on a decision point (#2749), read
+  // once per round, and the state of those questions now.
+  const waiting = steward ? await stewardWaits(steward, opts) : new Map<string, WaitState>();
+  // And those whose last run stopped at a gate (#2779), with whether it has
+  // been approved since.
+  const gated = steward ? await stewardGates(steward, opts) : new Map<string, GateWaitState>();
+
   for (const config of await roundOps(opts)) {
     const env = steward ? runEnvOf(config) : (envOf(config) ?? "unknown");
+    const wait = waiting.get(config.name);
+    const stopped = gated.get(config.name);
+    let resumed: string | undefined;
+    let approved: { op: string; gate: string } | undefined;
+
+    // A run waiting on a question that is now answered is resumed on this
+    // round, whatever its cron says. One whose question is still open is left
+    // until a person answers it, unless its cron fires, which asks again.
+    if (wait?.answered) resumed = wait.question;
+    // The same for a run stopped at a gate that is now approved (#2779).
+    if (stopped?.approved) approved = { op: stopped.gateOp, gate: stopped.gate };
+    const standing: OperatorTickEvent | undefined = wait
+      ? { kind: "waiting-on-point", op: config.name, env, point: wait.point, question: wait.question, state: wait.state }
+      : stopped
+        ? { kind: "waiting-on-gate", op: config.name, env, gateOp: stopped.gateOp, gate: stopped.gate }
+        : undefined;
+    const resuming = resumed !== undefined || approved !== undefined;
 
     // An Op that declares its own cadence (#2120) is ticked on that cron
     // rather than on every round. Level-triggered: the question is whether a
@@ -212,10 +427,15 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
       const lastSeen = opts.scheduleState?.get(config.name);
       const due = lastSeen === undefined ? cronMatches(cron, now) : cronDueBetween(cron, lastSeen, now);
       opts.scheduleState?.set(config.name, now);
-      if (!due) {
-        events.push({ kind: "skipped-not-due", op: config.name, env, cron });
+      if (!due && !resuming) {
+        events.push(standing ?? { kind: "skipped-not-due", op: config.name, env, cron });
         continue;
       }
+    } else if (steward && !resuming) {
+      // An unscheduled Op runs when someone asks for it; the steward runs it
+      // only to resume its own waiting or gated run.
+      if (standing) events.push(standing);
+      continue;
     }
 
     let acquired: AcquireLeaseResult;
@@ -242,8 +462,44 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
     }
 
     const lease = acquired.lease as LeaseRecord;
+
+    // A local steward is one turn at a time (#2750): a hand run (`chant run
+    // <op>`) holds this same lease for as long as it runs, so a round never
+    // ticks beside one. It is one lease for the whole steward, not one per
+    // op, so finding it held stops the round outright — every op still ahead
+    // of it in `roundOps` would fail the identical check.
+    let turn: AcquireLeaseResult | undefined;
+    if (steward) {
+      try {
+        turn = await acquireStewardTurn(steward.name, holder, { cwd: opts.cwd, ttlMs: opts.leaseTtlMs, now: opts.now });
+      } catch (err) {
+        // Not turn contention — the acquire attempt itself failed (most
+        // likely a StaleLockError). Same shape as the per-op acquire's own
+        // catch above: report it and stop, rather than let it propagate and
+        // leak the per-op lease this round just took.
+        await releaseLease(config.name, holder, lease.token, { cwd: opts.cwd }).catch(() => false);
+        const message = err instanceof StaleLockError ? err.message : err instanceof Error ? err.message : String(err);
+        events.push({ kind: "lease-error", op: stewardTurnLeaseName(steward.name), env, error: message });
+        break;
+      }
+      if (!turn.acquired) {
+        await releaseLease(config.name, holder, lease.token, { cwd: opts.cwd }).catch(() => false);
+        events.push({ kind: "turn-busy", op: config.name, env, steward: steward.name, heldBy: turn.heldBy?.holder });
+        break;
+      }
+    }
+
+    // The run is the steward's turn (#2749): a decision point it asks names the
+    // steward and makes its model call through the steward's broker, and an
+    // answer from inside the turn is refused.
+    const runId = randomUUID();
+    const restoreTurn = steward
+      ? enterStewardTurn({ steward: steward.name, capabilities: steward.capabilities, vault: steward.vault, run: runId })
+      : undefined;
     try {
       const result = await runOpLocally(config, opts.activities, opts.profiles, opts.signal, {
+        runId,
+        ...(steward ? { steward: steward.name } : {}),
         ledger: { cwd: opts.cwd },
         // A steward's turn claims work leases as `<steward>/<op>@<holder>`
         // (#2748), which is how `workspace status` finds the lease it holds.
@@ -263,7 +519,18 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
           }),
       });
       const held = await stillHoldsLease(config.name, holder, lease.token, { cwd: opts.cwd });
-      events.push(held ? { kind: "ticked", op: config.name, env, result } : { kind: "fenced", op: config.name, env });
+      events.push(
+        held
+          ? {
+              kind: "ticked",
+              op: config.name,
+              env,
+              result,
+              ...(resumed !== undefined ? { resumed } : {}),
+              ...(approved !== undefined ? { approved } : {}),
+            }
+          : { kind: "fenced", op: config.name, env },
+      );
     } catch (err) {
       // #2301: "see its ledger record" was the whole message, and it sent the
       // reader to an artifact that a failed ledger append means is missing —
@@ -277,6 +544,9 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
             ? err.message
             : String(err);
       events.push({ kind: "tick-failed", op: config.name, env, error: message });
+    } finally {
+      restoreTurn?.();
+      if (steward && turn?.acquired) await releaseStewardTurn(steward.name, holder, turn.lease!.token, { cwd: opts.cwd });
     }
     if (!(await holdSteward())) break;
   }
@@ -488,7 +758,14 @@ export function formatRoundLine(event: OperatorTickEvent): string {
   switch (event.kind) {
     case "ticked":
       return `operator: ${event.op}@${event.env} ticked=1 status=${event.result.status}` +
-        (event.result.gate ? ` gate="${event.result.gate.gate}"` : "");
+        (event.result.gate ? ` gate="${event.result.gate.gate}"` : "") +
+        (event.result.point ? ` point="${event.result.point.id}"` : "") +
+        (event.resumed ? ` resumed="${event.resumed}"` : "") +
+        (event.approved ? ` approved="${event.approved.op}/${event.approved.gate}"` : "");
+    case "waiting-on-point":
+      return `operator: ${event.op}@${event.env} waiting=1(point:${event.question}:${event.state})`;
+    case "waiting-on-gate":
+      return `operator: ${event.op}@${event.env} gated=1(gate:${event.gateOp}/${event.gate})`;
     case "skipped-lease-held":
       return `operator: ${event.op}@${event.env} skipped=1(lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
     case "skipped-not-due":
@@ -501,5 +778,7 @@ export function formatRoundLine(event: OperatorTickEvent): string {
       return `operator: ${event.op}@${event.env} error=1(lease acquire failed — ${event.error})`;
     case "steward-busy":
       return `operator: steward ${event.steward} skipped=1(steward-lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
+    case "turn-busy":
+      return `operator: ${event.op}@${event.env} skipped=1(turn-held${event.heldBy ? `:${event.heldBy}` : ""}, steward "${event.steward}" is already mid-turn)`;
   }
 }

package/src/op/runtime.ts

@@ -18,23 +18,27 @@
 
 import type { OpConfig } from "./types";
 import type { StepRecord, OpRunResult } from "./local-executor";
+import type { WaitingPoint } from "./steward-points";
 import type { GateResolutionRecord } from "../lifecycle/gate-ledger";
 import type { RunComponentsOptions, RunComponentsResult } from "../components/cli-support";
 
 /**
  * How a run ended, or that it has not. `gated` is the state a run reaches when
- * a gate has no recorded resolution — a fact, not a wait (#2119).
+ * a gate has no recorded resolution — a fact, not a wait (#2119). `waiting` is
+ * the state it reaches when an activity asked a decision point that is still
+ * open (#2749), a fact of the same kind: a person answers the question, and
+ * the next run reads the answer.
  *
  * This is the coarse state a *runtime* reports. The ledger records the
  * executor's own three-state outcome instead ({@link OpRunRecord.status}),
  * which has no `running` or `cancelled`: a record is only written once a run
  * has settled.
  */
-export type OpRunState = "running" | "completed" | "failed" | "gated" | "cancelled";
+export type OpRunState = "running" | "completed" | "failed" | "gated" | "waiting" | "cancelled";
 
 /** Map a settled run's ledger status onto the coarser runtime state. */
 export function runStateOf(status: OpRunRecord["status"]): OpRunState {
-  return status === "ok" ? "completed" : status === "gated" ? "gated" : "failed";
+  return status === "ok" ? "completed" : status === "gated" ? "gated" : status === "waiting" ? "waiting" : "failed";
 }
 
 /** One step's outcome as the run ledger keeps it — `StepRecord` minus the fields only a live renderer needs. */
@@ -51,6 +55,10 @@ export interface OpRunStepRecord {
   error?: string;
   /** Why a gate declined a standing approval that was for another plan (#2300) — see {@link StepRecord.refusal}. */
   refusal?: string;
+  /** The open decision point this step asked (#2749), for a step skipped because it is waiting on one. */
+  point?: WaitingPoint;
+  /** The gate this step's command stopped at (#2779), for a step skipped because of it. */
+  gate?: { op: string; gate: string };
 }
 
 /** One phase's steps and the verdict they add up to. */
@@ -81,8 +89,10 @@ export interface OpRunStatus {
   records?: StepRecord[];
   /** The executor's own result, when the runtime is executing in this process. */
   result?: OpRunResult;
-  /** The gate this run is waiting on, when `state` is `gated`. */
-  gate?: { name: string; since: string };
+  /** The gate this run is waiting on, when `state` is `gated`; `op` names the op it is recorded under when that isn't the run's own (#2779). */
+  gate?: { name: string; since: string; op?: string };
+  /** The open decision point this run is waiting on, when `state` is `waiting` (#2749). */
+  point?: WaitingPoint & { since: string };
   /** Free-text detail for a failed run. */
   error?: string;
 }
@@ -117,15 +127,28 @@ export interface OpRunRecord {
    * failure: the run reached a gate nobody has approved, recorded the fact,
    * and stopped (#2119).
    */
-  status: "ok" | "fail" | "gated";
+  status: "ok" | "fail" | "gated" | "waiting";
   /** The Op's own `labels`, copied at run time. */
   labels: Record<string, string>;
   /** Every `outcomeAttribute` the run captured, name → value. */
   outcomes: Record<string, unknown>;
   /** Per-phase, per-step status, in execution order. */
   phases: OpRunPhaseRecord[];
-  /** The gate the run stopped on, for `status: "gated"`. */
-  gate?: { name: string; since: string };
+  /**
+   * The gate the run stopped on, for `status: "gated"`. `op` is set when the
+   * gate is recorded under another op than the run's (#2779): a step whose
+   * command stopped at its own gate, such as `chant workspace upgrade`'s
+   * `workspace-upgrade` / `<scope>`.
+   */
+  gate?: { name: string; since: string; op?: string };
+  /**
+   * The open decision point the run stopped on, for `status: "waiting"`
+   * (#2749): the answer record's id, the point, its state then, and when the
+   * run stopped. `points --open` and hud read the question itself.
+   */
+  point?: WaitingPoint & { since: string };
+  /** The steward whose turn the run was, when a steward ran it (#2749). */
+  steward?: string;
 }
 
 /** An {@link OpRunRecord} before the ledger stamps its version and mints an id. */

package/src/op/runtimes/local.ts

@@ -52,7 +52,8 @@ function statusFrom(
     endedAt: new Date().toISOString(),
     records: result.records,
     result,
-    ...(result.gate ? { gate: { name: result.gate.gate, since: result.gate.timestamp } } : {}),
+    ...(result.gate ? { gate: { name: result.gate.gate, since: result.gate.timestamp, ...(result.gate.op !== op ? { op: result.gate.op } : {}) } } : {}),
+    ...(result.record.point ? { point: result.record.point } : {}),
   };
 }
 
@@ -70,6 +71,7 @@ function statusFromRecord(record: OpRunRecord): OpRunStatus {
     startedAt: record.started,
     endedAt: record.ended,
     ...(record.gate ? { gate: record.gate } : {}),
+    ...(record.point ? { point: record.point } : {}),
   };
 }
 

package/src/op/source-archive.ts

@@ -0,0 +1,161 @@
+/**
+ * A source tree as a release artifact (#2782): one directory of a commit,
+ * archived with `git archive`, and read back only when its bytes still hash
+ * to the digest a gate approved.
+ *
+ * `git archive <commit> -- <dir>` gives every entry the commit's own
+ * timestamp, so the same commit always archives to the same bytes, and the
+ * archive's sha256 names the tree exactly. A release plan carries that digest,
+ * the ship gate approves the plan, and whatever ships the release reads the
+ * archive through {@link readSourceArchive}, which refuses bytes that do not
+ * hash to the approved digest.
+ *
+ * Pure except for the git and file reads each function names.
+ */
+
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { dirname, isAbsolute, join, relative, resolve } from "node:path";
+
+/** One file of an archived tree. */
+export interface SourceFile {
+  /** Its path inside the archived directory, with `/` separators. */
+  path: string;
+  /** Whether git records it executable. */
+  executable: boolean;
+  data: Buffer;
+}
+
+/** A directory of a commit, archived. */
+export interface SourceArchive {
+  /** `sha256:<hex>` of the archive's bytes. */
+  digest: string;
+  /** Where the archive was written. */
+  archive: string;
+  /** The commit it was archived from. */
+  commit: string;
+  /** The archived directory, relative to the repository root (`.` for the root). */
+  dir: string;
+  /** Files in it. */
+  files: number;
+  /** Size of the archive in bytes. */
+  bytes: number;
+}
+
+export const sha256Digest = (data: Buffer | string): string => `sha256:${createHash("sha256").update(data).digest("hex")}`;
+
+function git(cwd: string, args: string[]): Buffer {
+  return execFileSync("git", args, { cwd, maxBuffer: 1024 * 1024 * 1024, stdio: ["ignore", "pipe", "pipe"] });
+}
+
+/**
+ * Archive `dir` (relative to `cwd`) as committed at `ref` (default HEAD) to
+ * `out` (default `<cwd>/dist/releases/<commit>.tar`). Uncommitted changes are
+ * not in it: a release ships what is committed. Running it again for the same
+ * commit writes the same bytes.
+ */
+export function archiveSourceTree(opts: { dir: string; ref?: string; out?: string; cwd?: string }): SourceArchive {
+  const cwd = resolve(opts.cwd ?? process.cwd());
+  // git names the root by its real path, so the directory is resolved the same way.
+  const root = realpathSync(git(cwd, ["rev-parse", "--show-toplevel"]).toString("utf-8").trim());
+  const commit = git(cwd, ["rev-parse", "--verify", `${opts.ref ?? "HEAD"}^{commit}`]).toString("utf-8").trim();
+  const target = resolve(realpathSync(cwd), opts.dir);
+  const dir = relative(root, target).split("\\").join("/") || ".";
+  if (dir.startsWith("..") || isAbsolute(dir)) throw new Error(`${opts.dir} is outside the repository at ${root}`);
+  const spec = dir === "." ? commit : `${commit}:${dir}`;
+  try {
+    git(root, ["cat-file", "-e", spec]);
+  } catch {
+    throw new Error(`${dir} is not in commit ${commit.slice(0, 12)}: commit it before releasing it`);
+  }
+  const data = git(root, ["archive", "--format=tar", commit, "--", dir]);
+  const archive = resolve(cwd, opts.out ?? join("dist", "releases", `${commit}.tar`));
+  mkdirSync(dirname(archive), { recursive: true });
+  writeFileSync(archive, data);
+  const files = tarEntries(data).filter((e) => e.type === "file").length;
+  return { digest: sha256Digest(data), archive, commit, dir, files, bytes: data.length };
+}
+
+interface TarEntry {
+  type: "file" | "dir" | "symlink" | "other";
+  path: string;
+  mode: number;
+  data: Buffer;
+}
+
+const field = (block: Buffer, at: number, len: number): string => {
+  const raw = block.subarray(at, at + len);
+  const end = raw.indexOf(0);
+  return raw.subarray(0, end < 0 ? raw.length : end).toString("utf-8");
+};
+const octal = (block: Buffer, at: number, len: number): number => parseInt(field(block, at, len).trim() || "0", 8);
+
+/** The records of a pax header: `<len> key=value\n`. */
+function paxRecords(data: Buffer): Record<string, string> {
+  const out: Record<string, string> = {};
+  let at = 0;
+  while (at < data.length) {
+    const space = data.indexOf(0x20, at);
+    if (space < 0) break;
+    const len = parseInt(data.subarray(at, space).toString("utf-8"), 10);
+    if (!Number.isFinite(len) || len <= 0) break;
+    const record = data.subarray(space + 1, at + len - 1).toString("utf-8");
+    const eq = record.indexOf("=");
+    if (eq > 0) out[record.slice(0, eq)] = record.slice(eq + 1);
+    at += len;
+  }
+  return out;
+}
+
+/** The entries of a ustar/pax archive, as `git archive` writes one. */
+function tarEntries(tar: Buffer): TarEntry[] {
+  const out: TarEntry[] = [];
+  let at = 0;
+  let pax: Record<string, string> = {};
+  while (at + 512 <= tar.length) {
+    const header = tar.subarray(at, at + 512);
+    if (header.every((b) => b === 0)) break;
+    const size = octal(header, 124, 12);
+    const flag = String.fromCharCode(header[156] || 0x30);
+    const data = tar.subarray(at + 512, at + 512 + size);
+    at += 512 + Math.ceil(size / 512) * 512;
+    if (flag === "x") {
+      pax = paxRecords(data);
+      continue;
+    }
+    if (flag === "g") continue;
+    const prefix = field(header, 345, 155);
+    const name = pax.path ?? (prefix ? `${prefix}/${field(header, 0, 100)}` : field(header, 0, 100));
+    pax = {};
+    const type = flag === "0" || flag === "\0" || flag === "7" ? "file" : flag === "5" ? "dir" : flag === "2" ? "symlink" : "other";
+    out.push({ type, path: name, mode: octal(header, 100, 8), data: Buffer.from(data) });
+  }
+  return out;
+}
+
+/**
+ * The files of an archive `archiveSourceTree` wrote, relative to the directory
+ * it archived, once its bytes hash to `digest`. Throws, naming both digests,
+ * when they do not: whatever would ship them is not what was approved.
+ *
+ * A symbolic link in the tree is refused rather than followed or dropped.
+ */
+export function readSourceArchive(opts: { archive: string; digest: string; dir?: string }): SourceFile[] {
+  const data = readFileSync(opts.archive);
+  const actual = sha256Digest(data);
+  if (actual !== opts.digest) {
+    throw new Error(`${opts.archive} is ${actual}, not the approved ${opts.digest}: it is not the tree the release was planned from`);
+  }
+  const entries = tarEntries(data);
+  const dir = opts.dir && opts.dir !== "." ? `${opts.dir.replace(/\/+$/, "")}/` : "";
+  const files: SourceFile[] = [];
+  for (const e of entries) {
+    if (e.type === "dir") continue;
+    if (!e.path.startsWith(dir)) throw new Error(`${opts.archive} holds ${e.path}, outside ${dir || "its root"}`);
+    const path = e.path.slice(dir.length);
+    if (e.type !== "file") throw new Error(`${opts.archive} holds ${path}, which is not a regular file; a source release ships regular files only`);
+    files.push({ path, executable: (e.mode & 0o111) !== 0, data: e.data });
+  }
+  return files;
+}