@intentius/chant 0.124.45 → 0.124.48

package/src/workspace/workspace-gates.ts

@@ -0,0 +1,200 @@
+/**
+ * Workspace gates (#2554, #2524 D17, ws-039).
+ *
+ * A workspace gate is one approval for a change across members. A run of
+ * `chant workspace run --gate <name>` whose member runs stop at their own
+ * gates records a pending fact under `_members/_workspace/_gates/<name>.jsonl`
+ * that lists each stopped member gate and the plan it waits on (a cover),
+ * with the digest of that list. `chant workspace run approve <name>` records
+ * an approval of one such digest.
+ *
+ * The rule (ws-039): a workspace gate never satisfies a member gate, unless
+ * both hold:
+ *
+ * 1. its approval's digest covers the member's plan: the approval lists the
+ *    member, the Op, the gate and the plan digest the member run reached it
+ *    with, and the list still hashes to the approval's digest;
+ * 2. the member opts in at base: its entry in `chant.workspace.json`, read
+ *    from the base revision as other policy is, sets
+ *    `acceptWorkspaceGates: true`. A declaration read only from the working
+ *    tree never opts a member in.
+ *
+ * A member run started under the gate (`CHANT_WORKSPACE_GATE`) asks this
+ * module through `GateCheckInput.workspace` after its own approvals have
+ * not passed the gate. Nothing is written to the member's gate ledger: the
+ * run's step record says `via: "workspace"` and names the workspace gate.
+ */
+
+import { samePlanDigest } from "../lifecycle/plan-digest";
+import { resolveMemberLedger } from "../lifecycle/member-ledger";
+import type { WorkspaceGateStood } from "../op/gate";
+import { appendWorkspaceLine, canonicalDigest, readWorkspaceLines, WORKSPACE_GATES_DIR } from "./workspace-ledger";
+
+/** One member gate a workspace gate covers. */
+export interface WorkspaceGateCover {
+  member: string;
+  op: string;
+  gate: string;
+  planDigest: string;
+  /** The member gate's environment, when it is bound to one. */
+  environment?: string;
+}
+
+export interface WorkspaceGatePending {
+  version: 1;
+  kind: "pending";
+  gate: string;
+  /** The workspace run that recorded it. */
+  run: string;
+  timestamp: string;
+  digest: string;
+  covers: WorkspaceGateCover[];
+}
+
+export interface WorkspaceGateApproval {
+  version: 1;
+  kind: "approval";
+  gate: string;
+  digest: string;
+  covers: WorkspaceGateCover[];
+  approvedBy: string;
+  timestamp: string;
+  note?: string;
+}
+
+export type WorkspaceGateLine = WorkspaceGatePending | WorkspaceGateApproval;
+
+/** Covers in one order, so the same set always has the same digest. */
+export function sortCovers(covers: readonly WorkspaceGateCover[]): WorkspaceGateCover[] {
+  const key = (c: WorkspaceGateCover) => `${c.member}\u0000${c.op}\u0000${c.gate}\u0000${c.environment ?? ""}\u0000${c.planDigest}`;
+  return [...covers]
+    .map((c) => ({ member: c.member, op: c.op, gate: c.gate, planDigest: c.planDigest, ...(c.environment !== undefined ? { environment: c.environment } : {}) }))
+    .sort((a, b) => (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
+}
+
+/** The digest a workspace gate's pending fact and approval carry: the member plans it covers. */
+export function workspaceGateDigest(covers: readonly WorkspaceGateCover[]): string {
+  return canonicalDigest(sortCovers(covers));
+}
+
+/** What a member gate asks about. */
+export interface MemberGateQuery {
+  op: string;
+  gate: string;
+  planDigest: string;
+  environment?: string;
+}
+
+/**
+ * The rule of ws-039, pure: the newest approval that stands in for the
+ * member gate `q`, or null. Null whenever the member has not opted in.
+ */
+export function workspaceGateStandsIn(
+  q: MemberGateQuery,
+  ctx: { member: string; optedIn: boolean; approvals: readonly WorkspaceGateApproval[] },
+): WorkspaceGateApproval | null {
+  if (!ctx.optedIn) return null;
+  const covering = ctx.approvals.filter(
+    (a) =>
+      // The covers must still hash to the digest that was approved.
+      workspaceGateDigest(a.covers) === a.digest &&
+      a.covers.some(
+        (c) =>
+          c.member === ctx.member && c.op === q.op && c.gate === q.gate &&
+          samePlanDigest(c.planDigest, q.planDigest) &&
+          (c.environment ?? null) === (q.environment ?? null),
+      ),
+  );
+  return covering.sort((a, b) => (a.timestamp < b.timestamp ? -1 : a.timestamp > b.timestamp ? 1 : 0)).at(-1) ?? null;
+}
+
+const fileFor = (gate: string) => `${gate}.jsonl`;
+
+function isCover(v: unknown): v is WorkspaceGateCover {
+  const c = v as WorkspaceGateCover;
+  return typeof c === "object" && c !== null && typeof c.member === "string" && typeof c.op === "string" && typeof c.gate === "string" && typeof c.planDigest === "string";
+}
+
+function isGateLine(v: unknown): v is WorkspaceGateLine {
+  const l = v as WorkspaceGateLine;
+  if (typeof l !== "object" || l === null || l.version !== 1 || typeof l.gate !== "string" || typeof l.digest !== "string" || typeof l.timestamp !== "string") return false;
+  if (!Array.isArray(l.covers) || !l.covers.every(isCover)) return false;
+  if (l.kind === "pending") return typeof l.run === "string";
+  if (l.kind === "approval") return typeof l.approvedBy === "string";
+  return false;
+}
+
+/** A workspace gate's ledger, oldest first. */
+export async function readWorkspaceGate(cwd: string, gate: string): Promise<{ pending: WorkspaceGatePending[]; approvals: WorkspaceGateApproval[]; malformed: number }> {
+  const { records, malformed } = await readWorkspaceLines(cwd, WORKSPACE_GATES_DIR, fileFor(gate), isGateLine);
+  return {
+    pending: records.filter((r): r is WorkspaceGatePending => r.kind === "pending"),
+    approvals: records.filter((r): r is WorkspaceGateApproval => r.kind === "approval"),
+    malformed,
+  };
+}
+
+export async function appendWorkspaceGatePending(cwd: string, input: Omit<WorkspaceGatePending, "version" | "kind" | "digest" | "covers"> & { covers: readonly WorkspaceGateCover[] }): Promise<WorkspaceGatePending> {
+  const covers = sortCovers(input.covers);
+  const record: WorkspaceGatePending = { version: 1, kind: "pending", gate: input.gate, run: input.run, timestamp: input.timestamp, digest: workspaceGateDigest(covers), covers };
+  await appendWorkspaceLine(cwd, WORKSPACE_GATES_DIR, fileFor(input.gate), record, `Workspace gate pending: ${input.gate}`);
+  return record;
+}
+
+/**
+ * Approve a workspace gate's pending fact: the newest one, or the one whose
+ * digest is `digest`. The approval copies its covers. Throws when there is no
+ * such fact.
+ */
+export async function approveWorkspaceGate(
+  cwd: string,
+  input: { gate: string; digest?: string; approvedBy: string; timestamp: string; note?: string },
+): Promise<WorkspaceGateApproval> {
+  const { pending } = await readWorkspaceGate(cwd, input.gate);
+  const candidates = input.digest === undefined ? pending : pending.filter((p) => p.digest === input.digest);
+  const fact = candidates.at(-1);
+  if (!fact) {
+    throw new Error(
+      input.digest === undefined
+        ? `workspace gate "${input.gate}" has no pending fact to approve; a chant workspace run --gate ${input.gate} whose members stop at their gates records one`
+        : `workspace gate "${input.gate}" has no pending fact with digest ${input.digest}`,
+    );
+  }
+  const record: WorkspaceGateApproval = {
+    version: 1,
+    kind: "approval",
+    gate: input.gate,
+    digest: fact.digest,
+    covers: fact.covers,
+    approvedBy: input.approvedBy,
+    timestamp: input.timestamp,
+    ...(input.note ? { note: input.note } : {}),
+  };
+  await appendWorkspaceLine(cwd, WORKSPACE_GATES_DIR, fileFor(input.gate), record, `Workspace gate approval: ${input.gate}`);
+  return record;
+}
+
+/**
+ * Whether the member holding `cwd` opts in to workspace gates at base, and
+ * its name. A member of a nested workspace, the root member and a directory
+ * no member owns never opt in.
+ */
+export async function memberAcceptsWorkspaceGates(cwd: string): Promise<{ member: string | null; accepts: boolean }> {
+  const ledger = await resolveMemberLedger(cwd);
+  if (ledger.members.length !== 1) return { member: ledger.members.join("/") || null, accepts: false };
+  const member = ledger.members[0]!;
+  const { scopeSource } = await import("./write-scope");
+  const source = scopeSource(cwd);
+  if (source.from !== "base" || !source.declaration) return { member, accepts: false };
+  const entry = source.declaration.members.find((m) => m.name === member);
+  return { member, accepts: entry?.acceptWorkspaceGates === true };
+}
+
+/** The stand-in a member run started under workspace gate `gate` asks (`GateCheckInput.workspace`). */
+export async function memberGateStandIn(cwd: string, gate: string, q: MemberGateQuery): Promise<WorkspaceGateStood | null> {
+  const { member, accepts } = await memberAcceptsWorkspaceGates(cwd);
+  if (!member || !accepts) return null;
+  const { approvals } = await readWorkspaceGate(cwd, gate);
+  const found = workspaceGateStandsIn(q, { member, optedIn: accepts, approvals });
+  return found ? { gate, digest: found.digest, approvedBy: found.approvedBy, timestamp: found.timestamp } : null;
+}

package/src/workspace/workspace-kinds.schema.json

@@ -46,7 +46,7 @@
         "name": {
           "type": "string",
           "pattern": "^[a-z][a-z0-9-]{0,39}$",
-          "description": "The kind's name, as a member's kind field names it. chant, workspace, other and examples are built in and can't be supplied."
+          "description": "The kind's name, as a member's kind field names it. chant, workspace, repo, design, other and examples are built in and can't be supplied."
         },
         "description": {
           "type": "string",
@@ -82,7 +82,10 @@
         "reader": {
           "type": "string",
           "pattern": "^[a-z][a-z0-9-]{0,39}$",
-          "description": "A link reader chant ships, by name, that derives a member's links from the member's own files (#3932): dbt-project reads a dbt project's sources. The reader is chant's code and runs offline, parsing files as data; the package supplies only the name. A name this chant doesn't have keeps the kind without a reader, and the load reports it. A chant older than this field fails a kinds file that uses it."
+          "description": "A link reader chant ships, by name, that derives a member's links from the member's own files (#3932): dbt-project reads a dbt project's sources; goose-dir, golang-migrate-dir, clickhouse-migrations-dir and flyway-dir list a migrations directory in order (#4400). The reader is chant's code and runs offline, parsing files as data; the package supplies only the name. A name this chant doesn't have keeps the kind without a reader, and the load reports it. A chant older than this field fails a kinds file that uses it."
+        },
+        "memberFile": {
+          "$ref": "#/$defs/memberFile"
         }
       },
       "patternProperties": {
@@ -159,13 +162,41 @@
             }
           },
           "additionalProperties": false
+        },
+        "anyLine": {
+          "type": "object",
+          "description": "The directory passes when a file directly in it, whose name matches one of in, has a line that starts with one of starts, after leading space and ignoring case (#4400). A goose directory is found by -- +goose Up. Added by the chant that adds the goose-dir reader.",
+          "required": [
+            "in",
+            "starts"
+          ],
+          "properties": {
+            "in": {
+              "type": "array",
+              "minItems": 1,
+              "uniqueItems": true,
+              "items": {
+                "$ref": "#/$defs/fileName"
+              }
+            },
+            "starts": {
+              "type": "array",
+              "minItems": 1,
+              "uniqueItems": true,
+              "items": {
+                "type": "string",
+                "minLength": 1
+              }
+            }
+          },
+          "additionalProperties": false
         }
       },
       "additionalProperties": false
     },
     "fileName": {
       "type": "string",
-      "description": "A file name, with no directory part. A * matches any run of characters.",
+      "description": "A file name, with no directory part. A * matches any run of characters, and [0-9] one digit.",
       "pattern": "^(?!\\.\\.?$)[^/\\\\]+$"
     },
     "graph": {
@@ -281,6 +312,30 @@
         }
       },
       "additionalProperties": false
+    },
+    "memberFile": {
+      "type": "object",
+      "description": "A file every member of this kind has, and the JSON Schema it must pass (#3672). chant workspace check reads the file from the member's directory, parses it as YAML when its name ends in .yml or .yaml and as JSON otherwise, and validates it against the schema (WSP012). The schema is a JSON file inside the supplying package, read as data with draft 2020-12. A chant older than this field fails a kinds file that uses it.",
+      "required": [
+        "path",
+        "schema"
+      ],
+      "properties": {
+        "path": {
+          "type": "string",
+          "description": "The file, relative to the member's directory, such as terragucci.yml. Forward slashes; no .. segment and no leading /.",
+          "pattern": "^(?!/)(?!.*(^|/)\\.\\.?(/|$))[^\\\\]+$"
+        },
+        "schema": {
+          "type": "string",
+          "description": "The JSON Schema the file must pass, relative to the supplying package's directory, such as schemas/terragucci.schema.json. It must be a .json file inside the package.",
+          "pattern": "^(?!/)(?!.*(^|/)\\.\\.?(/|$))[^\\\\]+\\.json$"
+        }
+      },
+      "patternProperties": {
+        "^x-": true
+      },
+      "additionalProperties": false
     }
   }
 }

package/src/workspace/workspace-ledger.ts

@@ -0,0 +1,130 @@
+/**
+ * The workspace's own ledger (#2554, #2524 D17, ws-030): `_members/_workspace/`
+ * on the `chant/lifecycle` branch, beside every member's `_members/<member>/`.
+ * `_workspace` is a reserved member name, so no member writes here.
+ *
+ * Three stores, each a JSON Lines file appended with the read-modify-write
+ * and CAS retry every ledger here uses:
+ *
+ * - `<env>/runs.jsonl`: one record per `chant workspace run`, naming every
+ *   `<member>/<op>` step, how it ended, and the member run it started. Each
+ *   member run cites the workspace run back (`OpRunRecord.workspaceRun`).
+ * - `_gates/<gate>.jsonl`: a workspace gate's pending facts and approvals.
+ *   Each names the member plans it covers and their digest
+ *   (`./workspace-gates.ts`).
+ * - `<env>/trains.jsonl`: release trains, each pinning one release of every
+ *   `chant` member (`./release-train.ts`).
+ *
+ * A malformed line is skipped and counted, never thrown on.
+ */
+
+import { createHash } from "node:crypto";
+import { readBlobBySha, readBlobFromPath, readPathSha, RefCASConflictError, writeBlobToPath, type LedgerLocation } from "../lifecycle/git";
+import { MEMBERS_DIR } from "../lifecycle/member-ledger";
+import { sortedJsonReplacer } from "../utils";
+
+/** The reserved member name the workspace's own ledger sits under. */
+export const WORKSPACE_LEDGER_NAME = "_workspace";
+/** The branch prefix of the workspace's ledger. */
+export const WORKSPACE_LEDGER_PREFIX = `${MEMBERS_DIR}/${WORKSPACE_LEDGER_NAME}/`;
+/** The environment directory a run with no `--env` records under, as a member Op run does. */
+export const DEFAULT_WORKSPACE_ENV = "local";
+export const WORKSPACE_GATES_DIR = "_gates";
+
+const APPEND_RETRY_ATTEMPTS = 5;
+
+function at(cwd: string): LedgerLocation {
+  return { cwd, ledgerPrefix: WORKSPACE_LEDGER_PREFIX };
+}
+
+/** Append one line to `<dir>/<file>` in the workspace ledger. Returns the ledger commit. */
+export async function appendWorkspaceLine(cwd: string, dir: string, file: string, value: unknown, message: string): Promise<string> {
+  const json = JSON.stringify(value, sortedJsonReplacer);
+  let lastErr: unknown;
+  for (let attempt = 1; attempt <= APPEND_RETRY_ATTEMPTS; attempt++) {
+    try {
+      const priorSha = await readPathSha(dir, file, at(cwd));
+      const existing = priorSha ? await readBlobBySha(priorSha, { cwd }) : null;
+      const content = `${existing ? `${existing.replace(/\n$/, "")}\n` : ""}${json}\n`;
+      return await writeBlobToPath(dir, file, content, message, { ...at(cwd), expectPriorPathSha: priorSha });
+    } catch (err) {
+      if (!(err instanceof RefCASConflictError)) throw err;
+      lastErr = err;
+    }
+  }
+  throw lastErr;
+}
+
+/** Read every line of `<dir>/<file>` that `accept` takes, oldest first. */
+export async function readWorkspaceLines<T>(cwd: string, dir: string, file: string, accept: (v: unknown) => v is T): Promise<{ records: T[]; malformed: number }> {
+  const content = await readBlobFromPath(dir, file, at(cwd));
+  if (!content) return { records: [], malformed: 0 };
+  const records: T[] = [];
+  let malformed = 0;
+  for (const line of content.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      const v = JSON.parse(line) as unknown;
+      if (accept(v)) records.push(v);
+      else malformed++;
+    } catch {
+      malformed++;
+    }
+  }
+  return { records, malformed };
+}
+
+/** `sha256:<hex>` over a value's canonical JSON (keys sorted). */
+export function canonicalDigest(value: unknown): string {
+  return `sha256:${createHash("sha256").update(JSON.stringify(value, sortedJsonReplacer)).digest("hex")}`;
+}
+
+// ── Runs ─────────────────────────────────────────────────────────────────────
+
+/** How one `<member>/<op>` step of a workspace run ended. */
+export type WorkspaceStepStatus = "ok" | "fail" | "gated" | "not-run";
+
+export interface WorkspaceRunStep {
+  /** `<member>/<op>`. */
+  target: string;
+  member: string;
+  op: string;
+  status: WorkspaceStepStatus;
+  /** The member's `chant run` exit code; null for a step that did not run. */
+  exitCode: number | null;
+  /** The member run's id, read back from the member's run ledger, where it cites this run. Null when none was found. */
+  memberRun: string | null;
+  /** For a gated step: the member gate it stopped at and the plan it waits on. */
+  gate?: { op: string; name: string; planDigest: string | null; environment: string | null };
+}
+
+export interface WorkspaceRunRecord {
+  version: 1;
+  id: string;
+  started: string;
+  ended: string;
+  /** `gated` when a step stopped at a gate; `fail` when a step failed. A run stops at the first step that does not pass. */
+  status: "ok" | "fail" | "gated";
+  /** `--env`, or null. */
+  env: string | null;
+  /** The workspace gate the run was started under (`--gate`), or null. */
+  gate: string | null;
+  /** `--dependents-of`, the member whose dependents the run ran, or null. */
+  dependentsOf: string | null;
+  steps: WorkspaceRunStep[];
+}
+
+const RUNS_FILE = "runs.jsonl";
+
+function isRunRecord(v: unknown): v is WorkspaceRunRecord {
+  const r = v as WorkspaceRunRecord;
+  return typeof r === "object" && r !== null && r.version === 1 && typeof r.id === "string" && Array.isArray(r.steps);
+}
+
+export async function appendWorkspaceRun(cwd: string, record: WorkspaceRunRecord): Promise<string> {
+  return appendWorkspaceLine(cwd, record.env ?? DEFAULT_WORKSPACE_ENV, RUNS_FILE, record, `Workspace run record: ${record.id}`);
+}
+
+export async function readWorkspaceRuns(cwd: string, env: string = DEFAULT_WORKSPACE_ENV): Promise<{ records: WorkspaceRunRecord[]; malformed: number }> {
+  return readWorkspaceLines(cwd, env, RUNS_FILE, isRunRecord);
+}