@intentius/chant 0.79.0 → 0.81.0

package/src/workspace/generated-files.test.ts

@@ -0,0 +1,114 @@
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterAll, describe, expect, test } from "vitest";
+import { classifyFile, declaredFilesFor, IGNORED_DIRS, IMPLICIT_GENERATED, SEED_FILES } from "./generated-files";
+import { declaredFilesAt, fileEntries, fileHash, LockError, type Lineage } from "./lineage-lock";
+import { applyUpstream } from "./lineage-update";
+
+const scratch: string[] = [];
+afterAll(() => {
+  for (const d of scratch) rmSync(d, { recursive: true, force: true });
+});
+function tree(files: Record<string, string>): string {
+  const root = realpathSync(mkdtempSync(join(tmpdir(), "chant-generated-files-")));
+  scratch.push(root);
+  mkdirSync(join(root, ".git"));
+  for (const [path, text] of Object.entries(files)) {
+    mkdirSync(dirname(join(root, path)), { recursive: true });
+    writeFileSync(join(root, path), text);
+  }
+  return root;
+}
+const bytes = (o: Record<string, string>) => new Map(Object.entries(o).map(([k, v]) => [k, Buffer.from(v)]));
+
+const DECLARATION = JSON.stringify({
+  name: "acme",
+  schema: 1,
+  members: [
+    {
+      name: "app",
+      dir: "apps/app",
+      kind: "chant",
+      generated: [
+        { path: "ci.yml", generator: "chant build ci --lexicon github -o ci.yml" },
+        { path: "skills/custom/SKILL.md", generator: "chant update", handWritten: { because: "trimmed by hand" } },
+      ],
+    },
+  ],
+});
+
+describe("the D14 file classes (#2541)", () => {
+  test("core's list: SKILL.md generated by chant update, .mcp.json a seed, .chant/types ignored", () => {
+    expect(IMPLICIT_GENERATED.map((i) => [i.glob, i.command])).toEqual([["skills/*/SKILL.md", "chant update"]]);
+    expect(SEED_FILES).toEqual([".mcp.json"]);
+    expect(IGNORED_DIRS).toEqual([".chant/types"]);
+    expect(classifyFile("skills/chant-aws/SKILL.md")).toEqual({ class: "generated", command: "chant update" });
+    expect(classifyFile("skills/a/b/SKILL.md")).toEqual({ class: "owned" });
+    expect(classifyFile(".mcp.json")).toEqual({ class: "seed" });
+    expect(classifyFile(".chant/types/core/index.d.ts")).toEqual({ class: "ignored" });
+    expect(classifyFile("src/main.ts")).toEqual({ class: "owned" });
+  });
+
+  test("a declared entry wins: generated by its generator, or owned when kept by hand", () => {
+    const declared = new Map([
+      ["ci.yml", { command: "chant build ci -o ci.yml", handWritten: false }],
+      ["skills/custom/SKILL.md", { command: "chant update", handWritten: true }],
+    ]);
+    expect(classifyFile("ci.yml", declared)).toEqual({ class: "generated", command: "chant build ci -o ci.yml" });
+    expect(classifyFile("skills/custom/SKILL.md", declared)).toEqual({ class: "owned" });
+  });
+
+  test("declaredFilesFor reads the declaration above a directory, relative to that directory", () => {
+    const root = tree({ "chant.workspace.json": DECLARATION });
+    expect([...declaredFilesFor(join(root, "apps/app"))]).toEqual([
+      ["ci.yml", { command: "chant build ci --lexicon github -o ci.yml", handWritten: false }],
+      ["skills/custom/SKILL.md", { command: "chant update", handWritten: true }],
+    ]);
+    expect([...declaredFilesFor(root)].map(([k]) => k)).toEqual(["apps/app/ci.yml", "apps/app/skills/custom/SKILL.md"]);
+    expect(declaredFilesFor(join(root, "apps/other")).size).toBe(0);
+    expect(declaredFilesFor(tree({})).size).toBe(0);
+  });
+
+  test("the lock's classes read the same list", () => {
+    const root = tree({ "chant.workspace.json": DECLARATION });
+    const files = fileEntries(
+      bytes({ "ci.yml": "on: push", "skills/custom/SKILL.md": "mine", "skills/chant-aws/SKILL.md": "s", ".mcp.json": "{}", ".chant/types/x.d.ts": "", "src/a.ts": "a" }),
+      declaredFilesAt(join(root, "apps/app")),
+    );
+    expect(files).toEqual({
+      ".mcp.json": { class: "seed", sha256: fileHash("{}") },
+      "ci.yml": { class: "generated", command: "chant build ci --lexicon github -o ci.yml", sha256: fileHash("on: push") },
+      "skills/chant-aws/SKILL.md": { class: "generated", command: "chant update", sha256: fileHash("s") },
+      "skills/custom/SKILL.md": { class: "owned", sha256: fileHash("mine") },
+      "src/a.ts": { class: "owned", sha256: fileHash("a") },
+    });
+  });
+
+  test("an unreadable declaration is a LockError, not an empty list", () => {
+    const root = tree({ "chant.workspace.json": "{ nope" });
+    expect(() => declaredFilesAt(root)).toThrow(LockError);
+    expect(() => declaredFilesAt(root)).toThrow(/declaration-unparseable/);
+  });
+
+  test("an update rebuilds a file the member lists after the lock was written, instead of merging it", () => {
+    const root = tree({ "chant.workspace.json": DECLARATION, "apps/app/ci.yml": "local", "apps/app/src/a.ts": "a" });
+    const lineage: Lineage = {
+      kind: "vendor",
+      name: "app",
+      template: "local:x",
+      source: { type: "local", path: "x" },
+      address: null,
+      parameters: {},
+      migrations: [],
+      files: { "ci.yml": { class: "owned", sha256: fileHash("base") }, "src/a.ts": { class: "owned", sha256: fileHash("a") } },
+      manualSteps: [],
+    };
+    const result = applyUpstream(join(root, "apps/app"), lineage, bytes({ "ci.yml": "upstream", "src/a.ts": "a2" }));
+    expect(result.skipped).toEqual([{ path: "ci.yml", class: "generated", command: "chant build ci --lexicon github -o ci.yml" }]);
+    expect(result.manualSteps).toEqual([]);
+    expect(lineage.files["ci.yml"]).toEqual({ class: "generated", command: "chant build ci --lexicon github -o ci.yml", sha256: fileHash("base") });
+    expect(readFileSync(join(root, "apps/app/ci.yml"), "utf-8")).toBe("local");
+    expect(readFileSync(join(root, "apps/app/src/a.ts"), "utf-8")).toBe("a2");
+  });
+});

package/src/workspace/generated-files.ts

@@ -0,0 +1,94 @@
+/**
+ * Which files are generated, and by what (#2524 D14, #2541).
+ *
+ * One list answers the question for everyone who asks it: the drift check in
+ * `checks/generated.ts` and the lineage lock's `generated` class
+ * (`lineage-lock.ts`). The list is the member's declared `generated` entries
+ * plus the files core registers itself:
+ *
+ * | File                  | Class     |
+ * |-----------------------|-----------|
+ * | `skills/*\/SKILL.md`  | generated, by `chant update` |
+ * | `.mcp.json`           | seed      |
+ * | `.chant/types/`       | ignored: gitignored, never listed |
+ *
+ * A declared entry wins over the implicit rules, so a member can list a
+ * `SKILL.md` it keeps by hand. A hand-written entry is the project's own
+ * file, so the lock classes it `owned`.
+ *
+ * Everything here loads only with the lineage and workspace commands.
+ */
+
+import { relative, resolve, sep } from "node:path";
+import { findWorkspaceRoot } from "../project-root";
+import { isInside, readDeclaration, type Declaration } from "./declaration";
+import { workingTree } from "./tree";
+
+/** What `chant update` rewrites, registered as implicit generated entries. */
+export const IMPLICIT_GENERATED = [{ glob: "skills/*/SKILL.md", pattern: /^skills\/[^/]+\/SKILL\.md$/, command: "chant update" }] as const;
+
+/** Written once at init and never again. */
+export const SEED_FILES = [".mcp.json"] as const;
+
+/** Gitignored output, out of scope for both the lock and the drift check. */
+export const IGNORED_DIRS = [".chant/types"] as const;
+
+/** A declared entry, keyed by its path relative to the directory asked about. */
+export interface DeclaredFile {
+  command: string;
+  handWritten: boolean;
+}
+export type DeclaredFiles = ReadonlyMap<string, DeclaredFile>;
+
+export type FileClassification =
+  | { class: "owned" }
+  | { class: "generated"; command: string }
+  | { class: "seed" }
+  | { class: "ignored" };
+
+/** The implicit entry `path` matches, if any. */
+export function implicitGenerated(path: string): (typeof IMPLICIT_GENERATED)[number] | undefined {
+  return IMPLICIT_GENERATED.find((i) => i.pattern.test(path));
+}
+
+/** Class a file by the declared entries first, then by the implicit rules (D14). */
+export function classifyFile(path: string, declared?: DeclaredFiles): FileClassification {
+  const entry = declared?.get(path);
+  if (entry) return entry.handWritten ? { class: "owned" } : { class: "generated", command: entry.command };
+  if (IGNORED_DIRS.some((d) => path === d || path.startsWith(`${d}/`))) return { class: "ignored" };
+  const implicit = implicitGenerated(path);
+  if (implicit) return { class: "generated", command: implicit.command };
+  if ((SEED_FILES as readonly string[]).includes(path)) return { class: "seed" };
+  return { class: "owned" };
+}
+
+/**
+ * Every declared generated file of `declaration` that sits under `dir`
+ * (relative to the workspace root, `"."` for all of it), keyed by its path
+ * relative to `dir`.
+ */
+export function declaredFilesUnder(declaration: Declaration, dir: string): Map<string, DeclaredFile> {
+  const out = new Map<string, DeclaredFile>();
+  for (const m of declaration.members) {
+    for (const g of m.generated) {
+      const full = m.dir === "." ? g.path : `${m.dir}/${g.path}`;
+      if (!isInside(full, dir)) continue;
+      out.set(dir === "." ? full : full.slice(dir.length + 1), { command: g.generator, handWritten: g.handWritten !== null });
+    }
+  }
+  return out;
+}
+
+/**
+ * The declared generated files for the absolute directory `absDir`, from the
+ * nearest `chant.workspace.json` above it. Empty when there is none, which is
+ * the plain-project case: only the implicit rules apply. A declaration that
+ * can't be read throws its `WorkspaceReadError`; it is never read as empty.
+ */
+export function declaredFilesFor(absDir: string): Map<string, DeclaredFile> {
+  const found = findWorkspaceRoot(absDir);
+  if (!found) return new Map();
+  const declaration = readDeclaration(workingTree(found.dir));
+  const rel = relative(found.dir, resolve(absDir)).split(sep).join("/");
+  return declaredFilesUnder(declaration, rel === "" ? "." : rel);
+}

package/src/workspace/glob.ts

@@ -0,0 +1,49 @@
+/**
+ * Expand an example group's globs to directories (ws-051).
+ *
+ * The walk goes one path segment at a time, so a glob such as
+ * `lexicons/*\/examples/*` reads only the directories it can match, never the
+ * whole tree. Within a segment, picomatch does the matching; `**` matches zero
+ * or more directories. `node_modules` and dot-directories are never entered
+ * by a wildcard, though a literal segment may name them.
+ */
+
+// @ts-ignore — picomatch has no types declaration
+import picomatch from "picomatch";
+import { joinPath, skippedDir, type WorkspaceTree } from "./tree";
+
+type Matcher = (name: string) => boolean;
+
+const isLiteral = (segment: string): boolean => !/[*?[\]{}()!+@]/.test(segment);
+
+/** Every directory `glob` matches in `tree`, sorted. */
+export function expandGlob(tree: WorkspaceTree, glob: string): string[] {
+  const segments = glob.split("/");
+  const matchers: (Matcher | "**" | string)[] = segments.map((s) =>
+    s === "**" ? "**" : isLiteral(s) ? s : (picomatch(s, { dot: false }) as Matcher),
+  );
+  const found = new Set<string>();
+  const subdirs = (dir: string): string[] =>
+    (tree.list(dir) ?? []).filter((e) => e.type === "dir" && !skippedDir(e.name)).map((e) => e.name);
+
+  const walk = (dir: string, index: number): void => {
+    if (index === matchers.length) {
+      if (dir !== "") found.add(dir);
+      return;
+    }
+    const m = matchers[index];
+    if (m === "**") {
+      walk(dir, index + 1);
+      for (const name of subdirs(dir)) walk(joinPath(dir, name), index);
+      return;
+    }
+    if (typeof m === "string") {
+      const next = joinPath(dir, m);
+      if (tree.stat(next) === "dir") walk(next, index + 1);
+      return;
+    }
+    for (const name of subdirs(dir)) if (m(name)) walk(joinPath(dir, name), index + 1);
+  };
+  walk("", 0);
+  return [...found].sort();
+}

package/src/workspace/graph-cli.ts

@@ -0,0 +1,220 @@
+/**
+ * `chant workspace graph [dir] [--at <rev>] [--member <name>] [-o <file>]
+ * [--env <env>] [--dry-run]` (#2537, #2536): every `chant` member's IR, read
+ * through the member's own toolchain and composed into one document
+ * (`compose-graph.ts`).
+ *
+ * The document is part of the read contract, described by `graph.schema.json`
+ * beside this file. It is printed for a failure too, with the error's reason
+ * code, so a reader always has JSON to parse.
+ *
+ * `--at <rev>` reads the declaration from git objects, as `ls --at` does. A
+ * member's IR comes from running its chant config, so for `--at` the
+ * workspace root's tree at that revision is exported to a temporary
+ * directory with `git archive`, the `node_modules` directories installed in
+ * the working tree are linked into it, and each member runs there under the
+ * toolchain it resolves, which is the one installed now. Nothing touches the
+ * checkout or the network. When no member needs running (a workspace with no
+ * `chant` members, or `--dry-run`) nothing is exported.
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { formatError } from "../cli/format";
+import type { CommandContext, ParsedArgs } from "../cli/registry";
+import { composeWorkspaceGraph, readMemberIr, type ComposeInput, type WorkspaceGraph } from "./compose-graph";
+import { readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError, type ErrorLocation, type WorkspaceErrorCode } from "./declaration";
+import { describePlan, emitDocument, executePlan, memberStatus, planJson, planMembers, type MemberPlan, type Toolchain, type UnitResult } from "./member-commands";
+import { loadKindRegistry } from "./kinds";
+import { workingTree } from "./tree";
+import { handToRootChant, locateWorkspace, type LocatedWorkspace } from "./which-chant";
+
+/** The version of the `graph` document this chant writes. */
+export const GRAPH_CONTRACT_VERSION = 1;
+
+/** `$id` of the JSON Schema for the document, shipped beside this file. */
+export const GRAPH_OUTPUT_SCHEMA_ID = "https://intentius.io/chant/schemas/workspace/graph/v1/graph.schema.json";
+
+/** Why the graph couldn't be read at all: the declaration's codes, `--at`'s included. */
+export const GRAPH_ERROR_CODES = WORKSPACE_ERROR_CODES;
+
+const USAGE = "chant workspace graph [dir] [--at <rev>] [--member <name>] [-o <file>] [--env <env>] [--dry-run]";
+
+interface Head {
+  $schema: string;
+  contract: number;
+  chant: string;
+}
+
+export type GraphDocument =
+  | (Head & { at: string | null } & WorkspaceGraph)
+  | (Head & { error: { code: WorkspaceErrorCode; message: string; location: ErrorLocation | null } });
+
+export interface GraphQuery {
+  /** Where the walk up to the declaration starts. */
+  cwd: string;
+  at?: string;
+  /** Only these members. */
+  members?: string[];
+  /** The flags each member's `chant graph` gets (`--env`). */
+  args?: Partial<ParsedArgs>;
+  /** The chant for members with none of their own; the running chant by default. */
+  reader?: Toolchain;
+  /** Called with each member's stderr, so the command can pass it on. */
+  onStderr?: (text: string) => void;
+}
+
+export interface GraphResult {
+  doc: GraphDocument;
+  /** A member failed or couldn't be read, or the declaration couldn't be read. */
+  failed: boolean;
+}
+
+/**
+ * Export the workspace root's tree at `at` into a temporary directory and link
+ * the working tree's `node_modules` into it, so members' configs resolve the
+ * packages installed now. Returns the directory; the caller removes it.
+ */
+export function exportRevision(located: LocatedWorkspace, memberDirs: string[]): string {
+  const { top, at, root, rootOnDisk } = located;
+  if (!top || !at) throw new Error("exportRevision needs a revision");
+  const scratch = mkdtempSync(join(tmpdir(), "chant-ws-at-"));
+  const out = join(scratch, "root");
+  const tar = join(scratch, "tree.tar");
+  execFileSync("git", ["archive", "--format=tar", "-o", tar, root === "." ? at : `${at}:${root}`], { cwd: top, stdio: ["ignore", "ignore", "pipe"] });
+  mkdirSync(out, { recursive: true });
+  execFileSync("tar", ["-xf", tar, "-C", out], { stdio: ["ignore", "ignore", "pipe"] });
+  rmSync(tar, { force: true });
+
+  // Every directory from the root down to each member that runs, where an
+  // install can sit: services/node_modules serves services/api and services/web.
+  const dirs = new Set<string>([""]);
+  for (const d of memberDirs) {
+    if (d === ".") continue;
+    const parts = d.split("/");
+    for (let i = 1; i <= parts.length; i++) dirs.add(parts.slice(0, i).join("/"));
+  }
+  for (const d of dirs) {
+    const src = join(rootOnDisk, ...d.split("/"), "node_modules");
+    const dst = join(out, ...d.split("/"), "node_modules");
+    if (existsSync(src) && existsSync(dirname(dst)) && !existsSync(dst)) symlinkSync(src, dst, "dir");
+  }
+  // A workspace root below the install (a monorepo package) resolves from above.
+  if (!existsSync(join(out, "node_modules"))) {
+    for (let dir = dirname(rootOnDisk); ; dir = dirname(dir)) {
+      if (existsSync(join(dir, "node_modules"))) {
+        symlinkSync(join(dir, "node_modules"), join(out, "node_modules"), "dir");
+        break;
+      }
+      if (dirname(dir) === dir) break;
+    }
+  }
+  return out;
+}
+
+function compose(plan: MemberPlan, results: UnitResult[], only: string[] | undefined, declarationMembers: { name: string; dir: string; kind: string }[]): { inputs: ComposeInput[]; failed: boolean } {
+  const byId = new Map(results.map((r) => [r.id, r]));
+  const inputs: ComposeInput[] = [];
+  let failed = plan.unreadable.length > 0;
+  for (const m of declarationMembers) {
+    if (only?.length && !only.includes(m.name)) continue;
+    const r = byId.get(m.name);
+    const skip = plan.skipped.find((s) => s.name === m.name) ?? plan.unreadable.find((s) => s.name === m.name);
+    if (!r) {
+      const status = plan.unreadable.some((s) => s.name === m.name) ? "failed" : "skipped";
+      inputs.push({ member: memberStatus(m.name, m.dir, m.kind, status, skip?.reason ?? null, null) });
+      continue;
+    }
+    if (r.exitCode !== 0) {
+      failed = true;
+      const tail = r.stderr.trim().split("\n").slice(-5).join("\n");
+      inputs.push({ member: memberStatus(m.name, m.dir, m.kind, "failed", { code: "command-failed", message: `chant graph exited ${r.exitCode}${tail ? `: ${tail}` : ""}` }, r.chant) });
+      continue;
+    }
+    const read = readMemberIr(r.stdout);
+    if ("reason" in read) {
+      failed = true;
+      inputs.push({ member: memberStatus(m.name, m.dir, m.kind, "failed", read.reason, r.chant) });
+      continue;
+    }
+    const member = memberStatus(m.name, m.dir, m.kind, "composed", null, r.chant);
+    member.irVersion = read.irVersion;
+    inputs.push({ member, ir: read.ir });
+  }
+  return { inputs, failed };
+}
+
+/** Plan a graph read without running anything, for `--dry-run`. Throws a {@link WorkspaceReadError}. */
+export function planGraph(query: GraphQuery): MemberPlan {
+  const located = locateWorkspace(query.cwd, query.at);
+  readDeclaration(located.tree, "", { rootChant: true });
+  return planMembers("graph", located.rootOnDisk, { only: query.members, reader: query.reader, tree: located.tree });
+}
+
+/** Read and compose the graph, and build the document. Never throws a {@link WorkspaceReadError}. */
+export async function workspaceGraph(query: GraphQuery): Promise<GraphResult> {
+  const head: Head = { $schema: GRAPH_OUTPUT_SCHEMA_ID, contract: GRAPH_CONTRACT_VERSION, chant: readerVersion() };
+  let exported: string | undefined;
+  try {
+    const located = locateWorkspace(query.cwd, query.at);
+    const declaration = readDeclaration(located.tree, "", { rootChant: true });
+    let plan = planMembers("graph", located.rootOnDisk, { only: query.members, reader: query.reader, tree: located.tree });
+    if (located.at !== null && plan.groups.length > 0) {
+      exported = exportRevision(located, plan.groups.flatMap((g) => g.units.map((u) => u.dir)));
+      plan = planMembers("graph", exported, { only: query.members, reader: query.reader, tree: workingTree(exported) });
+    }
+    const results = await executePlan(plan, (query.args ?? {}) as ParsedArgs);
+    for (const r of results) if (r.stderr.trim() && query.onStderr) query.onStderr(r.stderr.endsWith("\n") ? r.stderr : `${r.stderr}\n`);
+    const { inputs, failed } = compose(plan, results, query.members, declaration.members);
+    // Links (#2539) resolve against the declaration that was read, the revision's for --at, and the kinds installed now.
+    const kinds = loadKindRegistry(declaration.pins, located.rootOnDisk).registry;
+    const graph = composeWorkspaceGraph({ name: declaration.name, root: located.root }, inputs, { declaration, kinds });
+    return { doc: { ...head, at: located.at, ...graph }, failed };
+  } catch (err) {
+    if (!(err instanceof WorkspaceReadError)) throw err;
+    return { doc: { ...head, error: { code: err.code, message: err.message, location: err.location ?? null } }, failed: true };
+  } finally {
+    if (exported) rmSync(dirname(exported), { recursive: true, force: true });
+  }
+}
+
+export async function runWorkspaceGraph(ctx: CommandContext): Promise<number> {
+  const { args } = ctx;
+  const cwd = resolve(args.extraPositional ?? ".");
+  if (!existsSync(cwd)) {
+    console.error(formatError({ message: `${cwd} does not exist`, hint: USAGE }));
+    return 1;
+  }
+  // The root's chant reads the declaration (ws-021).
+  const handed = await handToRootChant(cwd, args.at);
+  if (handed !== undefined) return handed;
+
+  if (args.dryRun) {
+    let plan: MemberPlan;
+    try {
+      plan = planGraph({ cwd, at: args.at, members: args.members });
+    } catch (err) {
+      if (!(err instanceof WorkspaceReadError)) throw err;
+      console.error(formatError({ message: `${err.code}: ${err.describe()}`, hint: USAGE }));
+      return 1;
+    }
+    console.log(args.json ? JSON.stringify(planJson(plan), null, 2) : describePlan(plan));
+    return 0;
+  }
+
+  const { doc, failed } = await workspaceGraph({
+    cwd,
+    at: args.at,
+    members: args.members,
+    args,
+    onStderr: (text) => process.stderr.write(text),
+  });
+  emitDocument(doc, args.output);
+  if ("error" in doc) {
+    const l = doc.error.location;
+    console.error(formatError({ message: `${doc.error.code}: ${l ? `${l.file}:${l.line}:${l.column}: ` : ""}${doc.error.message}`, hint: USAGE }));
+  }
+  return failed ? 1 : 0;
+}