@intentius/chant 0.124.46 → 0.124.49

package/src/workspace/intent.ts

@@ -66,6 +66,7 @@ import { isGeneratedPath } from "./generated-files";
 import { entityDecisions, hasTrailer, readCommitJoins, runCommitJoins, type CommitJoins, type IntentCommit, type JoinedEntity, type PluginFinding } from "./intent-joins";
 import { loadKindRegistry } from "./kinds";
 import { resolveLinks, type LinkTableRow } from "./links";
+import { readKindLinks } from "./kind-link-readers";
 import { sourceMemberHandles } from "./member-handles";
 import { constraintCovers, isWorkspacePath, memberHolding } from "./record-assets";
 import type { DecidedIn } from "./record-decided";
@@ -78,6 +79,7 @@ import { memberDirOnDisk, memberSymbolResolvers, type SymbolPluginLoader } from
 import { resolveSymbol, symbolResolverFor, type SymbolResolver } from "./symbols";
 import { readTrailerJoins, type CommitTrailerJoins } from "./trailer-joins";
 import { buildTrace, type IntentTrace } from "./intent-trace";
+import { compareDecisionRank, type WhyRelevance } from "./decision-rank";
 import { joinPath, skippedDir, workingTree, type WorkspaceTree } from "./tree";
 import { activeAttestors, type ProvenanceLevel } from "./trust/attestor";
 import { commitProvenance, policyAtBase, resolveBase } from "./trust/provenance";
@@ -524,8 +526,7 @@ export interface WhySpan {
   via?: string[];
 }
 
-/** Why a decision is in `why`, most relevant first (#3034). */
-export type WhyRelevance = "carried" | "path" | "contract" | "issue" | "member" | "related";
+export type { WhyRelevance };
 
 export interface WhyDecision {
   decision: string;
@@ -1854,7 +1855,7 @@ async function walk(query: IntentQuery, head: Head): Promise<IntentResult> {
         atRevision: located.at !== null,
         ...(query.loadLexicon ? { load: query.loadLexicon as unknown as import("./lexicon-outputs").OutputsPluginLoader } : {}),
       });
-      rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry, lexiconOutputs)).filter((r) => r.origin === "declared");
+      rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry, lexiconOutputs), undefined, readKindLinks(declaration, located.tree, kindsRegistry)).filter((r) => r.origin === "declared");
     } catch (err) {
       if (!(err instanceof WorkspaceReadError)) throw err;
     }
@@ -2080,7 +2081,6 @@ async function walk(query: IntentQuery, head: Head): Promise<IntentResult> {
     // The decisions, most relevant first.
     const decisionNodes = [...nodes.values()].filter((n): n is DecisionNode => n.kind === "decision");
     const ownLines = (d: DecisionNode) => who.filter((w) => w.sha !== null && ownWork(joined.get(w.sha) ?? { contracts: [], authorship: [], decisions: [], carries: [] }, d)).length;
-    const ORDER: WhyRelevance[] = ["carried", "path", "contract", "issue", "member", "related"];
     const ranked = decisionNodes.map((d) => {
       const lines = ownLines(d);
       const carried = lines > 0 || (fileLines === null && touched.some((t) => ownWork(joined.get(t.sha) ?? { contracts: [], authorship: [], decisions: [], carries: [] }, d)));
@@ -2090,16 +2090,8 @@ async function walk(query: IntentQuery, head: Head): Promise<IntentResult> {
       const specificity = Math.max(0, ...d.constrains.filter((c) => c.granularity === "path").map((c) => c.entry.length));
       return { d, entry: { decision: d.id, relevance, current: d.supersededBy === null, closed: d.closed, lines } satisfies WhyDecision, specificity };
     });
-    ranked.sort(
-      (a, b) =>
-        Number(b.entry.current) - Number(a.entry.current) ||
-        ORDER.indexOf(a.entry.relevance) - ORDER.indexOf(b.entry.relevance) ||
-        b.entry.lines - a.entry.lines ||
-        Number(b.entry.closed) - Number(a.entry.closed) ||
-        b.specificity - a.specificity ||
-        (b.d.decidedIn?.date ?? "").localeCompare(a.d.decidedIn?.date ?? "") ||
-        a.d.id.localeCompare(b.d.id),
-    );
+    const key = (r: (typeof ranked)[number]) => ({ ...r.entry, id: r.d.id, specificity: r.specificity, decidedOn: r.d.decidedIn?.date ?? null });
+    ranked.sort((a, b) => compareDecisionRank(key(a), key(b)));
     const decisions = ranked.map((r) => r.entry);
 
     // The runs that wrote the current lines, most lines first; for a directory, every run in the walk.

package/src/workspace/kind-link-readers.ts

@@ -19,12 +19,24 @@
  * `resolveLinks` (`./links.ts`) resolves the links beside the declared ones,
  * with origin `kind-reader`; WSP093 and WSP094 name the reader's file and line,
  * and WSP099 reports the gaps.
+ *
+ * A migration directory's reader (#4400) lists its migrations in order, in
+ * {@link KindLinkRead.migrations}, and returns no links: what the migrations
+ * produce comes from a replay.
+ *
+ * A reader may also return outputs (#3935): names the member's files state,
+ * which other members link to. A dbt project's models are outputs, read the
+ * same way as its sources. `resolveLinks` adds them to the member's handles
+ * (`withReaderOutputs`), so a link to one resolves, and goes missing when the
+ * file that states it is gone.
  */
 
 import type { Declaration, Member } from "./declaration";
 import { resolveMemberFields, type FieldValue, type KindLinkReaderName, type KindRegistry } from "./kinds";
 import type { WorkspaceTree } from "./tree";
 import { readDbtProject } from "./dbt-project-reader";
+import { readClickHouseMigrationsDir, readGolangMigrateDir, readGooseDir } from "./migration-dirs";
+import { readFlywayDir } from "./flyway-dir";
 
 /** One link a reader found in the member's files. */
 export interface KindReadLink {
@@ -54,18 +66,67 @@ export interface KindReadGap {
   file: string;
   line: number;
   message: string;
+  /**
+   * True when the gap leaves an output out of {@link KindLinkRead.outputs}
+   * (#3935): a link to a name the member does not list is then unresolved,
+   * not missing.
+   */
+  outputs?: boolean;
+}
+
+/** One output a reader found in the member's files: a name other members may link to. */
+export interface KindReadOutput {
+  name: string;
+  /** The file that states it, relative to the workspace root. */
+  file: string;
+  /** Its line, from 1. */
+  line: number;
+}
+
+/**
+ * One migration a migration-directory reader found (#4400), in the order the
+ * tool applies it. The reader lists files and never runs the tool, so the
+ * schema the migrations produce comes from a replay, not from here.
+ */
+export interface KindReadMigration {
+  /** 1-based position in the order the tool applies the migrations. */
+  position: number;
+  /** The version as the file name writes it, such as `00002`, `20240101120000` or `1.2`. */
+  version: string;
+  name: string;
+  /** `versioned` runs once; `repeatable` runs again when its checksum changes (Flyway); `baseline` and `undo` are Flyway's B and U files. */
+  type: "versioned" | "repeatable" | "baseline" | "undo";
+  /** The file with the up statements, relative to the workspace root. */
+  file: string;
+  /** The line the up statements start on, from 1. */
+  line: number;
+  /** The file that undoes it, or true when the same file does (goose), or false when none does. */
+  down: string | boolean;
+  /** The SQL dialect the migration is written in: the member's `dialect` field, or what the directory's own configuration names. */
+  dialect: string;
+  /** False when the file asks to run outside a transaction (`-- +goose NO TRANSACTION`). */
+  transaction?: boolean;
 }
 
 export interface KindLinkRead {
-  /** The reader that ran. */
-  reader: KindLinkReaderName;
+  /** The reader that ran: a kind's reader, or `lexicon <name>` for links a member's lexicon read from its declarations (#4397). */
+  reader: KindLinkReaderName | `lexicon${string}`;
   links: KindReadLink[];
   gaps: KindReadGap[];
+  /**
+   * The outputs the member exposes that its files state (#3935), beside the
+   * ones its kind lists. `resolveLinks` adds them to the member's handles, so
+   * another member can link to them. A gap flagged `outputs` makes the list
+   * incomplete.
+   */
+  outputs?: KindReadOutput[];
   /**
    * The member field that names the producer, for messages that ask the
    * author to set it when the files don't say (`sqlMember` for dbt).
    */
   producerField?: string;
+  /** The migrations of a migration directory (#4400), in the order they apply; unset for other readers. */
+  migrations?: KindReadMigration[];
 }
 
 /** What a reader gets: the member, its directory and resolved fields, and the tree read. */
@@ -83,6 +144,10 @@ export type KindLinkReader = (input: KindReaderInput) => KindLinkRead;
 /** Every reader chant ships, by the name a kinds file uses. */
 export const LINK_READERS: Record<KindLinkReaderName, KindLinkReader> = {
   "dbt-project": readDbtProject,
+  "goose-dir": readGooseDir,
+  "golang-migrate-dir": readGolangMigrateDir,
+  "clickhouse-migrations-dir": readClickHouseMigrationsDir,
+  "flyway-dir": readFlywayDir,
 };
 
 /**
@@ -102,7 +167,7 @@ export function readKindLinks(declaration: Declaration, tree: WorkspaceTree, kin
     try {
       out.set(m.name, LINK_READERS[kind.reader]({ member: m, dir, fields, tree }));
     } catch (err) {
-      out.set(m.name, { reader: kind.reader, links: [], gaps: [{ file: dir || ".", line: 1, message: `the ${kind.reader} reader failed: ${(err as Error).message}` }] });
+      out.set(m.name, { reader: kind.reader, links: [], gaps: [{ file: dir || ".", line: 1, message: `the ${kind.reader} reader failed: ${(err as Error).message}`, outputs: true }] });
     }
   }
   return out;

package/src/workspace/kind-readers.test.ts

@@ -132,3 +132,39 @@ describe("workspace graph reads a package kind with a graph block (#2874)", () =
     expect(planMembers("graph", root, { kinds: registry }).groups[0].units[0].reader).toMatchObject({ lexicon: "fake" });
   });
 });
+
+describe("a graph reader module from a package that is not a lexicon (#4413)", () => {
+  /** A fake chant that prints the reader project's config, so the test sees what the member's chant is handed. */
+  const MODULE_CHANT = `#!/bin/sh
+[ "$1" = graph ] || { echo "Error: Unknown command: $1" >&2; exit 1; }
+printf '{"version":1,"nodes":[{"id":"org","kind":"Org","lexicon":"gov","attrs":{"config":'
+cat chant.config.json
+printf ',"linked":%s}}],"edges":[],"groups":{}}\\n' "$([ -e node_modules ] && echo true || echo false)"
+`;
+
+  test("the reader project names the module and links no lexicon", async () => {
+    const root = repo({
+      "chant.workspace.json": declaration([{ name: "forge", dir: "gov", kind: "gov" }], { pins: [{ package: "@acme/warden", version: "1.0.0" }] }),
+      "gov/governance.yml": "org: acme\n",
+      ".gitignore": "node_modules\n",
+      "node_modules/.bin/chant": { text: MODULE_CHANT, mode: 0o755 },
+      "node_modules/@acme/warden/package.json": JSON.stringify({ name: "@acme/warden", version: "1.0.0", exports: { "./workspace-kinds": "./workspace-kinds.json" } }),
+      "node_modules/@acme/warden/graph.mjs": "export default () => ({ nodes: [], edges: [] });\n",
+      "node_modules/@acme/warden/workspace-kinds.json": JSON.stringify({
+        schema: 1,
+        kinds: [{ name: "gov", description: "governance", precedence: 500, probe: { anyFile: ["governance.yml"] }, graph: { reader: "graph.mjs", config: { dir: "{dir}" } } }],
+      }),
+    });
+    const { doc, failed } = await workspaceGraph({ cwd: root });
+    const g = result(doc);
+    expectValid(g);
+    expect(failed).toBe(false);
+    expect(g.members.map((m) => [m.name, m.status])).toEqual([["forge", "composed"]]);
+    const attrs = g.nodes[0].attrs as { config: Record<string, unknown>; linked: boolean };
+    expect(attrs.config).toEqual({
+      lexicons: [],
+      graph: { lint: "report", reader: { module: join(root, "node_modules/@acme/warden/graph.mjs"), config: { dir: `${root}/gov` } } },
+    });
+    expect(attrs.linked).toBe(false);
+  });
+});

package/src/workspace/kind-readers.ts

@@ -20,6 +20,12 @@
  *   package, found from the workspace root as the kind's pin was. The lexicon
  *   chant loads is then the one the declaration pins.
  *
+ * A kind whose `graph` names a `reader` module instead (#4413) gets a
+ * project with `lexicons: []` and `graph.reader` set to the module, which the
+ * supplying package ships, so any package can supply one. The member's own
+ * `chant graph` loads it there, under the member's toolchain, and prints the
+ * IR it returns. There is no lexicon to link.
+ *
  * Nothing is written under the workspace, and the projects are removed when
  * the read ends. A lexicon that can't be found where its pin says fails the
  * member with `command-failed` and says what to install; the other members
@@ -57,6 +63,7 @@ function real(path: string): string {
 /** Why a unit's lexicon can't be loaded from `root`, or the `node_modules` to link. */
 function lexiconModules(unit: RunUnit, root: string): { modules: string } | { message: string } {
   const reader = unit.reader!;
+  if (!reader.lexicon) return { message: `kind ${unit.kind} names no lexicon` };
   const name = lexiconPackageName(reader.lexicon);
   const installed = findInstalledPackage(name, root);
   if (!installed) {
@@ -89,6 +96,16 @@ export function prepareKindReaders(plan: MemberPlan, root: string): PreparedRead
     for (const group of plan.groups) {
       group.units = group.units.filter((unit) => {
         if (!unit.reader) return true;
+        if (unit.reader.readerFile) {
+          // A graph reader module (#4413): no lexicon and no node_modules link. The module sits in the pinned package and resolves its own imports from there.
+          scratch ??= mkdtempSync(join(tmpdir(), "chant-ws-reader-"));
+          const project = join(scratch, unit.id);
+          mkdirSync(project, { recursive: true });
+          const config = substituteGraphConfig(unit.reader.config, { member: unit.member, dir: unit.abs, workspace: root });
+          writeFileSync(join(project, "chant.config.json"), `${JSON.stringify({ lexicons: [], graph: { lint: "report", reader: { module: unit.reader.readerFile, config } } }, null, 2)}\n`);
+          unit.abs = project;
+          return true;
+        }
         const found = lexiconModules(unit, root);
         if ("message" in found) {
           failed.push({ name: unit.member, dir: unit.dir, kind: unit.kind, reason: { code: "command-failed", message: found.message } });
@@ -101,7 +118,7 @@ export function prepareKindReaders(plan: MemberPlan, root: string): PreparedRead
         // `graph.lint: "report"` (behold#919): chant did not author a reader
         // member's source, so a lint error there is a finding about it, carried
         // on the member's `meta.lint`, not a reason to leave it out of the graph.
-        writeFileSync(join(project, "chant.config.json"), `${JSON.stringify({ lexicons: [unit.reader.lexicon], [unit.reader.lexicon]: config, graph: { lint: "report" } }, null, 2)}\n`);
+        writeFileSync(join(project, "chant.config.json"), `${JSON.stringify({ lexicons: [unit.reader.lexicon!], [unit.reader.lexicon!]: config, graph: { lint: "report" } }, null, 2)}\n`);
         symlinkSync(found.modules, join(project, "node_modules"), "dir");
         unit.abs = project;
         return true;

package/src/workspace/kinds.test.ts

@@ -61,10 +61,10 @@ function pkg(name: string, version: string, kinds: unknown[] | null, extra: Reco
 }
 
 describe("the built-in kinds (#2535)", () => {
-  test("are chant, workspace, design and other, and the group kind examples", () => {
-    expect(BUILTIN_KINDS.map((k) => `${k.name}:${k.shape}`)).toEqual(["chant:member", "workspace:member", "other:member", "design:member", "examples:group"]);
+  test("are chant, workspace, repo, design and other, and the group kind examples", () => {
+    expect(BUILTIN_KINDS.map((k) => `${k.name}:${k.shape}`)).toEqual(["chant:member", "workspace:member", "repo:member", "other:member", "design:member", "examples:group"]);
     const registry = builtinKindRegistry();
-    expect(registry.names()).toEqual(["chant", "design", "other", "workspace"]);
+    expect(registry.names()).toEqual(["chant", "design", "other", "repo", "workspace"]);
     expect(registry.get(EXAMPLES_KIND)?.shape).toBe("group");
   });
 
@@ -72,8 +72,13 @@ describe("the built-in kinds (#2535)", () => {
     const entry = declarationSchema.$defs.entry;
     expect(entry.if.properties.kind.const).toBe(EXAMPLES_KIND);
     expect(declarationSchema.$defs.group.properties.kind.const).toBe(EXAMPLES_KIND);
-    expect(declarationSchema.$defs.member.if.properties.kind.const).toBe("other");
-    expect(declarationSchema.$defs.member.then.required).toEqual(["because"]);
+    const [other, repo] = declarationSchema.$defs.member.allOf;
+    expect(other.if.properties.kind.const).toBe("other");
+    expect(other.then.required).toEqual(["because"]);
+    // A repo member (#3168) has repo and no dir; every other member has dir and no repo.
+    expect(repo.if.properties.kind.const).toBe("repo");
+    expect(repo.then.required).toEqual(["repo"]);
+    expect(repo.else?.required).toEqual(["dir"]);
   });
 
   test("a registry refuses a name twice", () => {
@@ -150,6 +155,29 @@ describe("a kind's graph block (#2874)", () => {
     expect(bad.registry.get("other-tf")).toBeDefined();
   });
 
+  test("a graph reader module is accepted from a package that is not a lexicon, and only inside the package (#4413)", () => {
+    const reader = { reader: "dist/graph.mjs", config: { dir: "{dir}" } };
+    const root = dir({
+      ...pkg("@acme/warden", "1.0.0", [{ ...tf, name: "gov", graph: reader }]),
+      "node_modules/@acme/warden/dist/graph.mjs": "export default () => ({ nodes: [], edges: [] });\n",
+      ...pkg("@acme/gone", "1.0.0", [{ ...tf, name: "gone", graph: reader }]),
+      ...pkg("@acme/up", "1.0.0", [{ ...tf, name: "up", graph: { reader: "../x.mjs" } }]),
+    });
+    const good = loadKindRegistry([{ package: "@acme/warden", version: "1.0.0", path: null }], root);
+    expect(good.problems).toEqual([]);
+    expect(good.registry.get("gov")?.graph).toEqual({ ...reader, readerFile: join(root, "node_modules/@acme/warden/dist/graph.mjs") });
+    const gone = loadKindRegistry([{ package: "@acme/gone", version: "1.0.0", path: null }], root);
+    expect(gone.problems.map((p) => p.message)).toEqual([expect.stringMatching(/graph reader dist\/graph.mjs, which does not exist/)]);
+    expect(gone.registry.get("gone")?.graph).toBeUndefined();
+    const up = loadKindRegistry([{ package: "@acme/up", version: "1.0.0", path: null }], root);
+    expect(up.problems.length).toBe(1);
+    expect(up.registry.get("up")?.graph).toBeUndefined();
+    // Not both, and an absolute path is refused.
+    expect(parseKindData(kindsFile([{ ...tf, graph: { reader: "a.mjs", lexicon: "tf", config: {} } }]), "p").problems.length).toBeGreaterThan(0);
+    expect(parseKindData(kindsFile([{ ...tf, graph: { reader: "/etc/x.mjs" } }]), "p").problems.length).toBeGreaterThan(0);
+    expect(parseKindData(kindsFile([{ ...tf, graph: { reader: "a.mjs" } }]), "p").problems).toEqual([]);
+  });
+
   test("the terraform lexicon's kinds read their members with it, named for the member at the workspace root", () => {
     const lexicon = join(import.meta.dirname, "..", "..", "..", "..", "lexicons", "terraform");
     const read = readPackageKinds(lexicon);
@@ -203,7 +231,7 @@ describe("loadKindRegistry: kinds from the pins", () => {
     const { registry, problems } = loadKindRegistry([{ package: "@acme/chant-lexicon-tf", version: "1.2.3", path: null }], root);
     expect(problems).toEqual([]);
     expect(registry.get("terraform")).toMatchObject({ source: "@acme/chant-lexicon-tf", precedence: 400 });
-    expect(registry.names()).toEqual(["chant", "design", "other", "terraform", "workspace"]);
+    expect(registry.names()).toEqual(["chant", "design", "other", "repo", "terraform", "workspace"]);
     expect(existsSync(join(root, "node_modules/@acme/chant-lexicon-tf/IMPORTED"))).toBe(false);
     expect((globalThis as { __chantKindsImported?: boolean }).__chantKindsImported).toBeUndefined();
   });
@@ -223,7 +251,7 @@ describe("loadKindRegistry: kinds from the pins", () => {
       ],
       root,
     );
-    expect(registry.names()).toEqual(["chant", "design", "other", "workspace"]);
+    expect(registry.names()).toEqual(["chant", "design", "other", "repo", "workspace"]);
     expect(problems).toEqual([
       { pin: 1, message: "pinned package absent is not installed; install it to read the kinds it supplies" },
       { pin: 2, message: "tf-kinds is pinned at 1.0.0, and 2.0.0 is installed" },