@intentius/chant 0.124.22 → 0.124.23

package/src/workspace/intent.ts

@@ -77,7 +77,7 @@ import { followSquashes, squashLineOrigins, type Forge, type SquashFollow } from
 import { memberDirOnDisk, memberSymbolResolvers, type SymbolPluginLoader } from "./lexicon-symbols";
 import { resolveSymbol, symbolResolverFor, type SymbolResolver } from "./symbols";
 import { readTrailerJoins, type CommitTrailerJoins } from "./trailer-joins";
-import { joinPath, skippedDir, type WorkspaceTree } from "./tree";
+import { joinPath, skippedDir, workingTree, type WorkspaceTree } from "./tree";
 import { activeAttestors, type ProvenanceLevel } from "./trust/attestor";
 import { commitProvenance, policyAtBase, resolveBase } from "./trust/provenance";
 import { locateWorkspace, type LocatedWorkspace } from "./which-chant";
@@ -1722,7 +1722,12 @@ async function walk(query: IntentQuery, head: Head): Promise<IntentResult> {
     try {
       const groups = resolveGroups(declaration, located.tree);
       const kindsRegistry = loadKindRegistry(declaration.pins, located.rootOnDisk).registry;
-      rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry)).filter((r) => r.origin === "declared");
+      const { gatherLexiconOutputs } = await import("./lexicon-outputs");
+      const lexiconOutputs = await gatherLexiconOutputs(located.rootOnDisk, declaration, located.at ? workingTree(located.rootOnDisk) : located.tree, groups, kindsRegistry, {
+        atRevision: located.at !== null,
+        ...(query.loadLexicon ? { load: query.loadLexicon as unknown as import("./lexicon-outputs").OutputsPluginLoader } : {}),
+      });
+      rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry, lexiconOutputs)).filter((r) => r.origin === "declared");
     } catch (err) {
       if (!(err instanceof WorkspaceReadError)) throw err;
     }

package/src/workspace/kind-link-readers.ts

@@ -0,0 +1,109 @@
+/**
+ * Links a kind reads from its member's own files (#3932).
+ *
+ * A member's links usually come from its entry in `chant.workspace.json`, or
+ * are inferred from a chant member's parameters by `joinKey()`. Some members
+ * already say what they read in files of their own: a dbt project names the
+ * tables it reads in its `sources:` YAML. A kind can name a link reader in its
+ * kinds file (`reader`), and chant runs that reader over the member's
+ * directory to derive the member's links.
+ *
+ * The readers are chant's code, listed in {@link KIND_LINK_READERS}: a kinds
+ * file stays data, and chant never imports the package that supplies it. A
+ * reader runs offline and parses files as data. It never invokes the tool
+ * whose files it reads, so `chant workspace check` keeps its promise of no
+ * network and no member code.
+ *
+ * A reader returns links with the file and line that state them, and gaps:
+ * the places it could not read without guessing, such as a templated value.
+ * `resolveLinks` (`./links.ts`) resolves the links beside the declared ones,
+ * with origin `kind-reader`; WSP093 and WSP094 name the reader's file and line,
+ * and WSP099 reports the gaps.
+ */
+
+import type { Declaration, Member } from "./declaration";
+import { resolveMemberFields, type FieldValue, type KindLinkReaderName, type KindRegistry } from "./kinds";
+import type { WorkspaceTree } from "./tree";
+import { readDbtProject } from "./dbt-project-reader";
+
+/** One link a reader found in the member's files. */
+export interface KindReadLink {
+  /**
+   * The producer the link names, or null when the member's files don't say
+   * which member produces it. A null producer is found among the members
+   * whose outputs include {@link anchor}; see `resolveLinks`.
+   */
+  member: string | null;
+  /** The producer's output the link reads. */
+  output: string;
+  /**
+   * The output that picks the producer when `member` is null; unset, the
+   * output itself. A dbt source column is anchored on its table, so a dropped
+   * column is missing from the member that still has the table.
+   */
+  anchor?: string;
+  /** The file that states the link, relative to the workspace root. */
+  file: string;
+  /** Its line, from 1. */
+  line: number;
+}
+
+/** A place the reader could not read without guessing. The links it would have stated are left out. */
+export interface KindReadGap {
+  /** Relative to the workspace root. */
+  file: string;
+  line: number;
+  message: string;
+}
+
+export interface KindLinkRead {
+  /** The reader that ran. */
+  reader: KindLinkReaderName;
+  links: KindReadLink[];
+  gaps: KindReadGap[];
+  /**
+   * The member field that names the producer, for messages that ask the
+   * author to set it when the files don't say (`sqlMember` for dbt).
+   */
+  producerField?: string;
+}
+
+/** What a reader gets: the member, its directory and resolved fields, and the tree read. */
+export interface KindReaderInput {
+  member: Member;
+  /** The member's directory, tree-relative; "" for the workspace root. */
+  dir: string;
+  /** The member's fields, defaults filled in (#3151); null when its kind declares none. */
+  fields: Record<string, FieldValue | null | Record<string, FieldValue | null>> | null;
+  tree: WorkspaceTree;
+}
+
+export type KindLinkReader = (input: KindReaderInput) => KindLinkRead;
+
+/** Every reader chant ships, by the name a kinds file uses. */
+export const LINK_READERS: Record<KindLinkReaderName, KindLinkReader> = {
+  "dbt-project": readDbtProject,
+};
+
+/**
+ * Run the link reader of every member whose kind has one. A member whose
+ * directory is missing is left out. A reader that throws is reported as a
+ * gap on the member's directory, so one unreadable project never stops
+ * the check.
+ */
+export function readKindLinks(declaration: Declaration, tree: WorkspaceTree, kinds: KindRegistry): Map<string, KindLinkRead> {
+  const out = new Map<string, KindLinkRead>();
+  for (const m of declaration.members) {
+    const kind = kinds.get(m.kind);
+    if (!kind?.reader) continue;
+    const dir = m.dir === "." ? "" : m.dir;
+    if (tree.stat(dir) !== "dir") continue;
+    const { fields } = resolveMemberFields(kind, m.fields);
+    try {
+      out.set(m.name, LINK_READERS[kind.reader]({ member: m, dir, fields, tree }));
+    } catch (err) {
+      out.set(m.name, { reader: kind.reader, links: [], gaps: [{ file: dir || ".", line: 1, message: `the ${kind.reader} reader failed: ${(err as Error).message}` }] });
+    }
+  }
+  return out;
+}

package/src/workspace/kinds.ts

@@ -128,8 +128,24 @@ export interface MemberKind {
   graph?: KindGraph;
   /** The fields a member entry of this kind may set, with their defaults (#3151); unset, the kind declares none. */
   fields?: Record<string, KindField>;
+  /**
+   * The link reader chant runs over a member's own files (#3932), by name, from
+   * {@link KIND_LINK_READERS}. It derives the member's links offline, from
+   * files it parses as data; unset, the member's links are only those its
+   * entry states.
+   */
+  reader?: KindLinkReaderName;
 }
 
+/**
+ * The link readers chant ships (#3932). A kind names one in its kinds file as
+ * `reader`. The readers are chant's own code, not the package's: a kinds file
+ * stays data, and chant never imports the package that supplies it. Closed:
+ * a kind naming another reader is kept without one, and the load reports it.
+ */
+export const KIND_LINK_READERS = ["dbt-project"] as const;
+export type KindLinkReaderName = (typeof KIND_LINK_READERS)[number];
+
 export interface KindRegistry {
   get(name: string): MemberKind | undefined;
   /** Every member kind's name, sorted, for "unknown kind" messages. */
@@ -384,7 +400,7 @@ export function parseKindData(text: string, source: string): KindData {
   const problems: string[] = [];
   const kinds: MemberKind[] = [];
   const seen = new Set<string>();
-  for (const k of (raw as { kinds: { name: string; description: string; precedence: number; probe: FileProbe; outputs?: string[]; graph?: KindGraph; fields?: Record<string, KindField> }[] }).kinds) {
+  for (const k of (raw as { kinds: { name: string; description: string; precedence: number; probe: FileProbe; outputs?: string[]; graph?: KindGraph; fields?: Record<string, KindField>; reader?: string }[] }).kinds) {
     if (BUILTIN_KIND_NAMES.includes(k.name)) {
       problems.push(`${source}: kind ${k.name} is built in and can't be supplied by a package`);
       continue;
@@ -408,7 +424,11 @@ export function parseKindData(text: string, source: string): KindData {
       source,
       ...(k.graph ? { graph: { lexicon: k.graph.lexicon, config: structuredClone(k.graph.config) } } : {}),
       ...(k.fields ? { fields: structuredClone(k.fields) } : {}),
+      ...(k.reader && (KIND_LINK_READERS as readonly string[]).includes(k.reader) ? { reader: k.reader as KindLinkReaderName } : {}),
     });
+    if (k.reader && !(KIND_LINK_READERS as readonly string[]).includes(k.reader)) {
+      problems.push(`${source}: kind ${k.name} names link reader ${k.reader}, which this chant does not have (it has ${KIND_LINK_READERS.join(", ")}); the kind is kept, and its members' links are only those their entries state`);
+    }
     for (const problem of fieldDefaultProblems(k.fields ?? {})) problems.push(`${source}: kind ${k.name} ${problem}`);
   }
   return { kinds, problems };

package/src/workspace/lexicon-outputs.ts

@@ -0,0 +1,272 @@
+/**
+ * The outputs a member's lexicons report about their own declarations
+ * (#3931), read offline for `chant workspace check`.
+ *
+ * A sql member exposes each table and each column as a workspace output
+ * (`public.orders`, `public.orders.customer_id`) without anyone writing
+ * `output(...)`. The names are the lexicon's to give, so the lexicon answers
+ * through its `workspaceOutputs` hook, the way a CI lexicon answers
+ * `graphEdges` (#3910).
+ *
+ * The check never runs member code. The member's source files are folded
+ * with `foldProject` under `sandbox`, which evaluates the fold subset and
+ * imports nothing from the project: a sql `table` template becomes the real
+ * entity because the tag is the lexicon's own function, called with the
+ * folded parts. A file that does not fold is not run either. It is handed to
+ * the hook as unread, and when it is one of that lexicon's files (it uses the
+ * lexicon's tags), the hook records a gap: the member's outputs are then
+ * incomplete, and a link to a name not in them is unresolved rather than
+ * missing.
+ *
+ * Folding is asynchronous and the checks are not, so the outputs are
+ * gathered before the checks run, only for the members some declared link
+ * targets, and only for the kinds whose outputs are read from source.
+ *
+ * Under `--at` the revision's files are not on disk to fold, so a member
+ * whose lexicons report outputs gets a gap instead.
+ */
+
+import { join, posix } from "node:path";
+import type { StaticFieldsRead } from "../config-static";
+import { isDeclarable, type Declarable } from "../declarable";
+import type { LexiconUnreadSource, LexiconWorkspaceOutput } from "../graph-ir";
+import type { BuildRootContext, BuildRootContribution, IntrinsicDef } from "../lexicon";
+import type { Declaration, ResolvedGroup } from "./declaration";
+import type { KindRegistry } from "./kinds";
+import { kindHandles } from "./links";
+import { readKindLinks } from "./kind-link-readers";
+import { memberExcludes, sourceFiles } from "./member-handles";
+import { memberDirOnDisk } from "./lexicon-symbols";
+import type { WorkspaceTree } from "./tree";
+
+/** What one member's lexicons report: its outputs, and why the list may be incomplete. */
+export interface MemberLexiconOutputs {
+  outputs: Pick<LexiconWorkspaceOutput, "name" | "previously">[];
+  gaps: string[];
+}
+
+/** The part of a lexicon plugin this reader uses. */
+export interface OutputsPlugin {
+  name?: string;
+  intrinsics?: () => IntrinsicDef[];
+  buildRoots?: (ctx: BuildRootContext) => Promise<BuildRootContribution>;
+  workspaceOutputs?: (entities: ReadonlyMap<string, Declarable>, unread?: readonly LexiconUnreadSource[]) => { outputs: LexiconWorkspaceOutput[]; gaps?: string[] } | undefined;
+}
+
+/** Loads a lexicon plugin by name; `loadPlugin` from the CLI unless a test swaps it. */
+export type OutputsPluginLoader = (name: string) => Promise<OutputsPlugin>;
+
+export interface GatherLexiconOutputsOptions {
+  /** Read under `--at`: the member's files are a revision's, not on disk, so nothing is folded. */
+  atRevision?: boolean;
+  load?: OutputsPluginLoader;
+}
+
+const firstLine = (err: unknown): string => (err instanceof Error ? err.message : String(err)).split("\n")[0];
+
+const memberDir = (dir: string) => (dir === "." ? "" : dir);
+
+/** The `sourceDir` a member's config sets, read without running it. */
+function staticSourceDir(read: StaticFieldsRead): string | undefined {
+  const value = read.status === "read" ? read.fields.sourceDir : undefined;
+  if (typeof value !== "string") return undefined;
+  const dir = posix.normalize(value.replace(/\\/g, "/")).replace(/\/+$/, "");
+  return dir === "." || dir === "" ? undefined : dir;
+}
+
+/**
+ * Only the files under the member's `sourceDir`, when its config sets one, as
+ * a build reads them (#3941). Without one, every source file of the member.
+ */
+function withinSourceDir(files: string[], member: string, sourceDir: string | undefined): string[] {
+  if (!sourceDir) return files;
+  const prefix = `${member ? `${member}/` : ""}${sourceDir}/`;
+  return files.filter((f) => f.startsWith(prefix));
+}
+
+/**
+ * A folded file's exports, less the entities of lexicons the member doesn't
+ * declare (#3941). An Op is a `chant` entity, and several files exporting
+ * `op` would otherwise collide and leave the whole member unread, though no
+ * hook reads Ops. Values that are not entities (a composite) are kept.
+ */
+function ownExports(exports: ReadonlyMap<string, unknown>, lexicons: readonly string[]): Record<string, unknown> {
+  const own = new Set(lexicons);
+  const out: Record<string, unknown> = {};
+  for (const [name, value] of exports) {
+    if (isDeclarable(value) && !own.has(value.lexicon)) continue;
+    out[name] = value;
+  }
+  return out;
+}
+
+/**
+ * The lexicon names a member's config declares, read without running it.
+ * Unlike the intent walk, a config that can't be read statically is not
+ * loaded: the check runs no member code, so it is a gap.
+ */
+async function staticMemberLexicons(dir: string): Promise<string[]> {
+  const { readLexiconDeclarationsStatically } = await import("../config-static");
+  const read = readLexiconDeclarationsStatically(dir);
+  if (read.status === "no-config") return [];
+  if (read.status !== "read") throw new Error(read.reason);
+  const { lexiconNames, registerLexiconDeclarations } = await import("../lexicon-module");
+  const { dirname } = await import("node:path");
+  registerLexiconDeclarations(read.entries, dirname(read.configPath));
+  return lexiconNames(read.entries);
+}
+
+/**
+ * The lexicon outputs of every member a declared link, or a link a kind's
+ * reader found (#3932), targets, keyed by
+ * member name. A member whose lexicons report nothing is left out.
+ * `tree` is the working tree under `root`.
+ */
+export async function gatherLexiconOutputs(
+  root: string,
+  declaration: Declaration,
+  tree: WorkspaceTree,
+  groups: readonly ResolvedGroup[],
+  kinds: KindRegistry,
+  options: GatherLexiconOutputsOptions = {},
+): Promise<Map<string, MemberLexiconOutputs>> {
+  const out = new Map<string, MemberLexiconOutputs>();
+  const targets = new Set(declaration.members.flatMap((m) => m.links.filter((l) => l.member !== m.name).map((l) => l.member)));
+  // Links a kind's reader finds in a member's files target members too (#3932). One that names no
+  // producer may be any other member's, so every other member is read.
+  for (const [consumer, read] of readKindLinks(declaration, tree, kinds)) {
+    for (const link of read.links) {
+      if (link.member !== null) targets.add(link.member);
+      else for (const m of declaration.members) if (m.name !== consumer) targets.add(m.name);
+    }
+  }
+  for (const m of declaration.members) {
+    if (!targets.has(m.name)) continue;
+    const kind = kinds.get(m.kind);
+    if (!kind || kindHandles(m, kind) || tree.stat(memberDir(m.dir)) !== "dir") continue;
+    const read = await readMemberLexiconOutputs(root, declaration, tree, groups, m, options);
+    if (read) out.set(m.name, read);
+  }
+  return out;
+}
+
+/** The lexicons the files import, by the package name chant gives a lexicon. */
+function importedLexicons(tree: WorkspaceTree, files: readonly string[]): string[] {
+  const names = new Set<string>();
+  for (const file of files) {
+    let text: string;
+    try {
+      text = tree.read(file);
+    } catch {
+      continue;
+    }
+    for (const m of text.matchAll(/from\s+["']@intentius\/chant-lexicon-([a-z0-9-]+)(?:\/[^"']*)?["']/g)) names.add(m[1]!);
+  }
+  return [...names].sort();
+}
+
+async function readMemberLexiconOutputs(
+  root: string,
+  declaration: Declaration,
+  tree: WorkspaceTree,
+  groups: readonly ResolvedGroup[],
+  m: Declaration["members"][number],
+  options: GatherLexiconOutputsOptions,
+): Promise<MemberLexiconOutputs | undefined> {
+  const dir = memberDirOnDisk(root, m.dir);
+  const { readConfigFieldsStatically } = await import("../config-static");
+  const files = withinSourceDir(
+    sourceFiles(tree, memberDir(m.dir), memberExcludes(declaration, groups, m)),
+    memberDir(m.dir),
+    staticSourceDir(readConfigFieldsStatically(dir, ["sourceDir"])),
+  );
+  let names: string[];
+  try {
+    names = await staticMemberLexicons(dir);
+  } catch {
+    // A config that can't be read statically is not run. The lexicons the
+    // source imports stand in for the ones it names.
+    names = importedLexicons(tree, files);
+  }
+  if (names.length === 0) return undefined;
+  const load = options.load ?? ((await import("../cli/plugins")).loadPlugin as unknown as OutputsPluginLoader);
+  const plugins: { name: string; plugin: OutputsPlugin }[] = [];
+  const gaps: string[] = [];
+  for (const name of names) {
+    try {
+      plugins.push({ name, plugin: await load(name) });
+    } catch (err) {
+      gaps.push(`lexicon "${name}" could not be loaded, so the outputs it reads were not (${firstLine(err)})`);
+    }
+  }
+  const reporting = plugins.filter((p) => typeof p.plugin.workspaceOutputs === "function");
+  if (reporting.length === 0) return gaps.length ? { outputs: [], gaps } : undefined;
+  if (options.atRevision) {
+    return { outputs: [], gaps: [`the outputs of lexicon ${reporting.map((p) => p.name).join(", ")} are read from the working tree's files, not a revision's`] };
+  }
+
+  // Fold the member's source, never run it.
+  const abs = (file: string) => join(root, ...file.split("/"));
+  const intrinsics = plugins.flatMap((p) => {
+    try {
+      return p.plugin.intrinsics?.() ?? [];
+    } catch {
+      return [];
+    }
+  });
+  let entities: Map<string, Declarable> = new Map();
+  const unread: LexiconUnreadSource[] = [];
+  try {
+    const { foldProject } = await import("../discovery/fold-import");
+    const { collectEntities } = await import("../discovery/collect");
+    const verdicts = await foldProject(files.map(abs), intrinsics, { lexicons: names, sandbox: true });
+    const folded: { file: string; exports: Record<string, unknown> }[] = [];
+    for (const file of files) {
+      const v = verdicts.get(abs(file));
+      if (v?.verdict === "fold" && v.exports) {
+        folded.push({ file: abs(file), exports: ownExports(v.exports, names) });
+        continue;
+      }
+      let text = "";
+      try {
+        text = tree.read(file);
+      } catch {
+        // An unreadable file is still unread; the hook decides whether it matters.
+      }
+      const reason = v?.reason ?? (v?.taintedBy ? `it imports ${v.taintedBy.from}, which does not fold` : "it does not fold");
+      unread.push({ file, text, reason });
+    }
+    entities = collectEntities(folded, dir);
+  } catch (err) {
+    return { outputs: [], gaps: [...gaps, `${m.dir}: its source could not be folded, so its lexicons' outputs were not read (${firstLine(err)})`] };
+  }
+
+  // A reporting lexicon's own build roots (the sql lexicon's `.sql` files) join
+  // the folded entities, as they join a build's. Only these lexicons' hooks
+  // run: they read files, and start nothing.
+  for (const { name, plugin } of reporting) {
+    if (typeof plugin.buildRoots !== "function") continue;
+    const read = readConfigFieldsStatically(dir, [name, "sourceDir"]);
+    const config = read.status === "read" ? read.fields : {};
+    const sourceDir = typeof config.sourceDir === "string" ? join(dir, config.sourceDir) : undefined;
+    try {
+      const contributed = await plugin.buildRoots({ projectRoot: dir, config, entities, ...(sourceDir ? { sourceDir } : {}) });
+      for (const [key, entity] of contributed.entities) if (!entities.has(key)) entities.set(key, entity);
+    } catch (err) {
+      gaps.push(`lexicon "${name}" could not read the member's other source files, so their declarations were not read (${firstLine(err)})`);
+    }
+  }
+
+  const outputs: MemberLexiconOutputs["outputs"] = [];
+  for (const { name, plugin } of reporting) {
+    try {
+      const report = plugin.workspaceOutputs!(entities, unread);
+      if (!report) continue;
+      for (const o of report.outputs) outputs.push({ name: o.name, ...(o.previously?.length ? { previously: o.previously } : {}) });
+      for (const g of report.gaps ?? []) gaps.push(g);
+    } catch (err) {
+      gaps.push(`lexicon "${name}" workspaceOutputs threw, so its outputs were not read (${firstLine(err)})`);
+    }
+  }
+  return { outputs, gaps };
+}

package/src/workspace/links.test.ts

@@ -308,6 +308,28 @@ describe("the graph's links section", () => {
   test("the linked member pairs, for the pipeline checks' environment rule", async () => {
     expect(declaredMemberLinks(decl)).toEqual([{ consumer: "web", producer: "shared" }]);
   });
+
+  test("a link to an output's previous name says what it was renamed to (#3931)", () => {
+    const d = parseDeclaration(
+      declaration([
+        { name: "db", dir: "db", kind: "chant" },
+        { name: "web", dir: "web", kind: "chant", links: [{ member: "db", output: "public.orders.customer_id" }] },
+      ]),
+      "chant.workspace.json",
+    );
+    const [row] = graphLinks(d, {
+      composed: ["db", "web"],
+      exports: [
+        { member: "db", name: "public.orders", node: "db/orders" },
+        { member: "db", name: "public.orders.buyer_id", node: "db/orders", previously: ["public.orders.customer_id"] },
+      ],
+      imports: [],
+    });
+    expect(row).toMatchObject({ status: "missing" });
+    expect(row!.reason).toBe(
+      "db has no output public.orders.customer_id; it was renamed to public.orders.buyer_id, so link to that instead; its outputs: public.orders, public.orders.buyer_id",
+    );
+  });
 });
 
 describe("the declaration's link fields", () => {