@intentius/chant 0.124.22 → 0.124.24

package/src/workspace/dbt-kind.test.ts

@@ -0,0 +1,283 @@
+/**
+ * The `dbt` member kind (#3932): the sql lexicon's kinds file, the
+ * `dbt-project` link reader, and `chant workspace check` over the links it
+ * finds. The producer here is an `other` member whose entry lists the
+ * outputs a sql member exposes (#3931), so the reader's names are checked
+ * against the naming contract without a sql project.
+ */
+
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, describe, expect, test } from "vitest";
+import { cleanScratch, contract, declaration, REPO, repo } from "./__fixtures__/contract-repo";
+import { runDeclarationChecks } from "./checks";
+import { parseDeclaration } from "./declaration";
+import { readDbtProject } from "./dbt-project-reader";
+import { readKindLinks } from "./kind-link-readers";
+import { createKindRegistry, parseKindData } from "./kinds";
+import { graphLinks, resolveLinks, type LinkTableRow, type MemberHandles } from "./links";
+import { workingTree } from "./tree";
+import graphSchema from "./graph.schema.json";
+
+afterAll(cleanScratch);
+
+const SQL_KINDS = readFileSync(join(REPO, "lexicons", "sql", "workspace-kinds.json"), "utf-8");
+const SQL_PACKAGE = JSON.stringify({ name: "@intentius/chant-lexicon-sql", version: "0.0.0", exports: { "./workspace-kinds": "./workspace-kinds.json" } });
+
+const ORDERS = ["public.orders", "public.orders.id", "public.orders.customer_id", "public.orders.status"];
+
+const SOURCES = `version: 2
+
+sources:
+  - name: shop
+    schema: public
+    tables:
+      - name: orders
+        columns:
+          - name: id
+          - name: customer_id
+            description: who ordered
+`;
+
+interface Fixture {
+  outputs?: string[];
+  dbtFields?: Record<string, unknown>;
+  members?: unknown[];
+  files?: Record<string, string>;
+}
+
+/** A workspace with a `db` producer, a `analytics` dbt project and the sql lexicon's kinds pinned by path. */
+function workspace(opts: Fixture = {}): string {
+  return repo({
+    "chant.workspace.json": declaration(
+      [
+        { name: "db", dir: "db", kind: "other", because: "stands in for a sql member (#3931)", suppress: [{ check: "WSP009", because: "a stand-in" }], outputs: opts.outputs ?? ORDERS },
+        { name: "analytics", dir: "analytics", kind: "dbt", ...(opts.dbtFields === undefined ? { fields: { sqlMember: "db" } } : Object.keys(opts.dbtFields).length ? { fields: opts.dbtFields } : {}) },
+        ...(opts.members ?? []),
+      ],
+      { pins: [{ path: "plugins/sql" }] },
+    ),
+    "plugins/sql/package.json": SQL_PACKAGE,
+    "plugins/sql/workspace-kinds.json": SQL_KINDS,
+    "db/README": "",
+    "analytics/dbt_project.yml": "name: analytics\nversion: '1.0'\nprofile: analytics\n",
+    "analytics/models/staging/sources.yml": SOURCES,
+    ...(opts.files ?? {}),
+  });
+}
+
+const rows = (links: LinkTableRow[]) =>
+  links
+    .filter((r) => r.origin === "kind-reader")
+    .map((r) => (r.status === "ambiguous" ? `${r.consumer} -> ? ${r.input} ambiguous ${r.candidates.map((c) => c.producer).join("|")}` : `${r.consumer} -> ${r.producer}.${r.output} ${r.status} ${r.file}:${r.line}`));
+
+describe("the dbt kind in the sql lexicon's kinds file (#3932)", () => {
+  test("validates, probes dbt_project.yml and names the dbt-project reader", () => {
+    const { kinds, problems } = parseKindData(SQL_KINDS, "@intentius/chant-lexicon-sql");
+    expect(problems).toEqual([]);
+    expect(kinds.map((k) => [k.name, k.reader, k.probe])).toEqual([["dbt", "dbt-project", { anyFile: ["dbt_project.yml"] }]]);
+    expect(Object.keys(kinds[0].fields ?? {})).toEqual(["sqlMember", "dialect"]);
+  });
+
+  test("a reader this chant doesn't have keeps the kind without one, and says so", () => {
+    const text = JSON.stringify({ schema: 1, kinds: [{ name: "lake", description: "a lake", precedence: 10, probe: { anyFile: ["lake.yml"] }, reader: "lake-tables" }] });
+    const { kinds, problems } = parseKindData(text, "pkg");
+    expect(kinds.map((k) => [k.name, k.reader])).toEqual([["lake", undefined]]);
+    expect(problems).toEqual([expect.stringMatching(/kind lake names link reader lake-tables, which this chant does not have/)]);
+  });
+});
+
+describe("dbt sources as links, in chant workspace check (#3932)", () => {
+  test("each source table and column links to the sql member's <schema>.<table>[.<column>] output", async () => {
+    const report = await runDeclarationChecks(workspace());
+    expect(report.diagnostics).toEqual([]);
+    expect(report.ok).toBe(true);
+    expect(rows(report.links)).toEqual([
+      "analytics -> db.public.orders resolved analytics/models/staging/sources.yml:7",
+      "analytics -> db.public.orders.id resolved analytics/models/staging/sources.yml:9",
+      "analytics -> db.public.orders.customer_id resolved analytics/models/staging/sources.yml:10",
+    ]);
+    expect(report.links[0]).toMatchObject({ origin: "kind-reader", reader: "dbt-project", kind: "output", resolves: "source" });
+  });
+
+  test("acceptance: dropping a column a source reads fails check and names the dbt source file and line", async () => {
+    const report = await runDeclarationChecks(workspace({ outputs: ["public.orders", "public.orders.id", "public.orders.status"] }));
+    expect(report.ok).toBe(false);
+    expect(report.diagnostics.map((d) => `${d.ruleId}:${d.severity}:${d.entity}:${d.file}:${d.line}`)).toEqual([
+      "WSP093:error:analytics:analytics/models/staging/sources.yml:10",
+    ]);
+    expect(report.diagnostics[0].message).toMatch(
+      /^analytics's link to db output public\.orders\.customer_id \(read by dbt-project from analytics\/models\/staging\/sources\.yml:10\) does not resolve: db has no output public\.orders\.customer_id/,
+    );
+  });
+
+  test("dropping the table fails the table and each column the source names", async () => {
+    const report = await runDeclarationChecks(workspace({ outputs: ["public.customers"] }));
+    expect(report.diagnostics.map((d) => `${d.ruleId}:${d.line}`)).toEqual(["WSP093:7", "WSP093:9", "WSP093:10"]);
+  });
+
+  test("a rename shows the old output missing with the near name", async () => {
+    const report = await runDeclarationChecks(workspace({ outputs: ["public.orders", "public.orders.id", "public.orders.customerId"] }));
+    expect(report.diagnostics[0].message).toMatch(/did you mean public\.orders\.customerId\? Links match exactly/);
+  });
+
+  test("a producer whose outputs can't all be read, such as a view's columns, keeps the link unresolved, not missing", () => {
+    const decl = parseDeclaration(
+      declaration([
+        { name: "db", dir: "db", kind: "chant" },
+        { name: "analytics", dir: "analytics", kind: "dbt" },
+      ]),
+      "chant.workspace.json",
+    );
+    const handles = new Map<string, MemberHandles>([
+      ["db", { member: "db", outputs: [{ name: "public.orders" }], inputs: [], complete: false, why: "the columns of view public.orders can't be derived statically", exposesNone: false }],
+    ]);
+    const read = new Map([["analytics", { reader: "dbt-project" as const, gaps: [], producerField: "sqlMember", links: [{ member: null, output: "public.orders.customer_id", anchor: "public.orders", file: "analytics/models/s.yml", line: 4 }] }]]);
+    const [row] = resolveLinks(decl, handles, undefined, read);
+    expect(row).toMatchObject({ origin: "kind-reader", producer: "db", status: "unresolved", file: "analytics/models/s.yml", line: 4 });
+    expect((row as { reason: string }).reason).toMatch(/whose outputs could not all be read in source \(the columns of view public\.orders/);
+  });
+
+  test("a column the sql member renamed says what it was renamed to (#3931's previously hint)", () => {
+    const decl = parseDeclaration(
+      declaration([
+        { name: "db", dir: "db", kind: "chant" },
+        { name: "analytics", dir: "analytics", kind: "dbt", fields: { sqlMember: "db" } },
+      ]),
+      "chant.workspace.json",
+    );
+    const handles = new Map<string, MemberHandles>([
+      ["db", { member: "db", outputs: [{ name: "public.orders" }, { name: "public.orders.buyer_id", previously: ["public.orders.customer_id"] }], inputs: [], complete: true, why: null, exposesNone: false }],
+    ]);
+    const read = new Map([["analytics", { reader: "dbt-project" as const, gaps: [], producerField: "sqlMember", links: [{ member: "db", output: "public.orders.customer_id", anchor: "public.orders", file: "analytics/models/s.yml", line: 4 }] }]]);
+    const [row] = resolveLinks(decl, handles, undefined, read);
+    expect(row).toMatchObject({ origin: "kind-reader", status: "missing" });
+    expect((row as { reason: string }).reason).toMatch(/it was renamed to public\.orders\.buyer_id, so link to that instead/);
+  });
+
+  test("without sqlMember, the one member that exposes the table is the producer", async () => {
+    const report = await runDeclarationChecks(workspace({ dbtFields: {} }));
+    expect(report.diagnostics).toEqual([]);
+    expect(rows(report.links)[2]).toBe("analytics -> db.public.orders.customer_id resolved analytics/models/staging/sources.yml:10");
+  });
+
+  test("without sqlMember, a column dropped from the only producer still fails", async () => {
+    const report = await runDeclarationChecks(workspace({ dbtFields: {}, outputs: ["public.orders", "public.orders.id"] }));
+    expect(report.diagnostics.map((d) => `${d.ruleId}:${d.line}`)).toEqual(["WSP093:10"]);
+  });
+
+  test("without sqlMember, a table two members expose is ambiguous, and the message names the field", async () => {
+    const report = await runDeclarationChecks(
+      workspace({ dbtFields: {}, members: [{ name: "replica", dir: "replica", kind: "other", because: "a copy", suppress: [{ check: "WSP009", because: "a stand-in" }], outputs: ORDERS }], files: { "replica/README": "" } }),
+    );
+    expect(rows(report.links)[0]).toBe("analytics -> ? public.orders ambiguous db|replica");
+    const ids = report.diagnostics.map((d) => `${d.ruleId}:${d.severity}:${d.file}:${d.line}`);
+    expect(ids).toContain("WSP095:warning:analytics/models/staging/sources.yml:7");
+    expect(report.diagnostics.find((d) => d.ruleId === "WSP095")!.message).toMatch(/set fields\.sqlMember on analytics to name it/);
+  });
+
+  test("a sqlMember that names no member is invalid, reported against the field", async () => {
+    const report = await runDeclarationChecks(workspace({ dbtFields: { sqlMember: "warehouse" } }));
+    expect(rows(report.links)[0]).toBe("analytics -> warehouse.public.orders invalid analytics/models/staging/sources.yml:7");
+    expect(report.links.find((r) => r.origin === "kind-reader")!.reason).toMatch(/^fields\.sqlMember: member analytics links to warehouse, which is not a member/);
+  });
+
+  test("a templated schema is a gap (WSP099), never rendered, and its tables are not linked", async () => {
+    const report = await runDeclarationChecks(
+      workspace({
+        files: {
+          "analytics/models/raw.yml": "version: 2\nsources:\n  - name: raw\n    schema: \"{{ target.schema }}_raw\"\n    tables:\n      - name: events\n",
+        },
+      }),
+    );
+    expect(report.ok).toBe(true);
+    expect(report.diagnostics.map((d) => `${d.ruleId}:${d.severity}:${d.file}:${d.line}`)).toEqual(["WSP099:info:analytics/models/raw.yml:4"]);
+    expect(report.diagnostics[0].message).toMatch(/source raw: schema .* is a Jinja template or not a name; chant reads names as declared and never renders them for a target/);
+    expect(rows(report.links).some((r) => r.includes("events"))).toBe(false);
+  });
+});
+
+describe("the dbt-project reader (#3932)", () => {
+  const read = (files: Record<string, string>, fields: Record<string, unknown> = {}) => {
+    const root = repo({ "dbt_project.yml": "name: p\n", ...files });
+    return readDbtProject({
+      member: { name: "p" } as never,
+      dir: "",
+      fields: { sqlMember: null, dialect: "postgres", ...fields } as never,
+      tree: workingTree(root),
+    });
+  };
+  const outputs = (r: ReturnType<typeof read>) => r.links.map((l) => l.output);
+
+  test("the schema defaults to the source name, as dbt's does, and identifier overrides the table name", () => {
+    const r = read({ "models/s.yml": "sources:\n  - name: shop\n    tables:\n      - name: orders\n        identifier: orders_v2\n" });
+    expect(outputs(r)).toEqual(["shop.orders_v2"]);
+    expect(r.gaps).toEqual([]);
+  });
+
+  test("postgres: dbt quotes schema and identifier by default, so they keep their case; unquoted names fold", () => {
+    const sources = (quoting: string) =>
+      `sources:\n  - name: Shop\n${quoting}    tables:\n      - name: Orders\n        columns:\n          - name: CustomerId\n          - name: OrderId\n            quote: true\n`;
+    expect(outputs(read({ "models/s.yml": sources("") }))).toEqual(["Shop.Orders", "Shop.Orders.customerid", "Shop.Orders.OrderId"]);
+    expect(outputs(read({ "models/s.yml": sources("    quoting:\n      schema: false\n      identifier: false\n") }))).toEqual([
+      "shop.orders",
+      "shop.orders.customerid",
+      "shop.orders.OrderId",
+    ]);
+    // dbt_project.yml's quoting applies below it.
+    expect(outputs(read({ "dbt_project.yml": "name: p\nquoting:\n  identifier: false\n", "models/s.yml": sources("") }))[0]).toBe("Shop.orders");
+  });
+
+  test("clickhouse keeps every name as written; the source schema is the ClickHouse database", () => {
+    const r = read({ "models/s.yml": "sources:\n  - name: events\n    schema: Analytics\n    tables:\n      - name: PageViews\n        columns:\n          - name: UserId\n" }, { dialect: "clickhouse" });
+    expect(outputs(r)).toEqual(["Analytics.PageViews", "Analytics.PageViews.UserId"]);
+  });
+
+  test("model-paths say where sources live, and a disabled source or table is left out", () => {
+    const r = read({
+      "dbt_project.yml": "name: p\nmodel-paths: [transform]\n",
+      "models/ignored.yml": "sources:\n  - name: a\n    tables:\n      - name: t\n",
+      "transform/nested/s.yaml":
+        "sources:\n  - name: b\n    tables:\n      - name: kept\n      - name: off\n        config:\n          enabled: false\n  - name: c\n    enabled: false\n    tables:\n      - name: t\n",
+    });
+    expect(outputs(r)).toEqual(["b.kept"]);
+  });
+
+  test("templated names are gaps with their lines, and the rest of the file is still read", () => {
+    const r = read({
+      "models/s.yml":
+        "sources:\n  - name: shop\n    tables:\n      - name: orders\n        identifier: \"{{ var('orders') }}\"\n      - name: items\n        columns:\n          - name: \"{{ env_var('COL') }}\"\n          - name: sku\n",
+    });
+    expect(outputs(r)).toEqual(["shop.items", "shop.items.sku"]);
+    expect(r.gaps.map((g) => `${g.file}:${g.line}`)).toEqual(["models/s.yml:5", "models/s.yml:8"]);
+  });
+
+  test("a table named like its source gets its own line; a column named like its table too", () => {
+    const r = read({ "models/s.yml": "sources:\n  - name: orders\n    schema: public\n    tables:\n      - name: orders\n        columns:\n          - name: orders\n" });
+    expect(r.links.map((l) => `${l.output}@${l.line}`)).toEqual(["public.orders@5", "public.orders.orders@7"]);
+  });
+
+  test("YAML that doesn't parse is a gap at its line", () => {
+    const r = read({ "models/s.yml": "sources:\n  - name: a\n    tables: [\n" });
+    expect(r.links).toEqual([]);
+    expect(r.gaps).toHaveLength(1);
+    expect(r.gaps[0].file).toBe("models/s.yml");
+  });
+});
+
+describe("dbt links in chant workspace graph (#3932)", () => {
+  test("graphLinks carries the reader's rows with their file and line, valid against graph.schema.json", () => {
+    const root = workspace();
+    const tree = workingTree(root);
+    const decl = parseDeclaration(tree.read("chant.workspace.json"), "chant.workspace.json");
+    const kinds = createKindRegistry(parseKindData(SQL_KINDS, "@intentius/chant-lexicon-sql").kinds);
+    const links = graphLinks(decl, { composed: [], exports: [], imports: [] }, kinds, readKindLinks(decl, tree, kinds));
+    expect(rows(links)).toEqual([
+      "analytics -> db.public.orders resolved analytics/models/staging/sources.yml:7",
+      "analytics -> db.public.orders.id resolved analytics/models/staging/sources.yml:9",
+      "analytics -> db.public.orders.customer_id resolved analytics/models/staging/sources.yml:10",
+    ]);
+    const linkSchema = { $schema: "https://json-schema.org/draft/2020-12/schema", $defs: (graphSchema as { $defs: object }).$defs, type: "array", items: { $ref: "#/$defs/link" } };
+    contract(linkSchema).expectValid(links);
+  });
+});

package/src/workspace/dbt-project-reader.ts

@@ -0,0 +1,291 @@
+/**
+ * The `dbt-project` link reader (#3932): a dbt project's sources, as links to
+ * the tables and columns a sql member exposes (#3931).
+ *
+ * The reader parses YAML and nothing else. It never runs dbt, never renders
+ * Jinja and never reads `profiles.yml` or a target, so the links it derives
+ * are the same in every environment.
+ *
+ * - The project is `dbt_project.yml`. Sources are read from every `.yml` and
+ *   `.yaml` file under its `model-paths` (`source-paths` in older projects;
+ *   `models` when it names neither), as dbt reads them.
+ * - Each enabled source table links to `<schema>.<table>`, and each of its
+ *   `columns` to `<schema>.<table>.<column>`. The schema is the source's
+ *   `schema`, or the source's `name` when it sets none, which is dbt's own
+ *   fallback. The table is its `identifier`, or its `name`. `database` is not
+ *   part of the name: a Postgres output has none, and dbt-clickhouse reads a
+ *   source's `schema` as the ClickHouse database.
+ * - Case follows the member's `dialect` field. For `postgres`, an unquoted
+ *   name folds to lowercase and a quoted one keeps its case. dbt quotes a
+ *   source's schema and identifier on Postgres unless `quoting:` (in
+ *   `dbt_project.yml`, on the source, or on the table) turns it off, and a
+ *   column is quoted only with `quote: true`. For `clickhouse`, names are
+ *   kept as written.
+ * - A value that is a Jinja template (`{{ ... }}` or `{% ... %}`) is not
+ *   resolved: the reader records a gap and leaves that source, table or
+ *   column out, rather than guess what a target would render.
+ * - The producer is the member the `sqlMember` field names. Unset, each link
+ *   leaves the producer to `resolveLinks`, which picks the one member that
+ *   exposes the table.
+ */
+
+import { posix } from "node:path";
+import { parseYAMLDocument, YAMLParseError } from "../yaml";
+import type { KindLinkRead, KindReadGap, KindReadLink, KindReaderInput } from "./kind-link-readers";
+import { joinPath, skippedDir, type WorkspaceTree } from "./tree";
+
+export const DBT_PROJECT_FILE = "dbt_project.yml";
+const DEFAULT_MODEL_PATHS = ["models"];
+
+/** The dialects whose naming the reader follows; the member's `dialect` field picks one. */
+export const DBT_DIALECTS = ["postgres", "clickhouse"] as const;
+type Dialect = (typeof DBT_DIALECTS)[number];
+
+interface Quoting {
+  schema: boolean;
+  identifier: boolean;
+}
+
+const isTemplated = (v: string) => /\{\{|\{%/.test(v);
+const isObject = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === "object" && !Array.isArray(v);
+
+export function readDbtProject(input: KindReaderInput): KindLinkRead {
+  const { dir, tree, fields } = input;
+  const links: KindReadLink[] = [];
+  const gaps: KindReadGap[] = [];
+  const result: KindLinkRead = { reader: "dbt-project", links, gaps, producerField: "sqlMember" };
+  const projectFile = joinPath(dir, DBT_PROJECT_FILE);
+
+  const producer = typeof fields?.sqlMember === "string" && fields.sqlMember !== "" ? fields.sqlMember : null;
+  const dialect: Dialect = fields?.dialect === "clickhouse" ? "clickhouse" : "postgres";
+
+  let project: unknown;
+  try {
+    project = parseYAMLDocument(tree.read(projectFile));
+  } catch (err) {
+    gaps.push({ file: projectFile, line: err instanceof YAMLParseError ? err.line : 1, message: `${DBT_PROJECT_FILE} could not be read (${(err as Error).message}), so no source was read` });
+    return result;
+  }
+  if (!isObject(project)) {
+    gaps.push({ file: projectFile, line: 1, message: `${DBT_PROJECT_FILE} is not a mapping, so no source was read` });
+    return result;
+  }
+  const projectLines = lines(tree.read(projectFile));
+
+  // Where sources live: model-paths, or the older source-paths.
+  const key = project["model-paths"] !== undefined ? "model-paths" : project["source-paths"] !== undefined ? "source-paths" : null;
+  const rawPaths = key ? project[key] : DEFAULT_MODEL_PATHS;
+  const paths: string[] = [];
+  if (!Array.isArray(rawPaths)) {
+    gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} is not a list, so no source was read` });
+    return result;
+  }
+  for (const p of rawPaths) {
+    if (typeof p !== "string" || isTemplated(p)) {
+      gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} entry ${JSON.stringify(p)} is not a plain path; its sources were not read` });
+      continue;
+    }
+    const at = posix.normalize(joinPath(dir, p));
+    if (at.startsWith("../") || at === ".." || posix.isAbsolute(p)) {
+      gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} entry ${p} lies outside the workspace; its sources were not read` });
+      continue;
+    }
+    paths.push(at === "." ? "" : at);
+  }
+
+  const quoting = readQuoting({ schema: true, identifier: true }, project.quoting, projectFile, keyLine(projectLines, "quoting"), "dbt_project.yml quoting", gaps);
+
+  const files = new Set<string>();
+  for (const p of paths) for (const f of yamlFiles(tree, p)) files.add(f);
+  const name = (v: string, quoted: boolean) => (dialect === "postgres" && !quoted ? v.toLowerCase() : v);
+
+  for (const file of [...files].sort()) {
+    let text: string;
+    let doc: unknown;
+    try {
+      text = tree.read(file);
+      doc = parseYAMLDocument(text);
+    } catch (err) {
+      gaps.push({ file, line: err instanceof YAMLParseError ? err.line : 1, message: `the file could not be read as YAML (${(err as Error).message}); any sources in it were not read` });
+      continue;
+    }
+    if (!isObject(doc) || doc.sources === undefined || doc.sources === null) continue;
+    const fileLines = lines(text);
+    const sourcesLine = keyLine(fileLines, "sources");
+    if (!Array.isArray(doc.sources)) {
+      gaps.push({ file, line: sourcesLine, message: "sources is not a list; its sources were not read" });
+      continue;
+    }
+    let cursor = sourcesLine;
+    for (const source of doc.sources) {
+      if (!isObject(source)) continue;
+      const sourceName = scalar(source.name);
+      const sourceLine = nameLine(fileLines, sourceName, cursor);
+      cursor = sourceLine;
+      if (sourceName === undefined) {
+        gaps.push({ file, line: sourceLine, message: "a source has no name; its tables were not read" });
+        continue;
+      }
+      const where = `source ${sourceName}`;
+      if (isTemplated(sourceName)) {
+        gaps.push({ file, line: sourceLine, message: `${where}: its name is a Jinja template, which chant does not render; its tables were not read` });
+        continue;
+      }
+      const enabled = readEnabled(source, file, sourceLine, where, gaps);
+      if (enabled !== true) continue;
+      const schema = source.schema === undefined || source.schema === null ? sourceName : scalar(source.schema);
+      if (schema === undefined || isTemplated(schema)) {
+        gaps.push({
+          file,
+          line: valueLine(fileLines, "schema", sourceLine),
+          message: `${where}: schema ${JSON.stringify(source.schema)} is a Jinja template or not a name; chant reads names as declared and never renders them for a target, so its tables were not read`,
+        });
+        continue;
+      }
+      const sourceQuoting = readQuoting(quoting, source.quoting, file, sourceLine, `${where} quoting`, gaps);
+      if (source.tables === undefined || source.tables === null) continue;
+      if (!Array.isArray(source.tables)) {
+        gaps.push({ file, line: sourceLine, message: `${where}: tables is not a list; its tables were not read` });
+        continue;
+      }
+      let tableCursor = sourceLine;
+      for (const table of source.tables) {
+        if (!isObject(table)) continue;
+        const tableName = scalar(table.name);
+        const tableLine = nameLine(fileLines, tableName, tableCursor);
+        tableCursor = tableLine;
+        if (tableName === undefined) {
+          gaps.push({ file, line: tableLine, message: `${where}: a table has no name; it was not read` });
+          continue;
+        }
+        const tableWhere = `${where} table ${tableName}`;
+        if (isTemplated(tableName)) {
+          gaps.push({ file, line: tableLine, message: `${tableWhere}: its name is a Jinja template, which chant does not render; it was not read` });
+          continue;
+        }
+        if (readEnabled(table, file, tableLine, tableWhere, gaps) !== true) continue;
+        const identifier = table.identifier === undefined || table.identifier === null ? tableName : scalar(table.identifier);
+        if (identifier === undefined || isTemplated(identifier)) {
+          gaps.push({
+            file,
+            line: valueLine(fileLines, "identifier", tableLine),
+            message: `${tableWhere}: identifier ${JSON.stringify(table.identifier)} is a Jinja template or not a name, which chant does not render; it was not read`,
+          });
+          continue;
+        }
+        const q = readQuoting(sourceQuoting, table.quoting, file, tableLine, `${tableWhere} quoting`, gaps);
+        const tableOutput = `${name(schema, q.schema)}.${name(identifier, q.identifier)}`;
+        links.push({ member: producer, output: tableOutput, file, line: tableLine });
+        if (table.columns === undefined || table.columns === null) continue;
+        if (!Array.isArray(table.columns)) {
+          gaps.push({ file, line: tableLine, message: `${tableWhere}: columns is not a list; its columns were not read` });
+          continue;
+        }
+        let columnCursor = tableLine;
+        for (const column of table.columns) {
+          if (!isObject(column)) continue;
+          const columnName = scalar(column.name);
+          const columnLine = nameLine(fileLines, columnName, columnCursor);
+          columnCursor = columnLine;
+          if (columnName === undefined || isTemplated(columnName)) {
+            gaps.push({ file, line: columnLine, message: `${tableWhere}: column ${JSON.stringify(column.name)} is a Jinja template or not a name, which chant does not render; it was not read` });
+            continue;
+          }
+          const quoted = column.quote === true;
+          links.push({ member: producer, output: `${tableOutput}.${name(columnName, quoted)}`, anchor: tableOutput, file, line: columnLine });
+        }
+      }
+    }
+  }
+  return result;
+}
+
+/** A name as text: a string, or a number or boolean YAML read as one. */
+function scalar(v: unknown): string | undefined {
+  if (typeof v === "string") return v;
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  return undefined;
+}
+
+/** Whether an entry is enabled: `enabled`, or `config.enabled`. A templated value is a gap, and the entry is left out. */
+function readEnabled(entry: Record<string, unknown>, file: string, line: number, where: string, gaps: KindReadGap[]): boolean {
+  const config = isObject(entry.config) ? entry.config : {};
+  const raw = entry.enabled ?? config.enabled;
+  if (raw === undefined || raw === null || raw === true) return true;
+  if (raw === false) return false;
+  gaps.push({ file, line, message: `${where}: enabled is ${JSON.stringify(raw)}, which chant does not render; it was not read` });
+  return false;
+}
+
+/** `base` with a `quoting:` block's schema and identifier applied. A value that isn't a boolean is a gap and keeps the base. */
+function readQuoting(base: Quoting, raw: unknown, file: string, line: number, where: string, gaps: KindReadGap[]): Quoting {
+  if (raw === undefined || raw === null) return base;
+  if (!isObject(raw)) {
+    gaps.push({ file, line, message: `${where} is not a mapping; the quoting above it applies` });
+    return base;
+  }
+  const out = { ...base };
+  for (const key of ["schema", "identifier"] as const) {
+    const v = raw[key];
+    if (v === undefined) continue;
+    if (typeof v === "boolean") out[key] = v;
+    else gaps.push({ file, line, message: `${where}.${key} is ${JSON.stringify(v)}, not true or false; the quoting above it applies` });
+  }
+  return out;
+}
+
+/** Every `.yml` and `.yaml` file under tree-relative directory `dir`, skipping dot directories and node_modules. */
+function yamlFiles(tree: WorkspaceTree, dir: string): string[] {
+  if (dir !== "" && tree.stat(dir) !== "dir") return [];
+  const out: string[] = [];
+  const walk = (at: string) => {
+    for (const e of tree.list(at) ?? []) {
+      const path = joinPath(at, e.name);
+      if (e.type === "dir") {
+        if (!skippedDir(e.name)) walk(path);
+      } else if (/\.ya?ml$/.test(e.name)) {
+        out.push(path);
+      }
+    }
+  };
+  walk(dir);
+  return out;
+}
+
+function lines(text: string): string[] {
+  return text.replace(/\r\n/g, "\n").split("\n");
+}
+
+function escapeRegExp(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/** The line (from 1) of top-level key `key`, or 1. */
+function keyLine(fileLines: string[], key: string): number {
+  const re = new RegExp(`^${escapeRegExp(key)}\\s*:`);
+  const i = fileLines.findIndex((l) => re.test(l));
+  return i < 0 ? 1 : i + 1;
+}
+
+/**
+ * The line of the first `name: <value>` entry after line `from`, or `from`
+ * itself. Sources, tables and columns are listed in file order, so
+ * searching on from the parent's line finds each one's own line. A file in
+ * flow style has no such lines, and its entries are reported at the parent.
+ */
+function nameLine(fileLines: string[], value: string | undefined, from: number): number {
+  return value === undefined ? from : valueLineOf(fileLines, "name", value, from);
+}
+
+/** The line of the first `<key>:` after line `from`, or `from`. */
+function valueLine(fileLines: string[], key: string, from: number): number {
+  const re = new RegExp(`^\\s*(?:-\\s+)?${escapeRegExp(key)}\\s*:`);
+  for (let i = Math.max(from, 0); i < fileLines.length; i++) if (re.test(fileLines[i])) return i + 1;
+  return from;
+}
+
+function valueLineOf(fileLines: string[], key: string, value: string, from: number): number {
+  const re = new RegExp(`^\\s*(?:-\\s+)?${escapeRegExp(key)}\\s*:\\s*(["']?)${escapeRegExp(value)}\\1\\s*(?:#.*)?$`);
+  for (let i = Math.max(from, 0); i < fileLines.length; i++) if (re.test(fileLines[i])) return i + 1;
+  return from;
+}

package/src/workspace/graph-cli.ts

@@ -36,6 +36,7 @@ import { openGraphCache, splitCached, storeReads, withoutUnits } from "./graph-c
 import { readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError, type ErrorLocation } from "./declaration";
 import { describePlan, emitDocument, executePlan, memberStatus, planJson, planMembers, type MemberPlan, type Toolchain, type UnitResult } from "./member-commands";
 import { loadKindRegistry } from "./kinds";
+import { readKindLinks } from "./kind-link-readers";
 import { prepareKindReaders, type PreparedReaders } from "./kind-readers";
 import { recordLinkRows } from "./record-assets";
 import { RecordReadError } from "./records";
@@ -258,7 +259,8 @@ export async function workspaceGraph(query: GraphQuery): Promise<GraphResult> {
       member.stamp = split?.stamps.get(member.name) ?? null;
     }
     // Links (#2539) resolve against the declaration that was read, the revision's for --at, and the kinds installed now.
-    const graph = composeWorkspaceGraph({ name: declaration.name, root: located.root }, inputs, { declaration, kinds });
+    // A kind's link reader (#3932) reads the member's files in the tree read, as check does.
+    const graph = composeWorkspaceGraph({ name: declaration.name, root: located.root }, inputs, { declaration, kinds, read: readKindLinks(declaration, located.tree, kinds) });
     // The box's factory and listing (#3146), from the declaration read, so --at gives them at that revision.
     for (const member of graph.members) member.box = graphBox(declaration.members.find((m) => m.name === member.name)?.box ?? null, located.tree, declaration.members);
     // Nested workspaces (#2551, ws-071): read-only, through their own chant workspace graph.

package/src/workspace/graph-contract.test.ts

@@ -15,7 +15,7 @@ import { existsSync, readdirSync, readFileSync, rmSync, utimesSync, writeFileSyn
 import { join } from "node:path";
 import { afterAll, describe, expect, test } from "vitest";
 import { cleanScratch, commitAll, contract, declaration, declaration as declaration_, FAKE_FILE_GRAPH_CHANT, FAKE_GRAPH_CHANT, git, REPO, repo, scratchDir, validSchema } from "./__fixtures__/contract-repo";
-import type { GraphIR } from "../graph-ir";
+import type { GraphIR, IREdgeKind } from "../graph-ir";
 import { composeWorkspaceGraph, MEMBER_RUN_REASON_CODES } from "./compose-graph";
 import { parseDeclaration, WORKSPACE_ERROR_CODES } from "./declaration";
 import type { LinkTableRow } from "./links";
@@ -106,6 +106,31 @@ describe("the links section (#2539)", () => {
   });
 });
 
+describe("CI edges, pipelines and dangling references (#3922)", () => {
+  test("the edge kinds match IREdgeKind, and a composed CI member validates", () => {
+    const kinds: IREdgeKind[] = ["ref", "needs", "calls", "triggers", "includes", "extends"];
+    expect(schema.$defs.edge.properties.kind.enum).toEqual(kinds);
+    const ir: GraphIR = {
+      version: 1,
+      nodes: [
+        { id: "ci", kind: "Workflow", lexicon: "github", attrs: {} },
+        { id: "build", kind: "Job", lexicon: "github", attrs: {} },
+        { id: "deploy", kind: "Job", lexicon: "github", attrs: {} },
+      ],
+      edges: [
+        { from: "deploy", to: "build", kind: "needs", viaAttr: "needs" },
+        { from: "build", to: "ci", kind: "calls", viaAttr: "uses", crossPipeline: true },
+      ],
+      groups: { byPipeline: { ci: ["build", "ci", "deploy"] } },
+      dangling: [{ from: "deploy", kind: "needs", path: "needs", value: "lint" }, { from: "build", kind: "calls", path: "uses", value: "o/r/.github/workflows/x.yml@v1", crossPipeline: true }],
+    };
+    const m = { name: "svc", dir: "svc", kind: "chant", status: "composed" as const, reason: null, chant: "0.124.20", irVersion: 1, live: false };
+    const graph = composeWorkspaceGraph({ name: "acme", root: "." }, [{ member: m, ir }]);
+    expect(graph.dangling).toHaveLength(2);
+    expectValid({ $schema: GRAPH_OUTPUT_SCHEMA_ID, contract: 1, chant: "0.124.20", at: null, ...graph });
+  });
+});
+
 describe("the collectors section (#2559)", () => {
   const topology = {
     pipelines: [{ id: "traces", signal: "traces", receivers: ["otlp"], processors: [], exporters: ["otlphttp"] }],