@intentius/chant 0.124.22 → 0.124.24

package/dist/workspace/dbt-project-reader.js

@@ -0,0 +1,290 @@
+/**
+ * The `dbt-project` link reader (#3932): a dbt project's sources, as links to
+ * the tables and columns a sql member exposes (#3931).
+ *
+ * The reader parses YAML and nothing else. It never runs dbt, never renders
+ * Jinja and never reads `profiles.yml` or a target, so the links it derives
+ * are the same in every environment.
+ *
+ * - The project is `dbt_project.yml`. Sources are read from every `.yml` and
+ *   `.yaml` file under its `model-paths` (`source-paths` in older projects;
+ *   `models` when it names neither), as dbt reads them.
+ * - Each enabled source table links to `<schema>.<table>`, and each of its
+ *   `columns` to `<schema>.<table>.<column>`. The schema is the source's
+ *   `schema`, or the source's `name` when it sets none, which is dbt's own
+ *   fallback. The table is its `identifier`, or its `name`. `database` is not
+ *   part of the name: a Postgres output has none, and dbt-clickhouse reads a
+ *   source's `schema` as the ClickHouse database.
+ * - Case follows the member's `dialect` field. For `postgres`, an unquoted
+ *   name folds to lowercase and a quoted one keeps its case. dbt quotes a
+ *   source's schema and identifier on Postgres unless `quoting:` (in
+ *   `dbt_project.yml`, on the source, or on the table) turns it off, and a
+ *   column is quoted only with `quote: true`. For `clickhouse`, names are
+ *   kept as written.
+ * - A value that is a Jinja template (`{{ ... }}` or `{% ... %}`) is not
+ *   resolved: the reader records a gap and leaves that source, table or
+ *   column out, rather than guess what a target would render.
+ * - The producer is the member the `sqlMember` field names. Unset, each link
+ *   leaves the producer to `resolveLinks`, which picks the one member that
+ *   exposes the table.
+ */
+import { posix } from "node:path";
+import { parseYAMLDocument, YAMLParseError } from "../yaml.js";
+import { joinPath, skippedDir } from "./tree.js";
+export const DBT_PROJECT_FILE = "dbt_project.yml";
+const DEFAULT_MODEL_PATHS = ["models"];
+/** The dialects whose naming the reader follows; the member's `dialect` field picks one. */
+export const DBT_DIALECTS = ["postgres", "clickhouse"];
+const isTemplated = (v) => /\{\{|\{%/.test(v);
+const isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
+export function readDbtProject(input) {
+    const { dir, tree, fields } = input;
+    const links = [];
+    const gaps = [];
+    const result = { reader: "dbt-project", links, gaps, producerField: "sqlMember" };
+    const projectFile = joinPath(dir, DBT_PROJECT_FILE);
+    const producer = typeof fields?.sqlMember === "string" && fields.sqlMember !== "" ? fields.sqlMember : null;
+    const dialect = fields?.dialect === "clickhouse" ? "clickhouse" : "postgres";
+    let project;
+    try {
+        project = parseYAMLDocument(tree.read(projectFile));
+    }
+    catch (err) {
+        gaps.push({ file: projectFile, line: err instanceof YAMLParseError ? err.line : 1, message: `${DBT_PROJECT_FILE} could not be read (${err.message}), so no source was read` });
+        return result;
+    }
+    if (!isObject(project)) {
+        gaps.push({ file: projectFile, line: 1, message: `${DBT_PROJECT_FILE} is not a mapping, so no source was read` });
+        return result;
+    }
+    const projectLines = lines(tree.read(projectFile));
+    // Where sources live: model-paths, or the older source-paths.
+    const key = project["model-paths"] !== undefined ? "model-paths" : project["source-paths"] !== undefined ? "source-paths" : null;
+    const rawPaths = key ? project[key] : DEFAULT_MODEL_PATHS;
+    const paths = [];
+    if (!Array.isArray(rawPaths)) {
+        gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} is not a list, so no source was read` });
+        return result;
+    }
+    for (const p of rawPaths) {
+        if (typeof p !== "string" || isTemplated(p)) {
+            gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} entry ${JSON.stringify(p)} is not a plain path; its sources were not read` });
+            continue;
+        }
+        const at = posix.normalize(joinPath(dir, p));
+        if (at.startsWith("../") || at === ".." || posix.isAbsolute(p)) {
+            gaps.push({ file: projectFile, line: keyLine(projectLines, key ?? ""), message: `${key} entry ${p} lies outside the workspace; its sources were not read` });
+            continue;
+        }
+        paths.push(at === "." ? "" : at);
+    }
+    const quoting = readQuoting({ schema: true, identifier: true }, project.quoting, projectFile, keyLine(projectLines, "quoting"), "dbt_project.yml quoting", gaps);
+    const files = new Set();
+    for (const p of paths)
+        for (const f of yamlFiles(tree, p))
+            files.add(f);
+    const name = (v, quoted) => (dialect === "postgres" && !quoted ? v.toLowerCase() : v);
+    for (const file of [...files].sort()) {
+        let text;
+        let doc;
+        try {
+            text = tree.read(file);
+            doc = parseYAMLDocument(text);
+        }
+        catch (err) {
+            gaps.push({ file, line: err instanceof YAMLParseError ? err.line : 1, message: `the file could not be read as YAML (${err.message}); any sources in it were not read` });
+            continue;
+        }
+        if (!isObject(doc) || doc.sources === undefined || doc.sources === null)
+            continue;
+        const fileLines = lines(text);
+        const sourcesLine = keyLine(fileLines, "sources");
+        if (!Array.isArray(doc.sources)) {
+            gaps.push({ file, line: sourcesLine, message: "sources is not a list; its sources were not read" });
+            continue;
+        }
+        let cursor = sourcesLine;
+        for (const source of doc.sources) {
+            if (!isObject(source))
+                continue;
+            const sourceName = scalar(source.name);
+            const sourceLine = nameLine(fileLines, sourceName, cursor);
+            cursor = sourceLine;
+            if (sourceName === undefined) {
+                gaps.push({ file, line: sourceLine, message: "a source has no name; its tables were not read" });
+                continue;
+            }
+            const where = `source ${sourceName}`;
+            if (isTemplated(sourceName)) {
+                gaps.push({ file, line: sourceLine, message: `${where}: its name is a Jinja template, which chant does not render; its tables were not read` });
+                continue;
+            }
+            const enabled = readEnabled(source, file, sourceLine, where, gaps);
+            if (enabled !== true)
+                continue;
+            const schema = source.schema === undefined || source.schema === null ? sourceName : scalar(source.schema);
+            if (schema === undefined || isTemplated(schema)) {
+                gaps.push({
+                    file,
+                    line: valueLine(fileLines, "schema", sourceLine),
+                    message: `${where}: schema ${JSON.stringify(source.schema)} is a Jinja template or not a name; chant reads names as declared and never renders them for a target, so its tables were not read`,
+                });
+                continue;
+            }
+            const sourceQuoting = readQuoting(quoting, source.quoting, file, sourceLine, `${where} quoting`, gaps);
+            if (source.tables === undefined || source.tables === null)
+                continue;
+            if (!Array.isArray(source.tables)) {
+                gaps.push({ file, line: sourceLine, message: `${where}: tables is not a list; its tables were not read` });
+                continue;
+            }
+            let tableCursor = sourceLine;
+            for (const table of source.tables) {
+                if (!isObject(table))
+                    continue;
+                const tableName = scalar(table.name);
+                const tableLine = nameLine(fileLines, tableName, tableCursor);
+                tableCursor = tableLine;
+                if (tableName === undefined) {
+                    gaps.push({ file, line: tableLine, message: `${where}: a table has no name; it was not read` });
+                    continue;
+                }
+                const tableWhere = `${where} table ${tableName}`;
+                if (isTemplated(tableName)) {
+                    gaps.push({ file, line: tableLine, message: `${tableWhere}: its name is a Jinja template, which chant does not render; it was not read` });
+                    continue;
+                }
+                if (readEnabled(table, file, tableLine, tableWhere, gaps) !== true)
+                    continue;
+                const identifier = table.identifier === undefined || table.identifier === null ? tableName : scalar(table.identifier);
+                if (identifier === undefined || isTemplated(identifier)) {
+                    gaps.push({
+                        file,
+                        line: valueLine(fileLines, "identifier", tableLine),
+                        message: `${tableWhere}: identifier ${JSON.stringify(table.identifier)} is a Jinja template or not a name, which chant does not render; it was not read`,
+                    });
+                    continue;
+                }
+                const q = readQuoting(sourceQuoting, table.quoting, file, tableLine, `${tableWhere} quoting`, gaps);
+                const tableOutput = `${name(schema, q.schema)}.${name(identifier, q.identifier)}`;
+                links.push({ member: producer, output: tableOutput, file, line: tableLine });
+                if (table.columns === undefined || table.columns === null)
+                    continue;
+                if (!Array.isArray(table.columns)) {
+                    gaps.push({ file, line: tableLine, message: `${tableWhere}: columns is not a list; its columns were not read` });
+                    continue;
+                }
+                let columnCursor = tableLine;
+                for (const column of table.columns) {
+                    if (!isObject(column))
+                        continue;
+                    const columnName = scalar(column.name);
+                    const columnLine = nameLine(fileLines, columnName, columnCursor);
+                    columnCursor = columnLine;
+                    if (columnName === undefined || isTemplated(columnName)) {
+                        gaps.push({ file, line: columnLine, message: `${tableWhere}: column ${JSON.stringify(column.name)} is a Jinja template or not a name, which chant does not render; it was not read` });
+                        continue;
+                    }
+                    const quoted = column.quote === true;
+                    links.push({ member: producer, output: `${tableOutput}.${name(columnName, quoted)}`, anchor: tableOutput, file, line: columnLine });
+                }
+            }
+        }
+    }
+    return result;
+}
+/** A name as text: a string, or a number or boolean YAML read as one. */
+function scalar(v) {
+    if (typeof v === "string")
+        return v;
+    if (typeof v === "number" || typeof v === "boolean")
+        return String(v);
+    return undefined;
+}
+/** Whether an entry is enabled: `enabled`, or `config.enabled`. A templated value is a gap, and the entry is left out. */
+function readEnabled(entry, file, line, where, gaps) {
+    const config = isObject(entry.config) ? entry.config : {};
+    const raw = entry.enabled ?? config.enabled;
+    if (raw === undefined || raw === null || raw === true)
+        return true;
+    if (raw === false)
+        return false;
+    gaps.push({ file, line, message: `${where}: enabled is ${JSON.stringify(raw)}, which chant does not render; it was not read` });
+    return false;
+}
+/** `base` with a `quoting:` block's schema and identifier applied. A value that isn't a boolean is a gap and keeps the base. */
+function readQuoting(base, raw, file, line, where, gaps) {
+    if (raw === undefined || raw === null)
+        return base;
+    if (!isObject(raw)) {
+        gaps.push({ file, line, message: `${where} is not a mapping; the quoting above it applies` });
+        return base;
+    }
+    const out = { ...base };
+    for (const key of ["schema", "identifier"]) {
+        const v = raw[key];
+        if (v === undefined)
+            continue;
+        if (typeof v === "boolean")
+            out[key] = v;
+        else
+            gaps.push({ file, line, message: `${where}.${key} is ${JSON.stringify(v)}, not true or false; the quoting above it applies` });
+    }
+    return out;
+}
+/** Every `.yml` and `.yaml` file under tree-relative directory `dir`, skipping dot directories and node_modules. */
+function yamlFiles(tree, dir) {
+    if (dir !== "" && tree.stat(dir) !== "dir")
+        return [];
+    const out = [];
+    const walk = (at) => {
+        for (const e of tree.list(at) ?? []) {
+            const path = joinPath(at, e.name);
+            if (e.type === "dir") {
+                if (!skippedDir(e.name))
+                    walk(path);
+            }
+            else if (/\.ya?ml$/.test(e.name)) {
+                out.push(path);
+            }
+        }
+    };
+    walk(dir);
+    return out;
+}
+function lines(text) {
+    return text.replace(/\r\n/g, "\n").split("\n");
+}
+function escapeRegExp(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/** The line (from 1) of top-level key `key`, or 1. */
+function keyLine(fileLines, key) {
+    const re = new RegExp(`^${escapeRegExp(key)}\\s*:`);
+    const i = fileLines.findIndex((l) => re.test(l));
+    return i < 0 ? 1 : i + 1;
+}
+/**
+ * The line of the first `name: <value>` entry after line `from`, or `from`
+ * itself. Sources, tables and columns are listed in file order, so
+ * searching on from the parent's line finds each one's own line. A file in
+ * flow style has no such lines, and its entries are reported at the parent.
+ */
+function nameLine(fileLines, value, from) {
+    return value === undefined ? from : valueLineOf(fileLines, "name", value, from);
+}
+/** The line of the first `<key>:` after line `from`, or `from`. */
+function valueLine(fileLines, key, from) {
+    const re = new RegExp(`^\\s*(?:-\\s+)?${escapeRegExp(key)}\\s*:`);
+    for (let i = Math.max(from, 0); i < fileLines.length; i++)
+        if (re.test(fileLines[i]))
+            return i + 1;
+    return from;
+}
+function valueLineOf(fileLines, key, value, from) {
+    const re = new RegExp(`^\\s*(?:-\\s+)?${escapeRegExp(key)}\\s*:\\s*(["']?)${escapeRegExp(value)}\\1\\s*(?:#.*)?$`);
+    for (let i = Math.max(from, 0); i < fileLines.length; i++)
+        if (re.test(fileLines[i]))
+            return i + 1;
+    return from;
+}

package/dist/workspace/graph-cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graph-cli.d.ts","sourceRoot":"","sources":["../../src/workspace/graph-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAA0B,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAA+E,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEnI,OAAO,EAA6E,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAgF,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAMnK,OAAO,EAAoC,KAAK,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAExF,6DAA6D;AAC7D,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC,2EAA2E;AAC3E,eAAO,MAAM,sBAAsB,4EAA4E,CAAC;AAEhH;;;GAGG;AACH,eAAO,MAAM,iBAAiB,2OAA0D,CAAC;AACzF,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAQhE,UAAU,IAAI;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,aAAa,GACrB,CAAC,IAAI,GAAG;IAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,cAAc,CAAC,GACrE,CAAC,IAAI,GAAG;IAAE,KAAK,EAAE;QAAE,IAAI,EAAE,cAAc,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAA;KAAE,CAAA;CAAE,CAAC,CAAC;AAElG,MAAM,WAAW,UAAU;IACzB,mDAAmD;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,8GAA8G;IAC9G,IAAI,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1G,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,aAAa,CAAC;IACnB,gFAAgF;IAChF,MAAM,EAAE,OAAO,CAAC;IAChB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,sEAAsE;AACtE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,EAAE,CAEtE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAmCtF;AAgDD,wGAAwG;AACxG,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAKvD;AAED,qGAAqG;AACrG,wBAAsB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAuE5E;AA8CD,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAsE5E"}
+{"version":3,"file":"graph-cli.d.ts","sourceRoot":"","sources":["../../src/workspace/graph-cli.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAA0B,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AACvE,OAAO,EAA+E,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEnI,OAAO,EAA6E,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAgF,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAOnK,OAAO,EAAoC,KAAK,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAExF,6DAA6D;AAC7D,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC,2EAA2E;AAC3E,eAAO,MAAM,sBAAsB,4EAA4E,CAAC;AAEhH;;;GAGG;AACH,eAAO,MAAM,iBAAiB,2OAA0D,CAAC;AACzF,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAQhE,UAAU,IAAI;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,aAAa,GACrB,CAAC,IAAI,GAAG;IAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,cAAc,CAAC,GACrE,CAAC,IAAI,GAAG;IAAE,KAAK,EAAE;QAAE,IAAI,EAAE,cAAc,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAA;KAAE,CAAA;CAAE,CAAC,CAAC;AAElG,MAAM,WAAW,UAAU;IACzB,mDAAmD;IACnD,GAAG,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,8GAA8G;IAC9G,IAAI,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3B,kFAAkF;IAClF,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1G,kGAAkG;IAClG,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,aAAa,CAAC;IACnB,gFAAgF;IAChF,MAAM,EAAE,OAAO,CAAC;IAChB,4FAA4F;IAC5F,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,sEAAsE;AACtE,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,MAAM,EAAE,CAEtE;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAmCtF;AAgDD,wGAAwG;AACxG,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAKvD;AAED,qGAAqG;AACrG,wBAAsB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAwE5E;AA8CD,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAsE5E"}

package/dist/workspace/graph-cli.js

@@ -34,6 +34,7 @@ import { openGraphCache, splitCached, storeReads, withoutUnits } from "./graph-c
 import { readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError } from "./declaration.js";
 import { describePlan, emitDocument, executePlan, memberStatus, planJson, planMembers } from "./member-commands.js";
 import { loadKindRegistry } from "./kinds.js";
+import { readKindLinks } from "./kind-link-readers.js";
 import { prepareKindReaders } from "./kind-readers.js";
 import { recordLinkRows } from "./record-assets.js";
 import { RecordReadError } from "./records.js";
@@ -198,7 +199,8 @@ export async function workspaceGraph(query) {
             member.stamp = split?.stamps.get(member.name) ?? null;
         }
         // Links (#2539) resolve against the declaration that was read, the revision's for --at, and the kinds installed now.
-        const graph = composeWorkspaceGraph({ name: declaration.name, root: located.root }, inputs, { declaration, kinds });
+        // A kind's link reader (#3932) reads the member's files in the tree read, as check does.
+        const graph = composeWorkspaceGraph({ name: declaration.name, root: located.root }, inputs, { declaration, kinds, read: readKindLinks(declaration, located.tree, kinds) });
         // The box's factory and listing (#3146), from the declaration read, so --at gives them at that revision.
         for (const member of graph.members)
             member.box = graphBox(declaration.members.find((m) => m.name === member.name)?.box ?? null, located.tree, declaration.members);

package/dist/workspace/graph.schema.json

@@ -93,6 +93,13 @@
             "$ref": "#/$defs/edge"
           }
         },
+        "dangling": {
+          "description": "Added in contract 1 by #3922. The members' references whose target is no node of their graph (#3910): a job `needs` a job no workflow declares, a `uses:` names another repository's workflow, a GitLab `include:` names a file chant did not generate. Each member's `ir.dangling` entry with `from` prefixed by its member and a `member` field, sorted by `from`, `path` and `value`. Absent when no member has one.",
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/dangling"
+          }
+        },
         "groups": {
           "type": "object",
           "required": [
@@ -158,6 +165,16 @@
                 }
               },
               "description": "Keys prefixed with the member."
+            },
+            "byPipeline": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "description": "Added in contract 1 by #3922. CI pipeline membership (#3910): a workflow or pipeline the member's CI lexicon reports, with the node ids it runs. Keys prefixed with the member, like byStack. Edges with `crossPipeline` run between these groups."
             }
           }
         },
@@ -181,6 +198,13 @@
               },
               "member": {
                 "type": "string"
+              },
+              "previously": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "description": "Names this output had before, from a rename hint a lexicon read in the member's source (a sql `-- previously:` comment). A link to one of them is told the new name."
               }
             }
           }
@@ -516,7 +540,15 @@
           "type": "string"
         },
         "kind": {
-          "type": "string"
+          "enum": [
+            "ref",
+            "needs",
+            "calls",
+            "triggers",
+            "includes",
+            "extends"
+          ],
+          "description": "`ref` for an edge from a typed reference. The CI kinds (#3910): `needs` (a job waits on another), `calls` (a reusable workflow or action is used), `triggers` (a pipeline starts another), `includes` (a file is pulled in), `extends` (a GitLab job inherits another). The list grows within contract 1; a reader ignores a kind it does not know."
         },
         "viaAttr": {
           "type": "string"
@@ -524,8 +556,50 @@
         "toAttr": {
           "type": "string"
         },
+        "crossPipeline": {
+          "type": "boolean",
+          "description": "Added in contract 1 by #3922. True when the edge leaves the pipeline `from` belongs to (#3910): a reusable-workflow call, a `workflow_run` trigger, a GitLab child or multi-project trigger, an include. Absent on every in-pipeline edge."
+        },
+        "member": {
+          "type": "string"
+        }
+      }
+    },
+    "dangling": {
+      "type": "object",
+      "required": [
+        "from",
+        "kind",
+        "path",
+        "value",
+        "member"
+      ],
+      "properties": {
+        "from": {
+          "type": "string",
+          "description": "The node holding the reference, prefixed with its member."
+        },
+        "kind": {
+          "$ref": "#/$defs/edge/properties/kind"
+        },
+        "path": {
+          "type": "string",
+          "description": "Where on `from` it was written, such as `needs`, `uses` or `trigger.include`."
+        },
+        "value": {
+          "type": "string",
+          "description": "The reference exactly as written."
+        },
+        "crossPipeline": {
+          "type": "boolean",
+          "description": "As on an edge."
+        },
         "member": {
           "type": "string"
+        },
+        "nested": {
+          "type": "string",
+          "description": "For a member of kind `workspace`: the nested workspace's member the reference was read through, as on a node."
         }
       }
     },
@@ -636,8 +710,23 @@
             "origin": {
               "enum": [
                 "declared",
+                "kind-reader",
                 "inferred:joinKey"
-              ]
+              ],
+              "description": "How the row was found: declared in chant.workspace.json, read from the member's own files by its kind's link reader (#3932), or inferred by joinKey()."
+            },
+            "reader": {
+              "type": "string",
+              "description": "A kind-reader row (#3932): the link reader that found it, such as dbt-project."
+            },
+            "file": {
+              "type": "string",
+              "description": "A kind-reader row (#3932): the file that states the link, from the workspace root."
+            },
+            "line": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "A kind-reader row (#3932): the line of file that states the link."
             },
             "resolves": {
               "const": "source"
@@ -728,8 +817,23 @@
             "origin": {
               "enum": [
                 "declared",
+                "kind-reader",
                 "inferred:joinKey"
-              ]
+              ],
+              "description": "How the row was found: declared in chant.workspace.json, read from the member's own files by its kind's link reader (#3932), or inferred by joinKey()."
+            },
+            "reader": {
+              "type": "string",
+              "description": "A kind-reader row (#3932): the link reader that found it, such as dbt-project."
+            },
+            "file": {
+              "type": "string",
+              "description": "A kind-reader row (#3932): the file that states the link, from the workspace root."
+            },
+            "line": {
+              "type": "integer",
+              "minimum": 1,
+              "description": "A kind-reader row (#3932): the line of file that states the link."
             },
             "resolves": {
               "const": "source"
@@ -746,7 +850,8 @@
               "description": "The consumer's parameter node, when known."
             },
             "input": {
-              "type": "string"
+              "type": "string",
+              "description": "The consumer's parameter the join would feed; for a kind-reader row (#3932), the output its file reads, which no single member was found to produce."
             },
             "status": {
               "const": "ambiguous"

package/dist/workspace/intent.js

@@ -75,7 +75,7 @@ import { followSquashes, squashLineOrigins } from "./squash.js";
 import { memberDirOnDisk, memberSymbolResolvers } from "./lexicon-symbols.js";
 import { resolveSymbol, symbolResolverFor } from "./symbols.js";
 import { readTrailerJoins } from "./trailer-joins.js";
-import { joinPath, skippedDir } from "./tree.js";
+import { joinPath, skippedDir, workingTree } from "./tree.js";
 import { activeAttestors } from "./trust/attestor.js";
 import { commitProvenance, policyAtBase, resolveBase } from "./trust/provenance.js";
 import { locateWorkspace } from "./which-chant.js";
@@ -1295,7 +1295,12 @@ async function walk(query, head) {
         try {
             const groups = resolveGroups(declaration, located.tree);
             const kindsRegistry = loadKindRegistry(declaration.pins, located.rootOnDisk).registry;
-            rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry)).filter((r) => r.origin === "declared");
+            const { gatherLexiconOutputs } = await import("./lexicon-outputs.js");
+            const lexiconOutputs = await gatherLexiconOutputs(located.rootOnDisk, declaration, located.at ? workingTree(located.rootOnDisk) : located.tree, groups, kindsRegistry, {
+                atRevision: located.at !== null,
+                ...(query.loadLexicon ? { load: query.loadLexicon } : {}),
+            });
+            rows = resolveLinks(declaration, sourceMemberHandles(declaration, located.tree, groups, kindsRegistry, lexiconOutputs)).filter((r) => r.origin === "declared");
         }
         catch (err) {
             if (!(err instanceof WorkspaceReadError))

package/dist/workspace/kind-link-readers.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Links a kind reads from its member's own files (#3932).
+ *
+ * A member's links usually come from its entry in `chant.workspace.json`, or
+ * are inferred from a chant member's parameters by `joinKey()`. Some members
+ * already say what they read in files of their own: a dbt project names the
+ * tables it reads in its `sources:` YAML. A kind can name a link reader in its
+ * kinds file (`reader`), and chant runs that reader over the member's
+ * directory to derive the member's links.
+ *
+ * The readers are chant's code, listed in {@link KIND_LINK_READERS}: a kinds
+ * file stays data, and chant never imports the package that supplies it. A
+ * reader runs offline and parses files as data. It never invokes the tool
+ * whose files it reads, so `chant workspace check` keeps its promise of no
+ * network and no member code.
+ *
+ * A reader returns links with the file and line that state them, and gaps:
+ * the places it could not read without guessing, such as a templated value.
+ * `resolveLinks` (`./links.ts`) resolves the links beside the declared ones,
+ * with origin `kind-reader`; WSP093 and WSP094 name the reader's file and line,
+ * and WSP099 reports the gaps.
+ */
+import type { Declaration, Member } from "./declaration.js";
+import { type FieldValue, type KindLinkReaderName, type KindRegistry } from "./kinds.js";
+import type { WorkspaceTree } from "./tree.js";
+/** One link a reader found in the member's files. */
+export interface KindReadLink {
+    /**
+     * The producer the link names, or null when the member's files don't say
+     * which member produces it. A null producer is found among the members
+     * whose outputs include {@link anchor}; see `resolveLinks`.
+     */
+    member: string | null;
+    /** The producer's output the link reads. */
+    output: string;
+    /**
+     * The output that picks the producer when `member` is null; unset, the
+     * output itself. A dbt source column is anchored on its table, so a dropped
+     * column is missing from the member that still has the table.
+     */
+    anchor?: string;
+    /** The file that states the link, relative to the workspace root. */
+    file: string;
+    /** Its line, from 1. */
+    line: number;
+}
+/** A place the reader could not read without guessing. The links it would have stated are left out. */
+export interface KindReadGap {
+    /** Relative to the workspace root. */
+    file: string;
+    line: number;
+    message: string;
+}
+export interface KindLinkRead {
+    /** The reader that ran. */
+    reader: KindLinkReaderName;
+    links: KindReadLink[];
+    gaps: KindReadGap[];
+    /**
+     * The member field that names the producer, for messages that ask the
+     * author to set it when the files don't say (`sqlMember` for dbt).
+     */
+    producerField?: string;
+}
+/** What a reader gets: the member, its directory and resolved fields, and the tree read. */
+export interface KindReaderInput {
+    member: Member;
+    /** The member's directory, tree-relative; "" for the workspace root. */
+    dir: string;
+    /** The member's fields, defaults filled in (#3151); null when its kind declares none. */
+    fields: Record<string, FieldValue | null | Record<string, FieldValue | null>> | null;
+    tree: WorkspaceTree;
+}
+export type KindLinkReader = (input: KindReaderInput) => KindLinkRead;
+/** Every reader chant ships, by the name a kinds file uses. */
+export declare const LINK_READERS: Record<KindLinkReaderName, KindLinkReader>;
+/**
+ * Run the link reader of every member whose kind has one. A member whose
+ * directory is missing is left out. A reader that throws is reported as a
+ * gap on the member's directory, so one unreadable project never stops
+ * the check.
+ */
+export declare function readKindLinks(declaration: Declaration, tree: WorkspaceTree, kinds: KindRegistry): Map<string, KindLinkRead>;
+//# sourceMappingURL=kind-link-readers.d.ts.map

package/dist/workspace/kind-link-readers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kind-link-readers.d.ts","sourceRoot":"","sources":["../../src/workspace/kind-link-readers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAuB,KAAK,UAAU,EAAE,KAAK,kBAAkB,EAAE,KAAK,YAAY,EAAE,MAAM,SAAS,CAAC;AAC3G,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAG5C,qDAAqD;AACrD,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,wBAAwB;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,uGAAuG;AACvG,MAAM,WAAW,WAAW;IAC1B,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,2BAA2B;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,4FAA4F;AAC5F,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,GAAG,EAAE,MAAM,CAAC;IACZ,yFAAyF;IACzF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;IACrF,IAAI,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,eAAe,KAAK,YAAY,CAAC;AAEtE,+DAA+D;AAC/D,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,kBAAkB,EAAE,cAAc,CAEnE,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAe3H"}

package/dist/workspace/kind-link-readers.js

@@ -0,0 +1,53 @@
+/**
+ * Links a kind reads from its member's own files (#3932).
+ *
+ * A member's links usually come from its entry in `chant.workspace.json`, or
+ * are inferred from a chant member's parameters by `joinKey()`. Some members
+ * already say what they read in files of their own: a dbt project names the
+ * tables it reads in its `sources:` YAML. A kind can name a link reader in its
+ * kinds file (`reader`), and chant runs that reader over the member's
+ * directory to derive the member's links.
+ *
+ * The readers are chant's code, listed in {@link KIND_LINK_READERS}: a kinds
+ * file stays data, and chant never imports the package that supplies it. A
+ * reader runs offline and parses files as data. It never invokes the tool
+ * whose files it reads, so `chant workspace check` keeps its promise of no
+ * network and no member code.
+ *
+ * A reader returns links with the file and line that state them, and gaps:
+ * the places it could not read without guessing, such as a templated value.
+ * `resolveLinks` (`./links.ts`) resolves the links beside the declared ones,
+ * with origin `kind-reader`; WSP093 and WSP094 name the reader's file and line,
+ * and WSP099 reports the gaps.
+ */
+import { resolveMemberFields } from "./kinds.js";
+import { readDbtProject } from "./dbt-project-reader.js";
+/** Every reader chant ships, by the name a kinds file uses. */
+export const LINK_READERS = {
+    "dbt-project": readDbtProject,
+};
+/**
+ * Run the link reader of every member whose kind has one. A member whose
+ * directory is missing is left out. A reader that throws is reported as a
+ * gap on the member's directory, so one unreadable project never stops
+ * the check.
+ */
+export function readKindLinks(declaration, tree, kinds) {
+    const out = new Map();
+    for (const m of declaration.members) {
+        const kind = kinds.get(m.kind);
+        if (!kind?.reader)
+            continue;
+        const dir = m.dir === "." ? "" : m.dir;
+        if (tree.stat(dir) !== "dir")
+            continue;
+        const { fields } = resolveMemberFields(kind, m.fields);
+        try {
+            out.set(m.name, LINK_READERS[kind.reader]({ member: m, dir, fields, tree }));
+        }
+        catch (err) {
+            out.set(m.name, { reader: kind.reader, links: [], gaps: [{ file: dir || ".", line: 1, message: `the ${kind.reader} reader failed: ${err.message}` }] });
+        }
+    }
+    return out;
+}