@intentius/chant 0.124.22 → 0.124.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/rules-doc.js +1 -1
- package/dist/ci-jobs.d.ts +41 -1
- package/dist/ci-jobs.d.ts.map +1 -1
- package/dist/ci-jobs.js +48 -1
- package/dist/cli/commands/build.d.ts.map +1 -1
- package/dist/cli/commands/build.js +14 -0
- package/dist/cli/handlers/approve.d.ts.map +1 -1
- package/dist/cli/handlers/approve.js +9 -0
- package/dist/cli/handlers/emulator.d.ts +13 -2
- package/dist/cli/handlers/emulator.d.ts.map +1 -1
- package/dist/cli/handlers/emulator.js +63 -3
- package/dist/cli/handlers/graph.js +24 -1
- package/dist/cli/handlers/promote.d.ts.map +1 -1
- package/dist/cli/handlers/promote.js +7 -1
- package/dist/cli/handlers/run-wave.d.ts.map +1 -1
- package/dist/cli/handlers/run-wave.js +5 -4
- package/dist/cli/handlers/run.d.ts +4 -3
- package/dist/cli/handlers/run.d.ts.map +1 -1
- package/dist/cli/handlers/run.js +73 -9
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +17 -4
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +31 -0
- package/dist/cli/registry.d.ts +12 -0
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/components/cli-support.d.ts +45 -6
- package/dist/components/cli-support.d.ts.map +1 -1
- package/dist/components/cli-support.js +97 -22
- package/dist/components/driver.d.ts.map +1 -1
- package/dist/components/driver.js +25 -10
- package/dist/graph-detail.d.ts.map +1 -1
- package/dist/graph-detail.js +24 -2
- package/dist/graph-ir.d.ts +43 -0
- package/dist/graph-ir.d.ts.map +1 -1
- package/dist/graph-ir.js +27 -0
- package/dist/lexicon.d.ts +25 -1
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/lifecycle/git.d.ts +11 -0
- package/dist/lifecycle/git.d.ts.map +1 -1
- package/dist/lifecycle/git.js +62 -6
- package/dist/lifecycle/no-record.d.ts +59 -0
- package/dist/lifecycle/no-record.d.ts.map +1 -0
- package/dist/lifecycle/no-record.js +88 -0
- package/dist/op/activities/shell.d.ts.map +1 -1
- package/dist/op/activities/shell.js +30 -9
- package/dist/op/emulator-lifecycle.d.ts +23 -0
- package/dist/op/emulator-lifecycle.d.ts.map +1 -1
- package/dist/op/emulator-lifecycle.js +39 -2
- package/dist/op/gate.d.ts.map +1 -1
- package/dist/op/gate.js +3 -0
- package/dist/op/index.d.ts +1 -1
- package/dist/op/index.d.ts.map +1 -1
- package/dist/op/index.js +1 -1
- package/dist/op/local-executor.d.ts +10 -0
- package/dist/op/local-executor.d.ts.map +1 -1
- package/dist/op/local-executor.js +7 -2
- package/dist/op/op-waves-notify.d.ts +2 -0
- package/dist/op/op-waves-notify.d.ts.map +1 -1
- package/dist/op/op-waves-notify.js +9 -3
- package/dist/op/op-waves-run.d.ts +8 -2
- package/dist/op/op-waves-run.d.ts.map +1 -1
- package/dist/op/op-waves-run.js +36 -11
- package/dist/op/op-waves.d.ts +85 -1
- package/dist/op/op-waves.d.ts.map +1 -1
- package/dist/op/op-waves.js +153 -1
- package/dist/op/runtimes/local.d.ts.map +1 -1
- package/dist/op/runtimes/local.js +6 -0
- package/dist/op/step-console.d.ts +65 -0
- package/dist/op/step-console.d.ts.map +1 -0
- package/dist/op/step-console.js +140 -0
- package/dist/point-decision.d.ts.map +1 -1
- package/dist/point-decision.js +3 -0
- package/dist/wave-runtime.mjs +240 -224
- package/dist/workspace/checks/links.d.ts +5 -0
- package/dist/workspace/checks/links.d.ts.map +1 -1
- package/dist/workspace/checks/links.js +54 -8
- package/dist/workspace/checks.d.ts +4 -1
- package/dist/workspace/checks.d.ts.map +1 -1
- package/dist/workspace/checks.js +6 -2
- package/dist/workspace/compose-graph.d.ts +26 -4
- package/dist/workspace/compose-graph.d.ts.map +1 -1
- package/dist/workspace/compose-graph.js +21 -7
- package/dist/workspace/dbt-project-reader.d.ts +36 -0
- package/dist/workspace/dbt-project-reader.d.ts.map +1 -0
- package/dist/workspace/dbt-project-reader.js +290 -0
- package/dist/workspace/graph-cli.d.ts.map +1 -1
- package/dist/workspace/graph-cli.js +3 -1
- package/dist/workspace/graph.schema.json +109 -4
- package/dist/workspace/intent.js +7 -2
- package/dist/workspace/kind-link-readers.d.ts +84 -0
- package/dist/workspace/kind-link-readers.d.ts.map +1 -0
- package/dist/workspace/kind-link-readers.js +53 -0
- package/dist/workspace/kinds.d.ts +15 -0
- package/dist/workspace/kinds.d.ts.map +1 -1
- package/dist/workspace/kinds.js +11 -0
- package/dist/workspace/lexicon-outputs.d.ts +63 -0
- package/dist/workspace/lexicon-outputs.d.ts.map +1 -0
- package/dist/workspace/lexicon-outputs.js +241 -0
- package/dist/workspace/links.d.ts +29 -7
- package/dist/workspace/links.d.ts.map +1 -1
- package/dist/workspace/links.js +109 -34
- package/dist/workspace/member-handles.d.ts +18 -1
- package/dist/workspace/member-handles.d.ts.map +1 -1
- package/dist/workspace/member-handles.js +31 -14
- package/dist/workspace/nested-graph.d.ts +1 -1
- package/dist/workspace/nested-graph.d.ts.map +1 -1
- package/dist/workspace/nested-graph.js +5 -1
- package/dist/workspace/records-checkout.d.ts +11 -1
- package/dist/workspace/records-checkout.d.ts.map +1 -1
- package/dist/workspace/records-checkout.js +38 -2
- package/dist/workspace/runs.d.ts.map +1 -1
- package/dist/workspace/runs.js +4 -2
- package/dist/workspace/status.d.ts.map +1 -1
- package/dist/workspace/status.js +3 -2
- package/dist/workspace/workspace-kinds.schema.json +5 -0
- package/package.json +1 -1
- package/src/audit/rules-doc.ts +1 -1
- package/src/ci-jobs.ts +58 -1
- package/src/cli/commands/build.ts +18 -0
- package/src/cli/handlers/approve.ts +9 -0
- package/src/cli/handlers/emulator.test.ts +61 -1
- package/src/cli/handlers/emulator.ts +65 -4
- package/src/cli/handlers/graph.ts +22 -1
- package/src/cli/handlers/promote.test.ts +7 -0
- package/src/cli/handlers/promote.ts +7 -1
- package/src/cli/handlers/run-no-record.e2e.test.ts +87 -0
- package/src/cli/handlers/run-wave.ts +5 -4
- package/src/cli/handlers/run.test.ts +126 -0
- package/src/cli/handlers/run.ts +69 -9
- package/src/cli/main.test.ts +24 -0
- package/src/cli/main.ts +17 -4
- package/src/cli/parse-args.ts +21 -0
- package/src/cli/registry.ts +12 -0
- package/src/components/cli-support.test.ts +75 -0
- package/src/components/cli-support.ts +130 -25
- package/src/components/driver.test.ts +28 -0
- package/src/components/driver.ts +23 -10
- package/src/components/generate-disabled.test.ts +56 -0
- package/src/graph-ci-edges.test.ts +50 -0
- package/src/graph-detail.ts +21 -2
- package/src/graph-ir.test.ts +23 -1
- package/src/graph-ir.ts +65 -0
- package/src/lexicon.ts +26 -1
- package/src/lifecycle/git.ts +61 -6
- package/src/lifecycle/no-record.test.ts +175 -0
- package/src/lifecycle/no-record.ts +108 -0
- package/src/op/activities/shell.ts +28 -6
- package/src/op/emulator-lifecycle.test.ts +24 -1
- package/src/op/emulator-lifecycle.ts +58 -2
- package/src/op/gate.ts +3 -0
- package/src/op/index.ts +1 -1
- package/src/op/local-executor.ts +20 -3
- package/src/op/op-waves-jobs.test.ts +213 -0
- package/src/op/op-waves-notify.ts +11 -5
- package/src/op/op-waves-run.ts +42 -11
- package/src/op/op-waves.ts +202 -2
- package/src/op/runtimes/local.ts +6 -0
- package/src/op/step-console.test.ts +150 -0
- package/src/op/step-console.ts +155 -0
- package/src/point-decision-bundle.test.ts +1 -0
- package/src/point-decision.ts +3 -0
- package/src/wave-loop-golden.test.ts +3 -0
- package/src/workspace/checks/links.ts +57 -8
- package/src/workspace/checks.e2e.test.ts +1 -1
- package/src/workspace/checks.ts +9 -2
- package/src/workspace/compose-graph.test.ts +55 -0
- package/src/workspace/compose-graph.ts +33 -9
- package/src/workspace/dbt-kind.e2e.test.ts +95 -0
- package/src/workspace/dbt-kind.test.ts +283 -0
- package/src/workspace/dbt-project-reader.ts +291 -0
- package/src/workspace/graph-cli.ts +3 -1
- package/src/workspace/graph-contract.test.ts +26 -1
- package/src/workspace/graph.schema.json +109 -4
- package/src/workspace/intent.ts +7 -2
- package/src/workspace/kind-link-readers.ts +109 -0
- package/src/workspace/kinds.ts +21 -1
- package/src/workspace/lexicon-outputs.ts +272 -0
- package/src/workspace/links.test.ts +22 -0
- package/src/workspace/links.ts +129 -34
- package/src/workspace/member-handles.ts +28 -8
- package/src/workspace/nested-graph.test.ts +14 -0
- package/src/workspace/nested-graph.ts +7 -2
- package/src/workspace/records-checkout.ts +36 -2
- package/src/workspace/records-uncommitted.test.ts +5 -1
- package/src/workspace/runs.ts +4 -2
- package/src/workspace/status.ts +3 -2
- package/src/workspace/workspace-kinds.schema.json +5 -0
|
@@ -17,31 +17,58 @@
|
|
|
17
17
|
* | WSP096 | a consumer states the same link twice |
|
|
18
18
|
* | WSP097 | an entry lists `outputs` for a kind that doesn't take them from the entry |
|
|
19
19
|
* | WSP098 | a link states a `protocol` and its kind is not `telemetry` (#2558) |
|
|
20
|
+
* | WSP099 | a kind's link reader could not read part of a member's files without guessing (#3932) |
|
|
21
|
+
*
|
|
22
|
+
* Links a kind's reader finds in a member's own files (#3932), such as a dbt
|
|
23
|
+
* project's sources, are checked by WSP093 to WSP095 like declared ones, and
|
|
24
|
+
* their findings name the member's file and line instead of the declaration.
|
|
20
25
|
*/
|
|
21
26
|
|
|
22
27
|
import type { WorkspaceCheck, WorkspaceCheckContext, WorkspaceDiagnostic } from "../checks";
|
|
23
28
|
import type { Member } from "../declaration";
|
|
24
29
|
import { sourceMemberHandles } from "../member-handles";
|
|
30
|
+
import { readKindLinks, type KindLinkRead } from "../kind-link-readers";
|
|
25
31
|
import { DEFAULT_LINK_KIND, LINK_KINDS, TELEMETRY_LINK_KIND, linkTargetProblem, resolveLinks, type AmbiguousRow, type LinkRow, type LinkTableRow } from "../links";
|
|
26
32
|
|
|
27
33
|
const tables = new WeakMap<WorkspaceCheckContext, LinkTableRow[]>();
|
|
34
|
+
const reads = new WeakMap<WorkspaceCheckContext, Map<string, KindLinkRead>>();
|
|
35
|
+
|
|
36
|
+
/** What each member's kind reader found (#3932), read once per check run. */
|
|
37
|
+
function kindReads(ctx: WorkspaceCheckContext): Map<string, KindLinkRead> {
|
|
38
|
+
let read = reads.get(ctx);
|
|
39
|
+
if (!read) {
|
|
40
|
+
read = readKindLinks(ctx.declaration, ctx.tree, ctx.kinds);
|
|
41
|
+
reads.set(ctx, read);
|
|
42
|
+
}
|
|
43
|
+
return read;
|
|
44
|
+
}
|
|
28
45
|
|
|
29
46
|
/** The link rows for a check run, resolved once and shared by every link check. */
|
|
30
47
|
export function linkTable(ctx: WorkspaceCheckContext): LinkTableRow[] {
|
|
31
48
|
let rows = tables.get(ctx);
|
|
32
49
|
if (!rows) {
|
|
33
|
-
rows = resolveLinks(ctx.declaration, sourceMemberHandles(ctx.declaration, ctx.tree, ctx.groups, ctx.kinds));
|
|
50
|
+
rows = resolveLinks(ctx.declaration, sourceMemberHandles(ctx.declaration, ctx.tree, ctx.groups, ctx.kinds, ctx.facts?.lexiconOutputs), undefined, kindReads(ctx));
|
|
34
51
|
tables.set(ctx, rows);
|
|
35
52
|
}
|
|
36
53
|
return rows;
|
|
37
54
|
}
|
|
38
55
|
|
|
39
|
-
|
|
56
|
+
/** The declared links, and the links a kind's reader found: the rows a member states. */
|
|
57
|
+
const statedRows = (ctx: WorkspaceCheckContext) => linkTable(ctx).filter((r): r is LinkRow => r.status !== "ambiguous" && r.origin !== "inferred:joinKey");
|
|
40
58
|
|
|
41
|
-
|
|
42
|
-
|
|
59
|
+
/** A finding at a declared link's field in the declaration, or at the file and line a reader's link came from. */
|
|
60
|
+
function linkFinding(check: WorkspaceCheck, ctx: WorkspaceCheckContext, row: LinkTableRow, field: string, message: string): WorkspaceDiagnostic {
|
|
61
|
+
const base = { checkId: check.id, severity: check.severity, message, entity: row.consumer };
|
|
62
|
+
if (row.origin === "kind-reader" && row.file !== undefined) {
|
|
63
|
+
const member = ctx.declaration.members.find((m) => m.name === row.consumer);
|
|
64
|
+
return { ...base, pointer: member?.pointer ?? "", treeFile: row.file, line: row.line ?? 1, column: 1 };
|
|
65
|
+
}
|
|
66
|
+
return { ...base, pointer: `${(row as LinkRow).pointer}/${field}` };
|
|
43
67
|
}
|
|
44
68
|
|
|
69
|
+
/** Where a reader's link came from, for its message; empty for a declared link. */
|
|
70
|
+
const statedAt = (row: LinkRow) => (row.origin === "kind-reader" ? ` (read by ${row.reader} from ${row.file}:${row.line})` : "");
|
|
71
|
+
|
|
45
72
|
const describeLink = (row: LinkRow) => `${row.consumer}'s link to ${row.producer} output ${row.output}`;
|
|
46
73
|
|
|
47
74
|
export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
@@ -87,9 +114,9 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
87
114
|
severity: "error",
|
|
88
115
|
configurable: true,
|
|
89
116
|
check(ctx) {
|
|
90
|
-
return
|
|
117
|
+
return statedRows(ctx)
|
|
91
118
|
.filter((r) => r.status === "missing")
|
|
92
|
-
.map((r) => linkFinding(this, r, "output", `${describeLink(r)} does not resolve: ${r.reason}`));
|
|
119
|
+
.map((r) => linkFinding(this, ctx, r, "output", `${describeLink(r)}${statedAt(r)} does not resolve: ${r.reason}`));
|
|
93
120
|
},
|
|
94
121
|
},
|
|
95
122
|
{
|
|
@@ -99,9 +126,9 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
99
126
|
severity: "info",
|
|
100
127
|
configurable: true,
|
|
101
128
|
check(ctx) {
|
|
102
|
-
return
|
|
129
|
+
return statedRows(ctx)
|
|
103
130
|
.filter((r) => r.status === "unresolved")
|
|
104
|
-
.map((r) => linkFinding(this, r, "output", `${describeLink(r)} is kept unresolved: ${r.reason}`));
|
|
131
|
+
.map((r) => linkFinding(this, ctx, r, "output", `${describeLink(r)}${statedAt(r)} is kept unresolved: ${r.reason}`));
|
|
105
132
|
},
|
|
106
133
|
},
|
|
107
134
|
{
|
|
@@ -115,6 +142,10 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
115
142
|
return linkTable(ctx)
|
|
116
143
|
.filter((r): r is AmbiguousRow => r.status === "ambiguous")
|
|
117
144
|
.map((r) => {
|
|
145
|
+
if (r.origin === "kind-reader") {
|
|
146
|
+
const choices = r.candidates.map((c) => c.producer).join(", ");
|
|
147
|
+
return linkFinding(this, ctx, r, "", `${r.consumer}'s link to ${r.input} (read by ${r.reader}) has no single producer: ${r.reason}${choices ? `. Candidates: ${choices}` : ""}`);
|
|
148
|
+
}
|
|
118
149
|
const choices = r.candidates.map((c) => `${c.producer} output ${c.output} (${c.label})`).join(", ");
|
|
119
150
|
const first = r.candidates[0];
|
|
120
151
|
return {
|
|
@@ -201,4 +232,22 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
201
232
|
);
|
|
202
233
|
},
|
|
203
234
|
},
|
|
235
|
+
{
|
|
236
|
+
id: "WSP099",
|
|
237
|
+
name: "kind-reader-gap",
|
|
238
|
+
description:
|
|
239
|
+
"A kind's link reader read every link in a member's files that it could read without guessing (#3932). A value it would have to render, such as a Jinja template in a dbt source, is reported here with its file and line, and the links it would state are not checked.",
|
|
240
|
+
severity: "info",
|
|
241
|
+
configurable: true,
|
|
242
|
+
check(ctx) {
|
|
243
|
+
const out: WorkspaceDiagnostic[] = [];
|
|
244
|
+
for (const [member, read] of kindReads(ctx)) {
|
|
245
|
+
const pointer = ctx.declaration.members.find((m) => m.name === member)?.pointer ?? "";
|
|
246
|
+
for (const gap of read.gaps) {
|
|
247
|
+
out.push({ checkId: this.id, severity: this.severity, message: `${member} (read by ${read.reader}): ${gap.message}`, entity: member, pointer, treeFile: gap.file, line: gap.line, column: 1 });
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return out;
|
|
251
|
+
},
|
|
252
|
+
},
|
|
204
253
|
];
|
|
@@ -75,7 +75,7 @@ describe("the WSP catalog", () => {
|
|
|
75
75
|
expect(list.filter((id) => /^WSP(07|08|09|10)\d$/.test(id))).toEqual([
|
|
76
76
|
"WSP071", "WSP072", "WSP073",
|
|
77
77
|
"WSP081", "WSP082", "WSP083",
|
|
78
|
-
"WSP091", "WSP092", "WSP093", "WSP094", "WSP095", "WSP096", "WSP097", "WSP098",
|
|
78
|
+
"WSP091", "WSP092", "WSP093", "WSP094", "WSP095", "WSP096", "WSP097", "WSP098", "WSP099",
|
|
79
79
|
"WSP101", "WSP102", "WSP103", "WSP104", "WSP105", "WSP106",
|
|
80
80
|
]);
|
|
81
81
|
});
|
package/src/workspace/checks.ts
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* | WSP001 to WSP011 | the declaration and member kinds (this file) |
|
|
25
25
|
* | WSP071, WSP072 | member ledgers |
|
|
26
26
|
* | WSP081 to WSP083 | recorded pipelines |
|
|
27
|
-
* | WSP091 to
|
|
27
|
+
* | WSP091 to WSP099 | member links (#2539, #2558), and those a kind's reader finds in a member's files (#3932) |
|
|
28
28
|
* | WSP101 to WSP106 | generated files |
|
|
29
29
|
* | WSP111 to WSP114 | records read with `--kind` (#2549) |
|
|
30
30
|
* | WSP115 | the record kinds the declaration names (#2680) |
|
|
@@ -59,6 +59,7 @@ import { BOX_CHECKS } from "./checks/boxes";
|
|
|
59
59
|
import { DIAGRAM_CHECKS } from "./checks/diagrams";
|
|
60
60
|
import { loadDeclaredKinds, type DeclaredKind } from "./declared-kinds";
|
|
61
61
|
import { resolveBoxIntents, type ResolvedBoxIntent } from "./box-intent";
|
|
62
|
+
import { gatherLexiconOutputs, type MemberLexiconOutputs } from "./lexicon-outputs";
|
|
62
63
|
|
|
63
64
|
/**
|
|
64
65
|
* What the checks beyond the declaration read, gathered from the checkout
|
|
@@ -80,6 +81,8 @@ export interface WorkspaceFacts {
|
|
|
80
81
|
live?: LiveLinkFacts;
|
|
81
82
|
/** The decision record each box's intent names, read from the working tree (#2850). */
|
|
82
83
|
boxIntents?: readonly ResolvedBoxIntent[];
|
|
84
|
+
/** The outputs linked members' lexicons report about their declarations, read by folding their source (#3931). */
|
|
85
|
+
lexiconOutputs?: ReadonlyMap<string, MemberLexiconOutputs>;
|
|
83
86
|
}
|
|
84
87
|
|
|
85
88
|
/** What every declaration check reads. */
|
|
@@ -355,7 +358,7 @@ export const WORKSPACE_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
355
358
|
...LEDGER_CHECKS,
|
|
356
359
|
// Recorded pipelines (#2542).
|
|
357
360
|
...PIPELINE_CHECKS,
|
|
358
|
-
// Member links (#2539, #2558), WSP091 to
|
|
361
|
+
// Member links (#2539, #2558, #3932), WSP091 to WSP099.
|
|
359
362
|
...LINK_CHECKS,
|
|
360
363
|
// Generated files (#2541).
|
|
361
364
|
...GENERATED_CHECKS,
|
|
@@ -521,7 +524,11 @@ export async function runDeclarationChecks(
|
|
|
521
524
|
const declaredKinds = options.gather === false ? undefined : await loadDeclaredKinds(declaration, tree, root, { load: !options.tree, acceptance: !options.tree });
|
|
522
525
|
// A box's intent is read from the working tree's records, so not under --at (#2850).
|
|
523
526
|
const boxIntents = declaredKinds && !options.tree ? await resolveBoxIntents(declaration, root, declaredKinds) : undefined;
|
|
527
|
+
// A linked member's lexicons name its tables and columns as outputs (#3931); reading them folds its source, never runs it.
|
|
528
|
+
const lexiconOutputs =
|
|
529
|
+
options.gather === false ? undefined : await gatherLexiconOutputs(root, declaration, options.tree ? workingTree(root) : tree, groups, registry, { atRevision: options.tree !== undefined });
|
|
524
530
|
const facts: WorkspaceFacts = {
|
|
531
|
+
...(lexiconOutputs && lexiconOutputs.size > 0 ? { lexiconOutputs } : {}),
|
|
525
532
|
...gathered,
|
|
526
533
|
...(options.records ? { records: options.records } : {}),
|
|
527
534
|
...(declaredKinds ? { declaredKinds } : {}),
|
|
@@ -104,6 +104,61 @@ describe("composeWorkspaceGraph", () => {
|
|
|
104
104
|
});
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
+
describe("CI edges, pipelines and dangling references (#3922)", () => {
|
|
108
|
+
/** A github member as `chant graph` reports it after #3910. */
|
|
109
|
+
const ci = (workflow: string, job: string): GraphIR => ({
|
|
110
|
+
version: 1,
|
|
111
|
+
nodes: [
|
|
112
|
+
{ id: workflow, kind: "Workflow", lexicon: "github", attrs: {} },
|
|
113
|
+
{ id: job, kind: "Job", lexicon: "github", attrs: {} },
|
|
114
|
+
{ id: "deploy", kind: "Job", lexicon: "github", attrs: {} },
|
|
115
|
+
],
|
|
116
|
+
edges: [
|
|
117
|
+
{ from: "deploy", to: job, kind: "needs", viaAttr: "needs" },
|
|
118
|
+
{ from: job, to: workflow, kind: "calls", viaAttr: "uses", crossPipeline: true },
|
|
119
|
+
],
|
|
120
|
+
groups: { byLexicon: { github: [job, "deploy", workflow] }, byPipeline: { ci: [job, "deploy", workflow] } },
|
|
121
|
+
dangling: [
|
|
122
|
+
{ from: "deploy", kind: "needs", path: "needs", value: "lint" },
|
|
123
|
+
{ from: job, kind: "calls", path: "uses", value: "other/repo/.github/workflows/x.yml@main", crossPipeline: true },
|
|
124
|
+
],
|
|
125
|
+
});
|
|
126
|
+
const doc = composeWorkspaceGraph({ name: "acme", root: "." }, [
|
|
127
|
+
{ member: member("svc", "svc"), ir: ci("ciWorkflow", "build") },
|
|
128
|
+
{ member: member("app", "app"), ir: ci("ciWorkflow", "test") },
|
|
129
|
+
{ member: member("db", "db"), ir: db },
|
|
130
|
+
]);
|
|
131
|
+
|
|
132
|
+
test("edges keep their kind and crossPipeline", () => {
|
|
133
|
+
expect(doc.edges.filter((e) => e.member === "svc")).toEqual([
|
|
134
|
+
{ from: "svc/build", to: "svc/ciWorkflow", kind: "calls", viaAttr: "uses", crossPipeline: true, member: "svc" },
|
|
135
|
+
{ from: "svc/deploy", to: "svc/build", kind: "needs", viaAttr: "needs", member: "svc" },
|
|
136
|
+
]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("byPipeline keys and ids are prefixed with the member", () => {
|
|
140
|
+
expect(doc.groups.byPipeline).toEqual({
|
|
141
|
+
"app/ci": ["app/ciWorkflow", "app/deploy", "app/test"],
|
|
142
|
+
"svc/ci": ["svc/build", "svc/ciWorkflow", "svc/deploy"],
|
|
143
|
+
});
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("dangling references keep their fields, with from prefixed and the member named, sorted", () => {
|
|
147
|
+
expect(doc.dangling).toEqual([
|
|
148
|
+
{ from: "app/deploy", kind: "needs", path: "needs", value: "lint", member: "app" },
|
|
149
|
+
{ from: "app/test", kind: "calls", path: "uses", value: "other/repo/.github/workflows/x.yml@main", crossPipeline: true, member: "app" },
|
|
150
|
+
{ from: "svc/build", kind: "calls", path: "uses", value: "other/repo/.github/workflows/x.yml@main", crossPipeline: true, member: "svc" },
|
|
151
|
+
{ from: "svc/deploy", kind: "needs", path: "needs", value: "lint", member: "svc" },
|
|
152
|
+
]);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("no dangling section and no byPipeline when no member reports either", () => {
|
|
156
|
+
const plain = composeWorkspaceGraph({ name: "acme", root: "." }, [{ member: member("db", "db"), ir: db }]);
|
|
157
|
+
expect("dangling" in plain).toBe(false);
|
|
158
|
+
expect(plain.groups.byPipeline).toBeUndefined();
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
|
|
107
162
|
describe("the links section (#2539)", () => {
|
|
108
163
|
test("inferred joins carry their label, and a declared link suppresses the inferred edge it covers", () => {
|
|
109
164
|
const consumer: GraphIR = {
|
|
@@ -11,8 +11,12 @@
|
|
|
11
11
|
* `runtimeOwner` and export and import nodes are rewritten the same way.
|
|
12
12
|
* - `groups.byMember` maps each member to its node ids. `byLexicon` and
|
|
13
13
|
* `byComposite` keep their keys (a lexicon or composite type is the same
|
|
14
|
-
* thing in every member) and merge the ids. `byStack`, `byContainer
|
|
15
|
-
* `byWave` name things inside one member, so their keys
|
|
14
|
+
* thing in every member) and merge the ids. `byStack`, `byContainer`,
|
|
15
|
+
* `byWave` and `byPipeline` name things inside one member, so their keys
|
|
16
|
+
* are prefixed too.
|
|
17
|
+
* - Edges keep their `kind` and `crossPipeline` (#3910). A member's dangling
|
|
18
|
+
* references (`ir.dangling`) land on the top-level `dangling` list with
|
|
19
|
+
* `from` prefixed and a `member` field, like edges (#3922).
|
|
16
20
|
* - `links` holds the member links (#2524 D6, #2539): the declared links and
|
|
17
21
|
* the joins inferred from members' `imports` and `exports` with the core
|
|
18
22
|
* `joinKey()`, labelled `exact` or `folded` (`./links.ts`). A declared link
|
|
@@ -33,9 +37,10 @@ import type { FactoryView, ListingView } from "./box-factory";
|
|
|
33
37
|
import type { Declaration } from "./declaration";
|
|
34
38
|
import type { KindRegistry } from "./kinds";
|
|
35
39
|
import { graphLinks, type LinkTableRow } from "./links";
|
|
40
|
+
import type { KindLinkRead } from "./kind-link-readers";
|
|
36
41
|
import type { RecordLinkRow } from "./record-assets";
|
|
37
42
|
import type { ReasonCode } from "./reason-codes";
|
|
38
|
-
import { GRAPH_IR_VERSION, type GraphIR, type IRExport, type IRGroups, type IRImport, type IREdge, type IRNode } from "../graph-ir";
|
|
43
|
+
import { GRAPH_IR_VERSION, type GraphIR, type IRExport, type IRDanglingRef, type IRGroups, type IRImport, type IREdge, type IRNode } from "../graph-ir";
|
|
39
44
|
|
|
40
45
|
/** The version of the composed document `chant workspace graph` writes. */
|
|
41
46
|
export const WORKSPACE_GRAPH_VERSION = 1;
|
|
@@ -164,6 +169,8 @@ export const COLLECTOR_META_KEY = "collector";
|
|
|
164
169
|
/** `nested` names the nested workspace's member a node was read through (#2551); absent for the outer workspace's own nodes. */
|
|
165
170
|
export type ComposedNode = IRNode & { member: string; nested?: string };
|
|
166
171
|
export type ComposedEdge = IREdge & { member: string };
|
|
172
|
+
/** A member's dangling reference (#3910, #3922): `from` is prefixed like a node id. `nested` as on {@link ComposedNode}. */
|
|
173
|
+
export type ComposedDangling = IRDanglingRef & { member: string; nested?: string };
|
|
167
174
|
|
|
168
175
|
export interface WorkspaceGraph {
|
|
169
176
|
version: number;
|
|
@@ -175,6 +182,12 @@ export interface WorkspaceGraph {
|
|
|
175
182
|
exports: (IRExport & { member: string })[];
|
|
176
183
|
imports: (IRImport & { member: string })[];
|
|
177
184
|
derivedAttrs?: Record<string, string[]>;
|
|
185
|
+
/**
|
|
186
|
+
* The members' dangling CI references (#3910, #3922): each `ir.dangling`
|
|
187
|
+
* entry with `from` prefixed by its member and a `member` field, sorted by
|
|
188
|
+
* `from`, `path` and `value`. Absent when no member has one.
|
|
189
|
+
*/
|
|
190
|
+
dangling?: ComposedDangling[];
|
|
178
191
|
/**
|
|
179
192
|
* Member links (#2524 D6, #2539), declared and inferred, then the links of
|
|
180
193
|
* the records read with `--kind` (#2549). Empty when no declaration is given.
|
|
@@ -288,6 +301,13 @@ function prefixRefs(value: unknown, member: string): unknown {
|
|
|
288
301
|
return out;
|
|
289
302
|
}
|
|
290
303
|
|
|
304
|
+
/** The groups whose keys name something inside one member, so are prefixed with it. */
|
|
305
|
+
export const SCOPED_GROUPS = ["byStack", "byContainer", "byWave", "byPipeline"] as const;
|
|
306
|
+
type ScopedGroup = (typeof SCOPED_GROUPS)[number];
|
|
307
|
+
|
|
308
|
+
/** The order the composed `dangling` list is sorted in. */
|
|
309
|
+
export const danglingKey = (d: IRDanglingRef): string => `${d.from}\u0000${d.path}\u0000${d.value}`;
|
|
310
|
+
|
|
291
311
|
function mergeInto(target: Record<string, string[]>, key: string, ids: string[]): void {
|
|
292
312
|
(target[key] ??= []).push(...ids);
|
|
293
313
|
}
|
|
@@ -302,12 +322,13 @@ function sorted(rec: Record<string, string[]>): Record<string, string[]> {
|
|
|
302
322
|
* Compose the members' IRs into the workspace document. Members keep the
|
|
303
323
|
* order they are given in. With `links`, the `links` section is filled from
|
|
304
324
|
* the declaration; `kinds` lets a member that isn't composed but whose kind
|
|
305
|
-
* lists its outputs (`other`, kinds from a package) be a link target
|
|
325
|
+
* lists its outputs (`other`, kinds from a package) be a link target, and
|
|
326
|
+
* `read` holds the links each member's kind reader found in its files (#3932).
|
|
306
327
|
*/
|
|
307
328
|
export function composeWorkspaceGraph(
|
|
308
329
|
workspace: { name: string; root: string },
|
|
309
330
|
inputs: ComposeInput[],
|
|
310
|
-
links?: { declaration: Declaration; kinds?: KindRegistry },
|
|
331
|
+
links?: { declaration: Declaration; kinds?: KindRegistry; read?: ReadonlyMap<string, KindLinkRead> },
|
|
311
332
|
): WorkspaceGraph {
|
|
312
333
|
const nodes: ComposedNode[] = [];
|
|
313
334
|
const edges: ComposedEdge[] = [];
|
|
@@ -315,7 +336,8 @@ export function composeWorkspaceGraph(
|
|
|
315
336
|
const imports: WorkspaceGraph["imports"] = [];
|
|
316
337
|
const byMember: Record<string, string[]> = {};
|
|
317
338
|
const shared: Record<"byLexicon" | "byComposite", Record<string, string[]>> = { byLexicon: {}, byComposite: {} };
|
|
318
|
-
const scoped: Record<
|
|
339
|
+
const scoped: Record<ScopedGroup, Record<string, string[]>> = { byStack: {}, byContainer: {}, byWave: {}, byPipeline: {} };
|
|
340
|
+
const dangling: ComposedDangling[] = [];
|
|
319
341
|
const derivedAttrs: Record<string, Set<string>> = {};
|
|
320
342
|
const collectors: MemberCollector[] = [];
|
|
321
343
|
|
|
@@ -335,9 +357,10 @@ export function composeWorkspaceGraph(
|
|
|
335
357
|
for (const key of ["byLexicon", "byComposite"] as const) {
|
|
336
358
|
for (const [k, v] of Object.entries(ir.groups[key] ?? {})) mergeInto(shared[key], k, v.map((id) => prefix(m, id)));
|
|
337
359
|
}
|
|
338
|
-
for (const key of
|
|
360
|
+
for (const key of SCOPED_GROUPS) {
|
|
339
361
|
for (const [k, v] of Object.entries(ir.groups[key] ?? {})) mergeInto(scoped[key], prefix(m, k), v.map((id) => prefix(m, id)));
|
|
340
362
|
}
|
|
363
|
+
for (const d of ir.dangling ?? []) dangling.push({ ...d, from: prefix(m, d.from), member: m });
|
|
341
364
|
for (const e of ir.exports ?? []) exports.push({ ...e, ...(e.node ? { node: prefix(m, e.node) } : {}), member: m });
|
|
342
365
|
for (const i of ir.imports ?? []) imports.push({ ...i, node: prefix(m, i.node), member: m });
|
|
343
366
|
for (const [kind, attrs] of Object.entries(ir.derivedAttrs ?? {})) for (const a of attrs) (derivedAttrs[kind] ??= new Set()).add(a);
|
|
@@ -358,7 +381,7 @@ export function composeWorkspaceGraph(
|
|
|
358
381
|
|
|
359
382
|
const groups: WorkspaceGraph["groups"] = { byMember };
|
|
360
383
|
for (const key of ["byLexicon", "byComposite"] as const) if (Object.keys(shared[key]).length) groups[key] = sorted(shared[key]);
|
|
361
|
-
for (const key of
|
|
384
|
+
for (const key of SCOPED_GROUPS) if (Object.keys(scoped[key]).length) groups[key] = sorted(scoped[key]);
|
|
362
385
|
|
|
363
386
|
const doc: WorkspaceGraph = {
|
|
364
387
|
version: WORKSPACE_GRAPH_VERSION,
|
|
@@ -370,11 +393,12 @@ export function composeWorkspaceGraph(
|
|
|
370
393
|
exports,
|
|
371
394
|
imports,
|
|
372
395
|
links: links
|
|
373
|
-
? graphLinks(links.declaration, { composed: inputs.filter((i) => i.ir).map((i) => i.member.name), collectors, exports, imports }, links.kinds)
|
|
396
|
+
? graphLinks(links.declaration, { composed: inputs.filter((i) => i.ir).map((i) => i.member.name), collectors, exports, imports }, links.kinds, links.read)
|
|
374
397
|
: [],
|
|
375
398
|
collectors,
|
|
376
399
|
records: [],
|
|
377
400
|
};
|
|
401
|
+
if (dangling.length) doc.dangling = dangling.sort((a, b) => danglingKey(a).localeCompare(danglingKey(b)));
|
|
378
402
|
if (Object.keys(derivedAttrs).length) {
|
|
379
403
|
doc.derivedAttrs = Object.fromEntries(Object.keys(derivedAttrs).sort().map((k) => [k, [...derivedAttrs[k]].sort()]));
|
|
380
404
|
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `dbt` kind end to end (#3932): the sql lexicon pinned as a package,
|
|
3
|
+
* its kinds file found through node_modules, and this checkout's chant CLI
|
|
4
|
+
* running `workspace check` and `workspace graph`. The producer is a real
|
|
5
|
+
* sql member, a chant project that declares `orders`, whose tables and
|
|
6
|
+
* columns are outputs through the sql lexicon's hook (#3931).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { spawnSync } from "node:child_process";
|
|
10
|
+
import { readFileSync, symlinkSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { pathToFileURL } from "node:url";
|
|
13
|
+
import { afterAll, describe, expect, test } from "vitest";
|
|
14
|
+
import { cleanScratch, contract, declaration, REPO, repo } from "./__fixtures__/contract-repo";
|
|
15
|
+
import schema from "./graph.schema.json";
|
|
16
|
+
|
|
17
|
+
afterAll(cleanScratch);
|
|
18
|
+
|
|
19
|
+
const TIMEOUT = 120_000;
|
|
20
|
+
const LEXICON = "@intentius/chant-lexicon-sql";
|
|
21
|
+
const VERSION = (JSON.parse(readFileSync(join(REPO, "lexicons", "sql", "package.json"), "utf-8")) as { version: string }).version;
|
|
22
|
+
|
|
23
|
+
function chant(cwd: string, ...argv: string[]) {
|
|
24
|
+
return spawnSync(process.execPath, ["--import", pathToFileURL(join(REPO, "node_modules/tsx/dist/loader.mjs")).href, join(REPO, "packages/core/src/cli/main.ts"), ...argv], {
|
|
25
|
+
cwd,
|
|
26
|
+
encoding: "utf-8",
|
|
27
|
+
timeout: TIMEOUT,
|
|
28
|
+
env: { ...process.env, NO_COLOR: "1", TSX_DISABLE_CACHE: "1" },
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const SQL_CONFIG = `import type { ChantConfig } from "@intentius/chant";\nexport default { lexicons: ["sql"], sql: { dialect: "postgres" } } satisfies ChantConfig;\n`;
|
|
33
|
+
|
|
34
|
+
const orders = (columns: string[]) =>
|
|
35
|
+
[
|
|
36
|
+
`import { table } from "@intentius/chant-lexicon-sql/postgres";`,
|
|
37
|
+
``,
|
|
38
|
+
`export const orders = table\``,
|
|
39
|
+
` CREATE TABLE orders (`,
|
|
40
|
+
...columns.map((c, i) => ` ${c}${i < columns.length - 1 ? "," : ""}`),
|
|
41
|
+
` )\`;`,
|
|
42
|
+
``,
|
|
43
|
+
].join("\n");
|
|
44
|
+
|
|
45
|
+
function workspace(columns: string[]): string {
|
|
46
|
+
const root = repo({
|
|
47
|
+
"chant.workspace.json": declaration(
|
|
48
|
+
[
|
|
49
|
+
{ name: "db", dir: "db", kind: "chant" },
|
|
50
|
+
{ name: "analytics", dir: "analytics", kind: "dbt", fields: { sqlMember: "db" } },
|
|
51
|
+
],
|
|
52
|
+
{ pins: [{ package: LEXICON, version: VERSION }] },
|
|
53
|
+
),
|
|
54
|
+
"db/chant.config.ts": SQL_CONFIG,
|
|
55
|
+
"db/src/orders.ts": orders(columns),
|
|
56
|
+
"analytics/dbt_project.yml": "name: analytics\nversion: '1.0'\nprofile: analytics\n",
|
|
57
|
+
"analytics/models/staging/sources.yml": "version: 2\n\nsources:\n - name: shop\n schema: public\n tables:\n - name: orders\n columns:\n - name: id\n - name: customer_id\n",
|
|
58
|
+
".gitignore": "node_modules\n",
|
|
59
|
+
});
|
|
60
|
+
symlinkSync(join(REPO, "node_modules"), join(root, "node_modules"), "dir");
|
|
61
|
+
return root;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
describe("a dbt member through the pinned sql lexicon (#3932)", () => {
|
|
65
|
+
test(
|
|
66
|
+
"check passes while the sql member has public.orders.customer_id, and fails naming the dbt source file once it is dropped",
|
|
67
|
+
() => {
|
|
68
|
+
const ok = chant(workspace(["id bigint PRIMARY KEY", "customer_id bigint NOT NULL"]), "workspace", "check");
|
|
69
|
+
expect(ok.status, ok.stdout + ok.stderr).toBe(0);
|
|
70
|
+
|
|
71
|
+
const dropped = chant(workspace(["id bigint PRIMARY KEY"]), "workspace", "check");
|
|
72
|
+
expect(dropped.status).toBe(1);
|
|
73
|
+
const text = dropped.stdout + dropped.stderr;
|
|
74
|
+
expect(text).toContain("analytics/models/staging/sources.yml:10");
|
|
75
|
+
expect(text).toContain("WSP093");
|
|
76
|
+
expect(text).toMatch(/link to db output public\.orders\.customer_id .*does not resolve/);
|
|
77
|
+
},
|
|
78
|
+
TIMEOUT,
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
test(
|
|
82
|
+
"workspace graph lists the dbt member's links with their file and line",
|
|
83
|
+
() => {
|
|
84
|
+
const run = chant(workspace(["id bigint PRIMARY KEY"]), "workspace", "graph");
|
|
85
|
+
const doc = JSON.parse(run.stdout) as { links: { origin: string; consumer: string; producer: string; output: string; status: string; file?: string; line?: number }[] };
|
|
86
|
+
contract(schema).expectValid(doc);
|
|
87
|
+
expect(doc.links.filter((l) => l.origin === "kind-reader").map((l) => `${l.consumer} -> ${l.producer}.${l.output} ${l.status} ${l.file}:${l.line}`)).toEqual([
|
|
88
|
+
"analytics -> db.public.orders resolved analytics/models/staging/sources.yml:7",
|
|
89
|
+
"analytics -> db.public.orders.id resolved analytics/models/staging/sources.yml:9",
|
|
90
|
+
"analytics -> db.public.orders.customer_id missing analytics/models/staging/sources.yml:10",
|
|
91
|
+
]);
|
|
92
|
+
},
|
|
93
|
+
TIMEOUT,
|
|
94
|
+
);
|
|
95
|
+
});
|