@intentius/chant 0.124.22 → 0.124.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit/rules-doc.js +1 -1
- package/dist/cli/handlers/graph.js +24 -1
- package/dist/graph-ir.d.ts +43 -0
- package/dist/graph-ir.d.ts.map +1 -1
- package/dist/graph-ir.js +27 -0
- package/dist/lexicon.d.ts +25 -1
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/wave-runtime.mjs +222 -206
- package/dist/workspace/checks/links.d.ts +5 -0
- package/dist/workspace/checks/links.d.ts.map +1 -1
- package/dist/workspace/checks/links.js +54 -8
- package/dist/workspace/checks.d.ts +4 -1
- package/dist/workspace/checks.d.ts.map +1 -1
- package/dist/workspace/checks.js +6 -2
- package/dist/workspace/compose-graph.d.ts +4 -1
- package/dist/workspace/compose-graph.d.ts.map +1 -1
- package/dist/workspace/compose-graph.js +3 -2
- package/dist/workspace/dbt-project-reader.d.ts +36 -0
- package/dist/workspace/dbt-project-reader.d.ts.map +1 -0
- package/dist/workspace/dbt-project-reader.js +290 -0
- package/dist/workspace/graph-cli.d.ts.map +1 -1
- package/dist/workspace/graph-cli.js +3 -1
- package/dist/workspace/graph.schema.json +41 -3
- package/dist/workspace/intent.js +7 -2
- package/dist/workspace/kind-link-readers.d.ts +84 -0
- package/dist/workspace/kind-link-readers.d.ts.map +1 -0
- package/dist/workspace/kind-link-readers.js +53 -0
- package/dist/workspace/kinds.d.ts +15 -0
- package/dist/workspace/kinds.d.ts.map +1 -1
- package/dist/workspace/kinds.js +11 -0
- package/dist/workspace/lexicon-outputs.d.ts +63 -0
- package/dist/workspace/lexicon-outputs.d.ts.map +1 -0
- package/dist/workspace/lexicon-outputs.js +241 -0
- package/dist/workspace/links.d.ts +29 -7
- package/dist/workspace/links.d.ts.map +1 -1
- package/dist/workspace/links.js +109 -34
- package/dist/workspace/member-handles.d.ts +18 -1
- package/dist/workspace/member-handles.d.ts.map +1 -1
- package/dist/workspace/member-handles.js +31 -14
- package/dist/workspace/records-checkout.d.ts +11 -1
- package/dist/workspace/records-checkout.d.ts.map +1 -1
- package/dist/workspace/records-checkout.js +38 -2
- package/dist/workspace/status.d.ts.map +1 -1
- package/dist/workspace/status.js +3 -2
- package/dist/workspace/workspace-kinds.schema.json +5 -0
- package/package.json +1 -1
- package/src/audit/rules-doc.ts +1 -1
- package/src/cli/handlers/graph.ts +22 -1
- package/src/graph-ir.test.ts +23 -1
- package/src/graph-ir.ts +65 -0
- package/src/lexicon.ts +26 -1
- package/src/wave-loop-golden.test.ts +3 -0
- package/src/workspace/checks/links.ts +57 -8
- package/src/workspace/checks.e2e.test.ts +1 -1
- package/src/workspace/checks.ts +9 -2
- package/src/workspace/compose-graph.ts +5 -3
- package/src/workspace/dbt-kind.e2e.test.ts +95 -0
- package/src/workspace/dbt-kind.test.ts +283 -0
- package/src/workspace/dbt-project-reader.ts +291 -0
- package/src/workspace/graph-cli.ts +3 -1
- package/src/workspace/graph.schema.json +41 -3
- package/src/workspace/intent.ts +7 -2
- package/src/workspace/kind-link-readers.ts +109 -0
- package/src/workspace/kinds.ts +21 -1
- package/src/workspace/lexicon-outputs.ts +272 -0
- package/src/workspace/links.test.ts +22 -0
- package/src/workspace/links.ts +129 -34
- package/src/workspace/member-handles.ts +28 -8
- package/src/workspace/records-checkout.ts +36 -2
- package/src/workspace/records-uncommitted.test.ts +5 -1
- package/src/workspace/status.ts +3 -2
- package/src/workspace/workspace-kinds.schema.json +5 -0
|
@@ -2,7 +2,7 @@ import { dirname, resolve } from "node:path";
|
|
|
2
2
|
import { discoverOps } from "../../op/discover";
|
|
3
3
|
import { discover } from "../../discovery/index";
|
|
4
4
|
import { partitionByLexicon, computeStackGraph, build, mergeBuildRootEntities } from "../../build";
|
|
5
|
-
import { applyLexiconGraphEdges, buildGraphIr, buildLiveGraphIr, collectUnobserved, overlayGraphs, sourceOverlayGraphs, GRAPH_IR_VERSION, type GraphIR, type IRPipeline, type LexiconGraphEdges, type LiveObservation } from "../../graph-ir";
|
|
5
|
+
import { applyLexiconGraphEdges, applyLexiconOutputs, buildGraphIr, buildLiveGraphIr, collectUnobserved, overlayGraphs, sourceOverlayGraphs, GRAPH_IR_VERSION, type GraphIR, type IRPipeline, type LexiconGraphEdges, type LexiconWorkspaceOutputs, type LiveObservation } from "../../graph-ir";
|
|
6
6
|
import { behaviourOverlay } from "../../behaviour-overlay";
|
|
7
7
|
import type { SerializerResult } from "../../serializer";
|
|
8
8
|
import { applyBehaviourOverlay, behaviourRequestFromIr } from "../../behaviour-graph";
|
|
@@ -746,6 +746,26 @@ function withLexiconEdges(ir: GraphIR, plugins: LexiconPlugin[], entities: Map<s
|
|
|
746
746
|
return reports.length === 0 ? ir : applyLexiconGraphEdges(ir, reports);
|
|
747
747
|
}
|
|
748
748
|
|
|
749
|
+
/**
|
|
750
|
+
* Add the outputs the lexicons' `workspaceOutputs` hooks report (#3931) to
|
|
751
|
+
* the IR's `exports`: a sql table and each of its columns, which `chant
|
|
752
|
+
* workspace graph` resolves member links against. A hook that throws costs
|
|
753
|
+
* its own outputs and a warning, not the graph.
|
|
754
|
+
*/
|
|
755
|
+
function withLexiconOutputs(ir: GraphIR, plugins: LexiconPlugin[], entities: Map<string, import("../../declarable").Declarable>): GraphIR {
|
|
756
|
+
const reports: LexiconWorkspaceOutputs[] = [];
|
|
757
|
+
for (const p of plugins) {
|
|
758
|
+
if (!p.workspaceOutputs) continue;
|
|
759
|
+
try {
|
|
760
|
+
const report = p.workspaceOutputs(entities);
|
|
761
|
+
if (report) reports.push(report);
|
|
762
|
+
} catch (err) {
|
|
763
|
+
console.error(formatWarning({ message: `the ${p.name} lexicon's workspaceOutputs failed (${err instanceof Error ? err.message : String(err)}); its outputs are left out` }));
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
return reports.length === 0 ? ir : applyLexiconOutputs(ir, reports);
|
|
767
|
+
}
|
|
768
|
+
|
|
749
769
|
/** Returned instead of a graph when the prediction cannot be attempted honestly. */
|
|
750
770
|
const REFUSED = Symbol("behaviour-refused");
|
|
751
771
|
|
|
@@ -888,6 +908,7 @@ async function runGraphView(
|
|
|
888
908
|
? ctx.plugins
|
|
889
909
|
: await loadPlugins(await projectLexiconsOrNone(projectPath));
|
|
890
910
|
ir = withLexiconEdges(ir, plugins as LexiconPlugin[], result.entities);
|
|
911
|
+
ir = withLexiconOutputs(ir, plugins as LexiconPlugin[], result.entities);
|
|
891
912
|
const predicted = await predictOntoIr(ir, plugins as LexiconPlugin[], ctx.args, {
|
|
892
913
|
// No environment on the declared path: the file is not deployed anywhere
|
|
893
914
|
// yet, and naming one would claim it was.
|
package/src/graph-ir.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, test, expect } from "vitest";
|
|
2
|
-
import { buildGraphIr } from "./graph-ir";
|
|
2
|
+
import { applyLexiconOutputs, buildGraphIr } from "./graph-ir";
|
|
3
3
|
import { DECLARABLE_MARKER, type Declarable } from "./declarable";
|
|
4
4
|
import { AttrRef } from "./attrref";
|
|
5
5
|
import { LexiconOutput } from "./lexicon-output";
|
|
@@ -370,3 +370,25 @@ describe("whole declarables held in an entity's fields (#3257)", () => {
|
|
|
370
370
|
expect(ir.edges).toEqual([{ from: "lb", to: "vpc", kind: "ref", viaAttr: "actions" }]);
|
|
371
371
|
});
|
|
372
372
|
});
|
|
373
|
+
|
|
374
|
+
describe("applyLexiconOutputs (#3931)", () => {
|
|
375
|
+
test("adds each reported output to exports on the node that declares it, keeping a hand-written export", () => {
|
|
376
|
+
const orders = decl({ lexicon: "sql", entityType: "Postgres::Table", props: {} });
|
|
377
|
+
const ir = buildGraphIr(new Map<string, Declarable>([["orders", orders]]));
|
|
378
|
+
const withOutputs = applyLexiconOutputs({ ...ir, exports: [{ name: "public.orders", node: "orders", attr: "name" }] }, [
|
|
379
|
+
{
|
|
380
|
+
outputs: [
|
|
381
|
+
{ name: "public.orders", entity: "orders" },
|
|
382
|
+
{ name: "public.orders.buyer_id", entity: "orders", previously: ["public.orders.customer_id"] },
|
|
383
|
+
{ name: "other.thing", entity: "notANode" },
|
|
384
|
+
],
|
|
385
|
+
},
|
|
386
|
+
]);
|
|
387
|
+
expect(withOutputs.exports).toEqual([
|
|
388
|
+
{ name: "other.thing" },
|
|
389
|
+
{ name: "public.orders", node: "orders", attr: "name" },
|
|
390
|
+
{ name: "public.orders.buyer_id", node: "orders", previously: ["public.orders.customer_id"] },
|
|
391
|
+
]);
|
|
392
|
+
expect(applyLexiconOutputs(ir, [{ outputs: [] }])).toBe(ir);
|
|
393
|
+
});
|
|
394
|
+
});
|
package/src/graph-ir.ts
CHANGED
|
@@ -155,6 +155,43 @@ export interface LexiconGraphReference {
|
|
|
155
155
|
dangling?: boolean;
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
+
/**
|
|
159
|
+
* One output a lexicon reports for its own declarations (#3931): a name a
|
|
160
|
+
* workspace link may target, such as a sql table's `public.orders` or a
|
|
161
|
+
* column's `public.orders.customer_id`.
|
|
162
|
+
*/
|
|
163
|
+
export interface LexiconWorkspaceOutput {
|
|
164
|
+
/** The output's name, as a link's `output` names it. */
|
|
165
|
+
name: string;
|
|
166
|
+
/** The entity key that declares it. */
|
|
167
|
+
entity: string;
|
|
168
|
+
/**
|
|
169
|
+
* Names the output had before, from a rename hint in source (a sql
|
|
170
|
+
* `-- previously:` comment). A link to one of them is told where it went.
|
|
171
|
+
*/
|
|
172
|
+
previously?: string[];
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** One file the reader was handed but could not read without running it (#3931). */
|
|
176
|
+
export interface LexiconUnreadSource {
|
|
177
|
+
/** The file's path, as the caller names it in messages. */
|
|
178
|
+
file: string;
|
|
179
|
+
text: string;
|
|
180
|
+
/** Why it was not read: the fold's located reason. */
|
|
181
|
+
reason: string;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** What a lexicon's `workspaceOutputs` hook returns (#3931). */
|
|
185
|
+
export interface LexiconWorkspaceOutputs {
|
|
186
|
+
outputs: LexiconWorkspaceOutput[];
|
|
187
|
+
/**
|
|
188
|
+
* Why the list may be incomplete, one line each: a declaration whose name
|
|
189
|
+
* can't be known statically, a file of this lexicon's that did not fold, a
|
|
190
|
+
* view whose columns can't be derived. Empty or absent when it is complete.
|
|
191
|
+
*/
|
|
192
|
+
gaps?: string[];
|
|
193
|
+
}
|
|
194
|
+
|
|
158
195
|
/** What a lexicon's `graphEdges` hook returns (#3910). */
|
|
159
196
|
export interface LexiconGraphEdges {
|
|
160
197
|
references: LexiconGraphReference[];
|
|
@@ -304,6 +341,8 @@ export interface IRExport {
|
|
|
304
341
|
value?: unknown;
|
|
305
342
|
/** The deployed stack that publishes it, on an observed graph (#1279). */
|
|
306
343
|
stack?: string;
|
|
344
|
+
/** Names the output had before, from a rename hint a lexicon read (#3931). */
|
|
345
|
+
previously?: string[];
|
|
307
346
|
}
|
|
308
347
|
|
|
309
348
|
/** A cross-stack import this stack consumes — a `Parameter` fed from another
|
|
@@ -834,6 +873,32 @@ export function applyLexiconGraphEdges(ir: GraphIR, reports: readonly LexiconGra
|
|
|
834
873
|
return out;
|
|
835
874
|
}
|
|
836
875
|
|
|
876
|
+
/**
|
|
877
|
+
* Add the outputs the lexicons' `workspaceOutputs` hooks report (#3931) to a
|
|
878
|
+
* built IR's `exports`, each on the node that declares it. An output whose
|
|
879
|
+
* name is already exported (a hand-written `output()`) keeps the existing
|
|
880
|
+
* export. Pure. Returns `ir` itself when no report added anything.
|
|
881
|
+
*/
|
|
882
|
+
export function applyLexiconOutputs(ir: GraphIR, reports: readonly LexiconWorkspaceOutputs[]): GraphIR {
|
|
883
|
+
const nodeIds = new Set(ir.nodes.map((n) => n.id));
|
|
884
|
+
const names = new Set((ir.exports ?? []).map((e) => e.name));
|
|
885
|
+
const added: IRExport[] = [];
|
|
886
|
+
for (const report of reports) {
|
|
887
|
+
for (const o of report.outputs) {
|
|
888
|
+
if (names.has(o.name)) continue;
|
|
889
|
+
names.add(o.name);
|
|
890
|
+
added.push({
|
|
891
|
+
name: o.name,
|
|
892
|
+
...(nodeIds.has(o.entity) ? { node: o.entity } : {}),
|
|
893
|
+
...(o.previously && o.previously.length > 0 ? { previously: [...o.previously] } : {}),
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (added.length === 0) return ir;
|
|
898
|
+
const exports = [...(ir.exports ?? []), ...added].sort((a, b) => a.name.localeCompare(b.name));
|
|
899
|
+
return { ...ir, exports };
|
|
900
|
+
}
|
|
901
|
+
|
|
837
902
|
function danglingKey(d: IRDanglingRef): string {
|
|
838
903
|
return `${d.from}\0${d.path}\0${d.value}`;
|
|
839
904
|
}
|
package/src/lexicon.ts
CHANGED
|
@@ -23,7 +23,7 @@ import type { OwnershipChannel, OwnershipMarker } from "./ownership";
|
|
|
23
23
|
import type { LexiconConfigSchema } from "./lexicon-config";
|
|
24
24
|
import type { RuleMeta } from "./audit/catalog";
|
|
25
25
|
import type { ReferenceCatalog } from "./graph-refs";
|
|
26
|
-
import type { IREdge, LexiconGraphEdges } from "./graph-ir";
|
|
26
|
+
import type { IREdge, LexiconGraphEdges, LexiconUnreadSource, LexiconWorkspaceOutputs } from "./graph-ir";
|
|
27
27
|
import type { DescribeResourcesResult, UnobservedReason } from "./observation";
|
|
28
28
|
import type { DescribeIdentityOptions, DescribeIdentityResult } from "./identity";
|
|
29
29
|
import type { DeepNormalizationHooks, DeepObservationResult } from "./deep-observation";
|
|
@@ -1450,6 +1450,31 @@ export interface LexiconPlugin {
|
|
|
1450
1450
|
*/
|
|
1451
1451
|
graphEdges?(entities: ReadonlyMap<string, Declarable>): LexiconGraphEdges | undefined;
|
|
1452
1452
|
|
|
1453
|
+
/**
|
|
1454
|
+
* The workspace outputs this lexicon's declarations expose (#3931): names
|
|
1455
|
+
* another member's link may target, without anyone writing `output(...)`.
|
|
1456
|
+
* The sql lexicon answers each table and view as `<schema>.<table>` and
|
|
1457
|
+
* each column as `<schema>.<table>.<column>`.
|
|
1458
|
+
*
|
|
1459
|
+
* Called two ways, with the same answer expected from both:
|
|
1460
|
+
*
|
|
1461
|
+
* - by `chant workspace check`, offline, with the entities the member's
|
|
1462
|
+
* source folds to (`foldProject` under `sandbox`: no member code runs),
|
|
1463
|
+
* and `unread` listing the files that did not fold, with their text. A
|
|
1464
|
+
* file of this lexicon's among them is a gap: the list is then incomplete
|
|
1465
|
+
* and a link to a name not in it is unresolved rather than missing;
|
|
1466
|
+
* - by `chant graph --format ir`, with the built entities and no `unread`.
|
|
1467
|
+
* The outputs land on the IR's `exports`, which `chant workspace graph`
|
|
1468
|
+
* resolves links against.
|
|
1469
|
+
*
|
|
1470
|
+
* Names come from the declarations as written in source, never from an
|
|
1471
|
+
* environment's or profile's rendering, so they stay the same everywhere
|
|
1472
|
+
* the member deploys. Pure: no I/O. Return `undefined` when the project
|
|
1473
|
+
* declares nothing this lexicon reports. Omit for lexicons with no such
|
|
1474
|
+
* declarations.
|
|
1475
|
+
*/
|
|
1476
|
+
workspaceOutputs?(entities: ReadonlyMap<string, Declarable>, unread?: readonly LexiconUnreadSource[]): LexiconWorkspaceOutputs | undefined;
|
|
1477
|
+
|
|
1453
1478
|
// LSP
|
|
1454
1479
|
/** Provide completions for LSP */
|
|
1455
1480
|
completionProvider?(ctx: CompletionContext): CompletionItem[];
|
|
@@ -43,6 +43,9 @@ beforeEach(() => {
|
|
|
43
43
|
// A CI job's locator would land on every pending fact.
|
|
44
44
|
vi.stubEnv("GITHUB_ACTIONS", "");
|
|
45
45
|
vi.stubEnv("GITLAB_CI", "");
|
|
46
|
+
// A pull request's address would land on every pending fact too.
|
|
47
|
+
vi.stubEnv("GITHUB_REF_NAME", "");
|
|
48
|
+
vi.stubEnv("CI_MERGE_REQUEST_IID", "");
|
|
46
49
|
});
|
|
47
50
|
afterEach(() => vi.unstubAllEnvs());
|
|
48
51
|
|
|
@@ -17,31 +17,58 @@
|
|
|
17
17
|
* | WSP096 | a consumer states the same link twice |
|
|
18
18
|
* | WSP097 | an entry lists `outputs` for a kind that doesn't take them from the entry |
|
|
19
19
|
* | WSP098 | a link states a `protocol` and its kind is not `telemetry` (#2558) |
|
|
20
|
+
* | WSP099 | a kind's link reader could not read part of a member's files without guessing (#3932) |
|
|
21
|
+
*
|
|
22
|
+
* Links a kind's reader finds in a member's own files (#3932), such as a dbt
|
|
23
|
+
* project's sources, are checked by WSP093 to WSP095 like declared ones, and
|
|
24
|
+
* their findings name the member's file and line instead of the declaration.
|
|
20
25
|
*/
|
|
21
26
|
|
|
22
27
|
import type { WorkspaceCheck, WorkspaceCheckContext, WorkspaceDiagnostic } from "../checks";
|
|
23
28
|
import type { Member } from "../declaration";
|
|
24
29
|
import { sourceMemberHandles } from "../member-handles";
|
|
30
|
+
import { readKindLinks, type KindLinkRead } from "../kind-link-readers";
|
|
25
31
|
import { DEFAULT_LINK_KIND, LINK_KINDS, TELEMETRY_LINK_KIND, linkTargetProblem, resolveLinks, type AmbiguousRow, type LinkRow, type LinkTableRow } from "../links";
|
|
26
32
|
|
|
27
33
|
const tables = new WeakMap<WorkspaceCheckContext, LinkTableRow[]>();
|
|
34
|
+
const reads = new WeakMap<WorkspaceCheckContext, Map<string, KindLinkRead>>();
|
|
35
|
+
|
|
36
|
+
/** What each member's kind reader found (#3932), read once per check run. */
|
|
37
|
+
function kindReads(ctx: WorkspaceCheckContext): Map<string, KindLinkRead> {
|
|
38
|
+
let read = reads.get(ctx);
|
|
39
|
+
if (!read) {
|
|
40
|
+
read = readKindLinks(ctx.declaration, ctx.tree, ctx.kinds);
|
|
41
|
+
reads.set(ctx, read);
|
|
42
|
+
}
|
|
43
|
+
return read;
|
|
44
|
+
}
|
|
28
45
|
|
|
29
46
|
/** The link rows for a check run, resolved once and shared by every link check. */
|
|
30
47
|
export function linkTable(ctx: WorkspaceCheckContext): LinkTableRow[] {
|
|
31
48
|
let rows = tables.get(ctx);
|
|
32
49
|
if (!rows) {
|
|
33
|
-
rows = resolveLinks(ctx.declaration, sourceMemberHandles(ctx.declaration, ctx.tree, ctx.groups, ctx.kinds));
|
|
50
|
+
rows = resolveLinks(ctx.declaration, sourceMemberHandles(ctx.declaration, ctx.tree, ctx.groups, ctx.kinds, ctx.facts?.lexiconOutputs), undefined, kindReads(ctx));
|
|
34
51
|
tables.set(ctx, rows);
|
|
35
52
|
}
|
|
36
53
|
return rows;
|
|
37
54
|
}
|
|
38
55
|
|
|
39
|
-
|
|
56
|
+
/** The declared links, and the links a kind's reader found: the rows a member states. */
|
|
57
|
+
const statedRows = (ctx: WorkspaceCheckContext) => linkTable(ctx).filter((r): r is LinkRow => r.status !== "ambiguous" && r.origin !== "inferred:joinKey");
|
|
40
58
|
|
|
41
|
-
|
|
42
|
-
|
|
59
|
+
/** A finding at a declared link's field in the declaration, or at the file and line a reader's link came from. */
|
|
60
|
+
function linkFinding(check: WorkspaceCheck, ctx: WorkspaceCheckContext, row: LinkTableRow, field: string, message: string): WorkspaceDiagnostic {
|
|
61
|
+
const base = { checkId: check.id, severity: check.severity, message, entity: row.consumer };
|
|
62
|
+
if (row.origin === "kind-reader" && row.file !== undefined) {
|
|
63
|
+
const member = ctx.declaration.members.find((m) => m.name === row.consumer);
|
|
64
|
+
return { ...base, pointer: member?.pointer ?? "", treeFile: row.file, line: row.line ?? 1, column: 1 };
|
|
65
|
+
}
|
|
66
|
+
return { ...base, pointer: `${(row as LinkRow).pointer}/${field}` };
|
|
43
67
|
}
|
|
44
68
|
|
|
69
|
+
/** Where a reader's link came from, for its message; empty for a declared link. */
|
|
70
|
+
const statedAt = (row: LinkRow) => (row.origin === "kind-reader" ? ` (read by ${row.reader} from ${row.file}:${row.line})` : "");
|
|
71
|
+
|
|
45
72
|
const describeLink = (row: LinkRow) => `${row.consumer}'s link to ${row.producer} output ${row.output}`;
|
|
46
73
|
|
|
47
74
|
export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
@@ -87,9 +114,9 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
87
114
|
severity: "error",
|
|
88
115
|
configurable: true,
|
|
89
116
|
check(ctx) {
|
|
90
|
-
return
|
|
117
|
+
return statedRows(ctx)
|
|
91
118
|
.filter((r) => r.status === "missing")
|
|
92
|
-
.map((r) => linkFinding(this, r, "output", `${describeLink(r)} does not resolve: ${r.reason}`));
|
|
119
|
+
.map((r) => linkFinding(this, ctx, r, "output", `${describeLink(r)}${statedAt(r)} does not resolve: ${r.reason}`));
|
|
93
120
|
},
|
|
94
121
|
},
|
|
95
122
|
{
|
|
@@ -99,9 +126,9 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
99
126
|
severity: "info",
|
|
100
127
|
configurable: true,
|
|
101
128
|
check(ctx) {
|
|
102
|
-
return
|
|
129
|
+
return statedRows(ctx)
|
|
103
130
|
.filter((r) => r.status === "unresolved")
|
|
104
|
-
.map((r) => linkFinding(this, r, "output", `${describeLink(r)} is kept unresolved: ${r.reason}`));
|
|
131
|
+
.map((r) => linkFinding(this, ctx, r, "output", `${describeLink(r)}${statedAt(r)} is kept unresolved: ${r.reason}`));
|
|
105
132
|
},
|
|
106
133
|
},
|
|
107
134
|
{
|
|
@@ -115,6 +142,10 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
115
142
|
return linkTable(ctx)
|
|
116
143
|
.filter((r): r is AmbiguousRow => r.status === "ambiguous")
|
|
117
144
|
.map((r) => {
|
|
145
|
+
if (r.origin === "kind-reader") {
|
|
146
|
+
const choices = r.candidates.map((c) => c.producer).join(", ");
|
|
147
|
+
return linkFinding(this, ctx, r, "", `${r.consumer}'s link to ${r.input} (read by ${r.reader}) has no single producer: ${r.reason}${choices ? `. Candidates: ${choices}` : ""}`);
|
|
148
|
+
}
|
|
118
149
|
const choices = r.candidates.map((c) => `${c.producer} output ${c.output} (${c.label})`).join(", ");
|
|
119
150
|
const first = r.candidates[0];
|
|
120
151
|
return {
|
|
@@ -201,4 +232,22 @@ export const LINK_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
201
232
|
);
|
|
202
233
|
},
|
|
203
234
|
},
|
|
235
|
+
{
|
|
236
|
+
id: "WSP099",
|
|
237
|
+
name: "kind-reader-gap",
|
|
238
|
+
description:
|
|
239
|
+
"A kind's link reader read every link in a member's files that it could read without guessing (#3932). A value it would have to render, such as a Jinja template in a dbt source, is reported here with its file and line, and the links it would state are not checked.",
|
|
240
|
+
severity: "info",
|
|
241
|
+
configurable: true,
|
|
242
|
+
check(ctx) {
|
|
243
|
+
const out: WorkspaceDiagnostic[] = [];
|
|
244
|
+
for (const [member, read] of kindReads(ctx)) {
|
|
245
|
+
const pointer = ctx.declaration.members.find((m) => m.name === member)?.pointer ?? "";
|
|
246
|
+
for (const gap of read.gaps) {
|
|
247
|
+
out.push({ checkId: this.id, severity: this.severity, message: `${member} (read by ${read.reader}): ${gap.message}`, entity: member, pointer, treeFile: gap.file, line: gap.line, column: 1 });
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return out;
|
|
251
|
+
},
|
|
252
|
+
},
|
|
204
253
|
];
|
|
@@ -75,7 +75,7 @@ describe("the WSP catalog", () => {
|
|
|
75
75
|
expect(list.filter((id) => /^WSP(07|08|09|10)\d$/.test(id))).toEqual([
|
|
76
76
|
"WSP071", "WSP072", "WSP073",
|
|
77
77
|
"WSP081", "WSP082", "WSP083",
|
|
78
|
-
"WSP091", "WSP092", "WSP093", "WSP094", "WSP095", "WSP096", "WSP097", "WSP098",
|
|
78
|
+
"WSP091", "WSP092", "WSP093", "WSP094", "WSP095", "WSP096", "WSP097", "WSP098", "WSP099",
|
|
79
79
|
"WSP101", "WSP102", "WSP103", "WSP104", "WSP105", "WSP106",
|
|
80
80
|
]);
|
|
81
81
|
});
|
package/src/workspace/checks.ts
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* | WSP001 to WSP011 | the declaration and member kinds (this file) |
|
|
25
25
|
* | WSP071, WSP072 | member ledgers |
|
|
26
26
|
* | WSP081 to WSP083 | recorded pipelines |
|
|
27
|
-
* | WSP091 to
|
|
27
|
+
* | WSP091 to WSP099 | member links (#2539, #2558), and those a kind's reader finds in a member's files (#3932) |
|
|
28
28
|
* | WSP101 to WSP106 | generated files |
|
|
29
29
|
* | WSP111 to WSP114 | records read with `--kind` (#2549) |
|
|
30
30
|
* | WSP115 | the record kinds the declaration names (#2680) |
|
|
@@ -59,6 +59,7 @@ import { BOX_CHECKS } from "./checks/boxes";
|
|
|
59
59
|
import { DIAGRAM_CHECKS } from "./checks/diagrams";
|
|
60
60
|
import { loadDeclaredKinds, type DeclaredKind } from "./declared-kinds";
|
|
61
61
|
import { resolveBoxIntents, type ResolvedBoxIntent } from "./box-intent";
|
|
62
|
+
import { gatherLexiconOutputs, type MemberLexiconOutputs } from "./lexicon-outputs";
|
|
62
63
|
|
|
63
64
|
/**
|
|
64
65
|
* What the checks beyond the declaration read, gathered from the checkout
|
|
@@ -80,6 +81,8 @@ export interface WorkspaceFacts {
|
|
|
80
81
|
live?: LiveLinkFacts;
|
|
81
82
|
/** The decision record each box's intent names, read from the working tree (#2850). */
|
|
82
83
|
boxIntents?: readonly ResolvedBoxIntent[];
|
|
84
|
+
/** The outputs linked members' lexicons report about their declarations, read by folding their source (#3931). */
|
|
85
|
+
lexiconOutputs?: ReadonlyMap<string, MemberLexiconOutputs>;
|
|
83
86
|
}
|
|
84
87
|
|
|
85
88
|
/** What every declaration check reads. */
|
|
@@ -355,7 +358,7 @@ export const WORKSPACE_CHECKS: readonly WorkspaceCheck[] = [
|
|
|
355
358
|
...LEDGER_CHECKS,
|
|
356
359
|
// Recorded pipelines (#2542).
|
|
357
360
|
...PIPELINE_CHECKS,
|
|
358
|
-
// Member links (#2539, #2558), WSP091 to
|
|
361
|
+
// Member links (#2539, #2558, #3932), WSP091 to WSP099.
|
|
359
362
|
...LINK_CHECKS,
|
|
360
363
|
// Generated files (#2541).
|
|
361
364
|
...GENERATED_CHECKS,
|
|
@@ -521,7 +524,11 @@ export async function runDeclarationChecks(
|
|
|
521
524
|
const declaredKinds = options.gather === false ? undefined : await loadDeclaredKinds(declaration, tree, root, { load: !options.tree, acceptance: !options.tree });
|
|
522
525
|
// A box's intent is read from the working tree's records, so not under --at (#2850).
|
|
523
526
|
const boxIntents = declaredKinds && !options.tree ? await resolveBoxIntents(declaration, root, declaredKinds) : undefined;
|
|
527
|
+
// A linked member's lexicons name its tables and columns as outputs (#3931); reading them folds its source, never runs it.
|
|
528
|
+
const lexiconOutputs =
|
|
529
|
+
options.gather === false ? undefined : await gatherLexiconOutputs(root, declaration, options.tree ? workingTree(root) : tree, groups, registry, { atRevision: options.tree !== undefined });
|
|
524
530
|
const facts: WorkspaceFacts = {
|
|
531
|
+
...(lexiconOutputs && lexiconOutputs.size > 0 ? { lexiconOutputs } : {}),
|
|
525
532
|
...gathered,
|
|
526
533
|
...(options.records ? { records: options.records } : {}),
|
|
527
534
|
...(declaredKinds ? { declaredKinds } : {}),
|
|
@@ -33,6 +33,7 @@ import type { FactoryView, ListingView } from "./box-factory";
|
|
|
33
33
|
import type { Declaration } from "./declaration";
|
|
34
34
|
import type { KindRegistry } from "./kinds";
|
|
35
35
|
import { graphLinks, type LinkTableRow } from "./links";
|
|
36
|
+
import type { KindLinkRead } from "./kind-link-readers";
|
|
36
37
|
import type { RecordLinkRow } from "./record-assets";
|
|
37
38
|
import type { ReasonCode } from "./reason-codes";
|
|
38
39
|
import { GRAPH_IR_VERSION, type GraphIR, type IRExport, type IRGroups, type IRImport, type IREdge, type IRNode } from "../graph-ir";
|
|
@@ -302,12 +303,13 @@ function sorted(rec: Record<string, string[]>): Record<string, string[]> {
|
|
|
302
303
|
* Compose the members' IRs into the workspace document. Members keep the
|
|
303
304
|
* order they are given in. With `links`, the `links` section is filled from
|
|
304
305
|
* the declaration; `kinds` lets a member that isn't composed but whose kind
|
|
305
|
-
* lists its outputs (`other`, kinds from a package) be a link target
|
|
306
|
+
* lists its outputs (`other`, kinds from a package) be a link target, and
|
|
307
|
+
* `read` holds the links each member's kind reader found in its files (#3932).
|
|
306
308
|
*/
|
|
307
309
|
export function composeWorkspaceGraph(
|
|
308
310
|
workspace: { name: string; root: string },
|
|
309
311
|
inputs: ComposeInput[],
|
|
310
|
-
links?: { declaration: Declaration; kinds?: KindRegistry },
|
|
312
|
+
links?: { declaration: Declaration; kinds?: KindRegistry; read?: ReadonlyMap<string, KindLinkRead> },
|
|
311
313
|
): WorkspaceGraph {
|
|
312
314
|
const nodes: ComposedNode[] = [];
|
|
313
315
|
const edges: ComposedEdge[] = [];
|
|
@@ -370,7 +372,7 @@ export function composeWorkspaceGraph(
|
|
|
370
372
|
exports,
|
|
371
373
|
imports,
|
|
372
374
|
links: links
|
|
373
|
-
? graphLinks(links.declaration, { composed: inputs.filter((i) => i.ir).map((i) => i.member.name), collectors, exports, imports }, links.kinds)
|
|
375
|
+
? graphLinks(links.declaration, { composed: inputs.filter((i) => i.ir).map((i) => i.member.name), collectors, exports, imports }, links.kinds, links.read)
|
|
374
376
|
: [],
|
|
375
377
|
collectors,
|
|
376
378
|
records: [],
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `dbt` kind end to end (#3932): the sql lexicon pinned as a package,
|
|
3
|
+
* its kinds file found through node_modules, and this checkout's chant CLI
|
|
4
|
+
* running `workspace check` and `workspace graph`. The producer is a real
|
|
5
|
+
* sql member, a chant project that declares `orders`, whose tables and
|
|
6
|
+
* columns are outputs through the sql lexicon's hook (#3931).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { spawnSync } from "node:child_process";
|
|
10
|
+
import { readFileSync, symlinkSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { pathToFileURL } from "node:url";
|
|
13
|
+
import { afterAll, describe, expect, test } from "vitest";
|
|
14
|
+
import { cleanScratch, contract, declaration, REPO, repo } from "./__fixtures__/contract-repo";
|
|
15
|
+
import schema from "./graph.schema.json";
|
|
16
|
+
|
|
17
|
+
afterAll(cleanScratch);
|
|
18
|
+
|
|
19
|
+
const TIMEOUT = 120_000;
|
|
20
|
+
const LEXICON = "@intentius/chant-lexicon-sql";
|
|
21
|
+
const VERSION = (JSON.parse(readFileSync(join(REPO, "lexicons", "sql", "package.json"), "utf-8")) as { version: string }).version;
|
|
22
|
+
|
|
23
|
+
function chant(cwd: string, ...argv: string[]) {
|
|
24
|
+
return spawnSync(process.execPath, ["--import", pathToFileURL(join(REPO, "node_modules/tsx/dist/loader.mjs")).href, join(REPO, "packages/core/src/cli/main.ts"), ...argv], {
|
|
25
|
+
cwd,
|
|
26
|
+
encoding: "utf-8",
|
|
27
|
+
timeout: TIMEOUT,
|
|
28
|
+
env: { ...process.env, NO_COLOR: "1", TSX_DISABLE_CACHE: "1" },
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const SQL_CONFIG = `import type { ChantConfig } from "@intentius/chant";\nexport default { lexicons: ["sql"], sql: { dialect: "postgres" } } satisfies ChantConfig;\n`;
|
|
33
|
+
|
|
34
|
+
const orders = (columns: string[]) =>
|
|
35
|
+
[
|
|
36
|
+
`import { table } from "@intentius/chant-lexicon-sql/postgres";`,
|
|
37
|
+
``,
|
|
38
|
+
`export const orders = table\``,
|
|
39
|
+
` CREATE TABLE orders (`,
|
|
40
|
+
...columns.map((c, i) => ` ${c}${i < columns.length - 1 ? "," : ""}`),
|
|
41
|
+
` )\`;`,
|
|
42
|
+
``,
|
|
43
|
+
].join("\n");
|
|
44
|
+
|
|
45
|
+
function workspace(columns: string[]): string {
|
|
46
|
+
const root = repo({
|
|
47
|
+
"chant.workspace.json": declaration(
|
|
48
|
+
[
|
|
49
|
+
{ name: "db", dir: "db", kind: "chant" },
|
|
50
|
+
{ name: "analytics", dir: "analytics", kind: "dbt", fields: { sqlMember: "db" } },
|
|
51
|
+
],
|
|
52
|
+
{ pins: [{ package: LEXICON, version: VERSION }] },
|
|
53
|
+
),
|
|
54
|
+
"db/chant.config.ts": SQL_CONFIG,
|
|
55
|
+
"db/src/orders.ts": orders(columns),
|
|
56
|
+
"analytics/dbt_project.yml": "name: analytics\nversion: '1.0'\nprofile: analytics\n",
|
|
57
|
+
"analytics/models/staging/sources.yml": "version: 2\n\nsources:\n - name: shop\n schema: public\n tables:\n - name: orders\n columns:\n - name: id\n - name: customer_id\n",
|
|
58
|
+
".gitignore": "node_modules\n",
|
|
59
|
+
});
|
|
60
|
+
symlinkSync(join(REPO, "node_modules"), join(root, "node_modules"), "dir");
|
|
61
|
+
return root;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
describe("a dbt member through the pinned sql lexicon (#3932)", () => {
|
|
65
|
+
test(
|
|
66
|
+
"check passes while the sql member has public.orders.customer_id, and fails naming the dbt source file once it is dropped",
|
|
67
|
+
() => {
|
|
68
|
+
const ok = chant(workspace(["id bigint PRIMARY KEY", "customer_id bigint NOT NULL"]), "workspace", "check");
|
|
69
|
+
expect(ok.status, ok.stdout + ok.stderr).toBe(0);
|
|
70
|
+
|
|
71
|
+
const dropped = chant(workspace(["id bigint PRIMARY KEY"]), "workspace", "check");
|
|
72
|
+
expect(dropped.status).toBe(1);
|
|
73
|
+
const text = dropped.stdout + dropped.stderr;
|
|
74
|
+
expect(text).toContain("analytics/models/staging/sources.yml:10");
|
|
75
|
+
expect(text).toContain("WSP093");
|
|
76
|
+
expect(text).toMatch(/link to db output public\.orders\.customer_id .*does not resolve/);
|
|
77
|
+
},
|
|
78
|
+
TIMEOUT,
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
test(
|
|
82
|
+
"workspace graph lists the dbt member's links with their file and line",
|
|
83
|
+
() => {
|
|
84
|
+
const run = chant(workspace(["id bigint PRIMARY KEY"]), "workspace", "graph");
|
|
85
|
+
const doc = JSON.parse(run.stdout) as { links: { origin: string; consumer: string; producer: string; output: string; status: string; file?: string; line?: number }[] };
|
|
86
|
+
contract(schema).expectValid(doc);
|
|
87
|
+
expect(doc.links.filter((l) => l.origin === "kind-reader").map((l) => `${l.consumer} -> ${l.producer}.${l.output} ${l.status} ${l.file}:${l.line}`)).toEqual([
|
|
88
|
+
"analytics -> db.public.orders resolved analytics/models/staging/sources.yml:7",
|
|
89
|
+
"analytics -> db.public.orders.id resolved analytics/models/staging/sources.yml:9",
|
|
90
|
+
"analytics -> db.public.orders.customer_id missing analytics/models/staging/sources.yml:10",
|
|
91
|
+
]);
|
|
92
|
+
},
|
|
93
|
+
TIMEOUT,
|
|
94
|
+
);
|
|
95
|
+
});
|