@untangle-code/context-artifacts 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +247 -2
- package/THIRD_PARTY_LICENSES.md +24 -0
- package/dist/adapters/asyncapi.d.ts +20 -0
- package/dist/adapters/asyncapi.d.ts.map +1 -0
- package/dist/adapters/asyncapi.js +95 -0
- package/dist/adapters/asyncapi.js.map +1 -0
- package/dist/adapters/docker.d.ts +43 -0
- package/dist/adapters/docker.d.ts.map +1 -0
- package/dist/adapters/docker.js +97 -0
- package/dist/adapters/docker.js.map +1 -0
- package/dist/adapters/kubernetes.d.ts +22 -0
- package/dist/adapters/kubernetes.d.ts.map +1 -0
- package/dist/adapters/kubernetes.js +56 -0
- package/dist/adapters/kubernetes.js.map +1 -0
- package/dist/adapters/markdown.d.ts +35 -0
- package/dist/adapters/markdown.d.ts.map +1 -0
- package/dist/adapters/markdown.js +36 -0
- package/dist/adapters/markdown.js.map +1 -0
- package/dist/adapters/openapi.d.ts +21 -0
- package/dist/adapters/openapi.d.ts.map +1 -0
- package/dist/adapters/openapi.js +101 -0
- package/dist/adapters/openapi.js.map +1 -0
- package/dist/adapters/protobuf.d.ts +34 -0
- package/dist/adapters/protobuf.d.ts.map +1 -0
- package/dist/adapters/protobuf.js +221 -0
- package/dist/adapters/protobuf.js.map +1 -0
- package/dist/adapters/shared.d.ts +65 -0
- package/dist/adapters/shared.d.ts.map +1 -0
- package/dist/adapters/shared.js +95 -0
- package/dist/adapters/shared.js.map +1 -0
- package/dist/adapters/sql.d.ts +104 -0
- package/dist/adapters/sql.d.ts.map +1 -0
- package/dist/adapters/sql.js +427 -0
- package/dist/adapters/sql.js.map +1 -0
- package/dist/adapters/terraform.d.ts +40 -0
- package/dist/adapters/terraform.d.ts.map +1 -0
- package/dist/adapters/terraform.js +186 -0
- package/dist/adapters/terraform.js.map +1 -0
- package/dist/adapters/text.d.ts +24 -0
- package/dist/adapters/text.d.ts.map +1 -0
- package/dist/adapters/text.js +31 -0
- package/dist/adapters/text.js.map +1 -0
- package/dist/config-validation.d.ts +18 -0
- package/dist/config-validation.d.ts.map +1 -0
- package/dist/config-validation.js +30 -0
- package/dist/config-validation.js.map +1 -0
- package/dist/coverage.d.ts +114 -0
- package/dist/coverage.d.ts.map +1 -0
- package/dist/coverage.js +0 -0
- package/dist/coverage.js.map +1 -0
- package/dist/ids.d.ts +12 -0
- package/dist/ids.d.ts.map +1 -0
- package/dist/ids.js +42 -0
- package/dist/ids.js.map +1 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/registry.d.ts +51 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +86 -0
- package/dist/registry.js.map +1 -0
- package/dist/staleness.d.ts +44 -0
- package/dist/staleness.d.ts.map +1 -0
- package/dist/staleness.js +35 -0
- package/dist/staleness.js.map +1 -0
- package/dist/text/delimiter-balance.d.ts +95 -0
- package/dist/text/delimiter-balance.d.ts.map +1 -0
- package/dist/text/delimiter-balance.js +209 -0
- package/dist/text/delimiter-balance.js.map +1 -0
- package/dist/text/line-index.d.ts +61 -0
- package/dist/text/line-index.d.ts.map +1 -0
- package/dist/text/line-index.js +75 -0
- package/dist/text/line-index.js.map +1 -0
- package/dist/text/yaml-lite.d.ts +73 -0
- package/dist/text/yaml-lite.d.ts.map +1 -0
- package/dist/text/yaml-lite.js +486 -0
- package/dist/text/yaml-lite.js.map +1 -0
- package/package.json +61 -3
- package/src/adapters/__golden__/adr-0001-sections.json +68 -0
- package/src/adapters/__golden__/app-compose-services.json +24 -0
- package/src/adapters/__golden__/app-image-stages.json +24 -0
- package/src/adapters/__golden__/infra-main-sections.json +46 -0
- package/src/adapters/__golden__/orders-events-operations.json +24 -0
- package/src/adapters/__golden__/orders-proto-sections.json +57 -0
- package/src/adapters/__golden__/orders-schema-statements.json +68 -0
- package/src/adapters/__golden__/users-api-operations.json +46 -0
- package/src/adapters/__golden__/web-manifests-resources.json +24 -0
- package/src/adapters/asyncapi.ts +147 -0
- package/src/adapters/docker.ts +170 -0
- package/src/adapters/kubernetes.ts +103 -0
- package/src/adapters/markdown.ts +88 -0
- package/src/adapters/openapi.ts +156 -0
- package/src/adapters/protobuf.ts +286 -0
- package/src/adapters/shared.ts +151 -0
- package/src/adapters/sql.ts +555 -0
- package/src/adapters/terraform.ts +261 -0
- package/src/adapters/text.ts +72 -0
- package/src/config-validation.ts +41 -0
- package/src/coverage.ts +336 -0
- package/src/ids.ts +57 -0
- package/src/index.ts +105 -0
- package/src/registry.ts +155 -0
- package/src/staleness.ts +69 -0
- package/src/text/delimiter-balance.ts +295 -0
- package/src/text/line-index.ts +108 -0
- package/src/text/yaml-lite.ts +565 -0
package/src/coverage.ts
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skip ledger (docs/BUILD_PLAN.md §12 E4: "KEIN stilles Ueberspringen von
|
|
3
|
+
* Inhalt — Skip-Ledger mit Grund").
|
|
4
|
+
*
|
|
5
|
+
* `ArtifactChunk`/`ArtifactAdapter` (architecture.md §20, frozen in
|
|
6
|
+
* `@untangle-code/domain`) carry no ledger field of their own — chunk
|
|
7
|
+
* ranges are the only signal a caller has about what was turned into a
|
|
8
|
+
* searchable chunk. `buildArtifactSkipLedger()` is this package's answer to
|
|
9
|
+
* E4 within that contract: given a document and the chunks an adapter
|
|
10
|
+
* produced for it, it finds every byte range NOT covered by any chunk and
|
|
11
|
+
* returns it as a labeled entry, so a caller (indexer/CLI) can log or
|
|
12
|
+
* surface "artifact X, bytes N..M were not turned into a chunk, because
|
|
13
|
+
* Y" — visibly, with a reason, never silently.
|
|
14
|
+
*
|
|
15
|
+
* A gap is not automatically a defect: whitespace between two SQL
|
|
16
|
+
* statements, or a comment line between two Terraform blocks, is expected
|
|
17
|
+
* and labeled `"whitespace_or_comment"`. A gap that still contains
|
|
18
|
+
* non-trivial content is labeled `"unclassified_content"` — a signal that
|
|
19
|
+
* an adapter's section boundaries missed something real.
|
|
20
|
+
*
|
|
21
|
+
* Byte coverage alone is BLIND to one real defect class (found in review of
|
|
22
|
+
* WP-81): an adapter can produce chunks whose union of byte ranges looks
|
|
23
|
+
* complete while a real section's identity was still lost — e.g. two SQL
|
|
24
|
+
* statements sharing one physical line, where a broken span computation
|
|
25
|
+
* attributed BOTH statements' bytes to the first statement's chunk and then
|
|
26
|
+
* emitted a second, byte-EMPTY chunk for the one that lost its own range.
|
|
27
|
+
* That second chunk carries no gap (there are no uncovered bytes left to
|
|
28
|
+
* report) yet is definitive proof that the adapter detected a section
|
|
29
|
+
* boundary and then failed to capture anything for it — a section was
|
|
30
|
+
* found, not classified.
|
|
31
|
+
*
|
|
32
|
+
* This module cannot re-parse an arbitrary format to notice that kind of
|
|
33
|
+
* loss (coverage.ts is intentionally format-agnostic — it only ever sees
|
|
34
|
+
* `ArtifactChunk[]`, never an adapter's own grammar). What it CAN check,
|
|
35
|
+
* generically and with zero false positives, is the structural invariant
|
|
36
|
+
* every legitimate chunk must satisfy: a non-empty, non-inverted byte range
|
|
37
|
+
* (`range.endByte > range.startByte`), and — when two chunks claim the
|
|
38
|
+
* EXACT same byte range — agreement on what that range even is (same
|
|
39
|
+
* `kind`+`title`; protobuf's intentionally nested service/RPC overlap never
|
|
40
|
+
* trips this, since an RPC's range is a strict sub-range of its service's,
|
|
41
|
+
* never byte-identical to it). Either violation is reported as
|
|
42
|
+
* `"semantic_loss"`: content IS covered by some chunk, but a section the
|
|
43
|
+
* adapter itself tried to carve out was not actually captured or was
|
|
44
|
+
* captured twice with conflicting identities.
|
|
45
|
+
*
|
|
46
|
+
* WP-118 fand die dritte Blindstelle derselben Familie: nichts fehlt,
|
|
47
|
+
* nichts ist leer, nichts überlappt — aber ein Fragment ist MITTEN DURCH
|
|
48
|
+
* ein Konstrukt geschnitten (die SQL-Zerlegung zerriss ein Statement an
|
|
49
|
+
* einem Semikolon INNERHALB einer Zeichenkette). Das kann dieses Modul
|
|
50
|
+
* nicht selbst erkennen, weil ihm dafür die Konventionen des Formats
|
|
51
|
+
* fehlen. Es nimmt sie deshalb als Rückruf entgegen:
|
|
52
|
+
* `buildArtifactSkipLedger(document, chunks, { checkPlausibility })`
|
|
53
|
+
* meldet jedes so beanstandete Fragment als `"implausible_fragment"`.
|
|
54
|
+
* Die Adapter reichen ihre eigene Prüfung über
|
|
55
|
+
* `adapters/shared.ts::buildChunksReportingSkips` herein, damit auch das
|
|
56
|
+
* keine Bringschuld des Aufrufers wird (E4).
|
|
57
|
+
*/
|
|
58
|
+
import type { ArtifactChunk, ArtifactDocument, SourceRange } from "@untangle-code/domain";
|
|
59
|
+
|
|
60
|
+
import { buildLineIndex, lineNumberForByte, type LineIndex } from "./text/line-index.js";
|
|
61
|
+
|
|
62
|
+
export type ArtifactSkipReason =
|
|
63
|
+
"whitespace_or_comment" | "unclassified_content" | "semantic_loss" | "implausible_fragment";
|
|
64
|
+
|
|
65
|
+
export interface ArtifactSkipLedgerEntry {
|
|
66
|
+
artifactName: string;
|
|
67
|
+
/** "/"-normalized document path. */
|
|
68
|
+
path: string;
|
|
69
|
+
range: SourceRange;
|
|
70
|
+
reason: ArtifactSkipReason;
|
|
71
|
+
/** First 200 chars of the skipped content, for a human reviewing the ledger. */
|
|
72
|
+
contentPreview: string;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Callback invoked with the actionable (non-`"whitespace_or_comment"`)
|
|
77
|
+
* entries of a document's skip ledger, from the NORMAL `chunk()` path (see
|
|
78
|
+
* `adapters/shared.ts`'s `buildChunksReportingSkips` — every built-in
|
|
79
|
+
* adapter's `chunk()` goes through it, so this fires without a caller ever
|
|
80
|
+
* having to call `buildArtifactSkipLedger` separately, BUILD_PLAN §12 E4).
|
|
81
|
+
*/
|
|
82
|
+
export type SkipLedgerReporter = (
|
|
83
|
+
entries: readonly ArtifactSkipLedgerEntry[],
|
|
84
|
+
document: ArtifactDocument,
|
|
85
|
+
) => void;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Default `SkipLedgerReporter`: one JSON-lines-shaped warning per entry on
|
|
89
|
+
* `stderr`. Deliberately dependency-free (no `@untangle-code/observability`
|
|
90
|
+
* `Logger` — that requires an injected `Clock` and isn't available to a
|
|
91
|
+
* bare `create*Adapter()` factory call) so a loss is visible out of the box;
|
|
92
|
+
* a caller with a real structured logger passes its own `onSkipLedgerGap`
|
|
93
|
+
* to any `create*Adapter()`/`createArtifactAdapterRegistry()` instead.
|
|
94
|
+
*/
|
|
95
|
+
export const defaultSkipLedgerReporter: SkipLedgerReporter = (entries, document) => {
|
|
96
|
+
for (const entry of entries) {
|
|
97
|
+
process.stderr.write(
|
|
98
|
+
`${JSON.stringify({
|
|
99
|
+
level: "warn",
|
|
100
|
+
event: "context_artifact_skip_ledger_gap",
|
|
101
|
+
artifactName: document.artifactName,
|
|
102
|
+
path: entry.path,
|
|
103
|
+
range: entry.range,
|
|
104
|
+
reason: entry.reason,
|
|
105
|
+
contentPreview: entry.contentPreview,
|
|
106
|
+
})}\n`,
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
const PREVIEW_LENGTH = 200;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Strips full-line `#`/`--`/`//` comments and any `/* *\/` block comment
|
|
115
|
+
* (which may itself span several lines — stripped whole, before the
|
|
116
|
+
* per-line check, precisely so a gap that is JUST a multi-line block
|
|
117
|
+
* comment — e.g. between two Terraform resources — is recognized as
|
|
118
|
+
* benign) so a comment-only gap is classified `"whitespace_or_comment"`,
|
|
119
|
+
* never `"unclassified_content"`.
|
|
120
|
+
*/
|
|
121
|
+
function isWhitespaceOrCommentOnly(text: string): boolean {
|
|
122
|
+
const withoutBlockComments = text.replace(/\/\*[\s\S]*?\*\//g, "");
|
|
123
|
+
return withoutBlockComments.split("\n").every((line) => /^\s*(#.*|--.*|\/\/.*)?\s*$/.test(line));
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function classify(content: string): ArtifactSkipReason {
|
|
127
|
+
return isWhitespaceOrCommentOnly(content) ? "whitespace_or_comment" : "unclassified_content";
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Prüft EIN fertiges Fragment auf Sinnhaftigkeit und liefert bei einem
|
|
132
|
+
* Befund den Begründungstext, sonst `undefined` (WP-118).
|
|
133
|
+
*
|
|
134
|
+
* Byte-Abdeckung beantwortet nur „fehlt etwas?"; sie ist blind dafür, ob
|
|
135
|
+
* ein Fragment an einer sinnvollen Stelle endet. Genau diese zweite Frage
|
|
136
|
+
* kann `coverage.ts` nicht selbst beantworten — sie braucht die
|
|
137
|
+
* Konventionen des jeweiligen Formats. Der zuständige Adapter reicht sie
|
|
138
|
+
* deshalb als Funktion herein (`adapters/sql.ts`, `adapters/protobuf.ts`,
|
|
139
|
+
* `adapters/terraform.ts` nutzen dafür `text/delimiter-balance.ts`),
|
|
140
|
+
* während dieses Modul format-agnostisch bleibt.
|
|
141
|
+
*/
|
|
142
|
+
export type ChunkPlausibilityCheck = (chunk: ArtifactChunk) => string | undefined;
|
|
143
|
+
|
|
144
|
+
export interface ArtifactSkipLedgerOptions {
|
|
145
|
+
/** Siehe {@link ChunkPlausibilityCheck}. Ohne diese Option verhält sich der Ledger wie vor WP-118. */
|
|
146
|
+
readonly checkPlausibility?: ChunkPlausibilityCheck;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Finds every byte range of `document.content` not covered by any of
|
|
151
|
+
* `chunks`' ranges and returns one labeled ledger entry per gap.
|
|
152
|
+
*
|
|
153
|
+
* Overlapping/out-of-order chunk ranges (e.g. Protobuf's intentionally
|
|
154
|
+
* overlapping service/RPC chunks, see the protobuf adapter's
|
|
155
|
+
* "Entscheidungen" entry) are handled correctly: ranges are merged before
|
|
156
|
+
* gaps are computed, so an RPC chunk nested inside its service chunk never
|
|
157
|
+
* produces a spurious gap either side of it.
|
|
158
|
+
*/
|
|
159
|
+
export function buildArtifactSkipLedger(
|
|
160
|
+
document: ArtifactDocument,
|
|
161
|
+
chunks: readonly ArtifactChunk[],
|
|
162
|
+
options: ArtifactSkipLedgerOptions = {},
|
|
163
|
+
): ArtifactSkipLedgerEntry[] {
|
|
164
|
+
const index = buildLineIndex(document.content);
|
|
165
|
+
const buffer = index.buffer;
|
|
166
|
+
const totalBytes = buffer.length;
|
|
167
|
+
|
|
168
|
+
const semanticLossEntries = findSemanticLossEntries(document, index, chunks);
|
|
169
|
+
const plausibilityEntries = findImplausibleFragmentEntries(
|
|
170
|
+
document,
|
|
171
|
+
chunks,
|
|
172
|
+
options.checkPlausibility,
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
if (totalBytes === 0) {
|
|
176
|
+
return [...semanticLossEntries, ...plausibilityEntries];
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const sortedSpans = chunks
|
|
180
|
+
.map((chunk): [number, number] => [chunk.range.startByte, chunk.range.endByte])
|
|
181
|
+
.filter(([start, end]) => end > start)
|
|
182
|
+
.sort((a, b) => a[0] - b[0]);
|
|
183
|
+
|
|
184
|
+
const merged: [number, number][] = [];
|
|
185
|
+
for (const [start, end] of sortedSpans) {
|
|
186
|
+
const last = merged.at(-1);
|
|
187
|
+
if (last !== undefined && start <= last[1]) {
|
|
188
|
+
last[1] = Math.max(last[1], end);
|
|
189
|
+
} else {
|
|
190
|
+
merged.push([start, end]);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const gapEntries: ArtifactSkipLedgerEntry[] = [];
|
|
195
|
+
let cursor = 0;
|
|
196
|
+
for (const [start, end] of merged) {
|
|
197
|
+
if (start > cursor) {
|
|
198
|
+
gapEntries.push(buildEntry(document, index, cursor, start));
|
|
199
|
+
}
|
|
200
|
+
cursor = Math.max(cursor, end);
|
|
201
|
+
}
|
|
202
|
+
if (cursor < totalBytes) {
|
|
203
|
+
gapEntries.push(buildEntry(document, index, cursor, totalBytes));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Byte gaps first (in document order), semantic-loss findings after — both
|
|
207
|
+
// are real E4-relevant findings, but a gap is "content with no chunk at
|
|
208
|
+
// all" while semantic loss is "a chunk exists but its own identity is
|
|
209
|
+
// broken", a useful distinction for a human scanning the ledger.
|
|
210
|
+
return [...gapEntries, ...semanticLossEntries, ...plausibilityEntries];
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Baut je Chunk, den `check` als unplausibel meldet, einen Ledger-Eintrag
|
|
215
|
+
* mit Grund `"implausible_fragment"`. Ohne `check` ist das Ergebnis leer —
|
|
216
|
+
* ein Aufrufer, der `buildArtifactSkipLedger` direkt und ohne Optionen
|
|
217
|
+
* benutzt, sieht exakt dasselbe Verhalten wie vor WP-118.
|
|
218
|
+
*/
|
|
219
|
+
function findImplausibleFragmentEntries(
|
|
220
|
+
document: ArtifactDocument,
|
|
221
|
+
chunks: readonly ArtifactChunk[],
|
|
222
|
+
check: ChunkPlausibilityCheck | undefined,
|
|
223
|
+
): ArtifactSkipLedgerEntry[] {
|
|
224
|
+
if (check === undefined) return [];
|
|
225
|
+
const entries: ArtifactSkipLedgerEntry[] = [];
|
|
226
|
+
for (const chunk of chunks) {
|
|
227
|
+
const detail = check(chunk);
|
|
228
|
+
if (detail === undefined) continue;
|
|
229
|
+
entries.push({
|
|
230
|
+
artifactName: document.artifactName,
|
|
231
|
+
path: document.path,
|
|
232
|
+
range: chunk.range,
|
|
233
|
+
reason: "implausible_fragment",
|
|
234
|
+
contentPreview: `${detail} (kind="${chunk.kind}" title=${
|
|
235
|
+
chunk.title === undefined ? "(none)" : JSON.stringify(chunk.title)
|
|
236
|
+
})`.slice(0, PREVIEW_LENGTH),
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
return entries;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function buildEntry(
|
|
243
|
+
document: ArtifactDocument,
|
|
244
|
+
index: LineIndex,
|
|
245
|
+
startByte: number,
|
|
246
|
+
endByte: number,
|
|
247
|
+
): ArtifactSkipLedgerEntry {
|
|
248
|
+
const gapContent = index.buffer.toString("utf8", startByte, endByte);
|
|
249
|
+
const startLine = lineNumberForByte(index, startByte);
|
|
250
|
+
const endLine = endByte > startByte ? lineNumberForByte(index, endByte - 1) : startLine;
|
|
251
|
+
return {
|
|
252
|
+
artifactName: document.artifactName,
|
|
253
|
+
path: document.path,
|
|
254
|
+
range: { startLine, endLine, startByte, endByte },
|
|
255
|
+
reason: classify(gapContent),
|
|
256
|
+
contentPreview: gapContent.slice(0, PREVIEW_LENGTH),
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* See the module doc's "Byte coverage alone is BLIND..." note: finds chunks
|
|
262
|
+
* that violate the structural invariant every legitimate `ArtifactChunk`
|
|
263
|
+
* must satisfy, independent of any particular format's grammar.
|
|
264
|
+
*
|
|
265
|
+
* 1. A chunk whose own range is empty or inverted (`endByte <= startByte`)
|
|
266
|
+
* — an adapter tried to carve out a section and captured zero bytes for
|
|
267
|
+
* it. `chunk.content`/`contentHash` are trusted as the preview source
|
|
268
|
+
* here (not re-sliced from `document`) precisely because the chunk's own
|
|
269
|
+
* declared range cannot be trusted to slice anything meaningful.
|
|
270
|
+
* 2. Two (or more) chunks that claim the EXACT same byte range but disagree
|
|
271
|
+
* on `kind`/`title` — the same bytes were classified twice, differently.
|
|
272
|
+
* Deliberately exact-range-only: a legitimately nested overlap (e.g. a
|
|
273
|
+
* Protobuf RPC inside its service) never has an identical range to its
|
|
274
|
+
* parent, so this never fires on that intentional case.
|
|
275
|
+
*/
|
|
276
|
+
function findSemanticLossEntries(
|
|
277
|
+
document: ArtifactDocument,
|
|
278
|
+
index: LineIndex,
|
|
279
|
+
chunks: readonly ArtifactChunk[],
|
|
280
|
+
): ArtifactSkipLedgerEntry[] {
|
|
281
|
+
const entries: ArtifactSkipLedgerEntry[] = [];
|
|
282
|
+
|
|
283
|
+
for (const chunk of chunks) {
|
|
284
|
+
if (chunk.range.endByte > chunk.range.startByte) continue;
|
|
285
|
+
entries.push({
|
|
286
|
+
artifactName: document.artifactName,
|
|
287
|
+
path: document.path,
|
|
288
|
+
range: chunk.range,
|
|
289
|
+
reason: "semantic_loss",
|
|
290
|
+
contentPreview: `degenerate chunk: kind="${chunk.kind}" title=${
|
|
291
|
+
chunk.title === undefined ? "(none)" : JSON.stringify(chunk.title)
|
|
292
|
+
} produced an empty/inverted range [${chunk.range.startByte}, ${chunk.range.endByte}) — a section boundary was found but no content was captured for it`.slice(
|
|
293
|
+
0,
|
|
294
|
+
PREVIEW_LENGTH,
|
|
295
|
+
),
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
const byRange = new Map<string, ArtifactChunk[]>();
|
|
300
|
+
for (const chunk of chunks) {
|
|
301
|
+
if (chunk.range.endByte <= chunk.range.startByte) continue; // already reported above
|
|
302
|
+
const key = `${chunk.range.startByte}:${chunk.range.endByte}`;
|
|
303
|
+
const group = byRange.get(key);
|
|
304
|
+
if (group) {
|
|
305
|
+
group.push(chunk);
|
|
306
|
+
} else {
|
|
307
|
+
byRange.set(key, [chunk]);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
for (const group of byRange.values()) {
|
|
311
|
+
if (group.length < 2) continue;
|
|
312
|
+
const identities = new Set(group.map((chunk) => `${chunk.kind}${chunk.title ?? ""}`));
|
|
313
|
+
if (identities.size < 2) continue; // same range, same identity: not a conflict
|
|
314
|
+
const first = group[0]!;
|
|
315
|
+
const startLine = lineNumberForByte(index, first.range.startByte);
|
|
316
|
+
const endLine =
|
|
317
|
+
first.range.endByte > first.range.startByte
|
|
318
|
+
? lineNumberForByte(index, first.range.endByte - 1)
|
|
319
|
+
: startLine;
|
|
320
|
+
entries.push({
|
|
321
|
+
artifactName: document.artifactName,
|
|
322
|
+
path: document.path,
|
|
323
|
+
range: { startLine, endLine, startByte: first.range.startByte, endByte: first.range.endByte },
|
|
324
|
+
reason: "semantic_loss",
|
|
325
|
+
contentPreview:
|
|
326
|
+
`conflicting classification: ${group.length} chunks share byte range [${first.range.startByte}, ${
|
|
327
|
+
first.range.endByte
|
|
328
|
+
}) with different identities: ${group.map((chunk) => `kind="${chunk.kind}" title=${chunk.title ?? "(none)"}`).join(" vs. ")}`.slice(
|
|
329
|
+
0,
|
|
330
|
+
PREVIEW_LENGTH,
|
|
331
|
+
),
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
return entries;
|
|
336
|
+
}
|
package/src/ids.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic `ArtifactChunk.chunkId` derivation.
|
|
3
|
+
*
|
|
4
|
+
* `packages/domain/src/identity.ts` defines one `buildXId` helper per ID
|
|
5
|
+
* kind from spec.md §5, but `ArtifactChunk` (architecture.md §20) is an
|
|
6
|
+
* Untangle-decision shape with no spec.md §5 identity formula of its own —
|
|
7
|
+
* domain deliberately leaves it to this package (see README
|
|
8
|
+
* "Entscheidungen"). This mirrors `buildChunkId`'s approach exactly: a
|
|
9
|
+
* `uuidv5` under the shared Untangle namespace, over a kind-tagged,
|
|
10
|
+
* unit-separator-joined component list, so a re-chunk of unchanged input
|
|
11
|
+
* reproduces the exact same id (stable across index runs) while chunk ids
|
|
12
|
+
* from different artifacts/documents/kinds can never collide.
|
|
13
|
+
*
|
|
14
|
+
* No `Date.now()`/`Math.random()` anywhere — pure function of its inputs
|
|
15
|
+
* (rule 5).
|
|
16
|
+
*/
|
|
17
|
+
import { ID_COMPONENT_SEPARATOR, uuidv5 } from "@untangle-code/domain";
|
|
18
|
+
|
|
19
|
+
export interface ArtifactChunkIdInput {
|
|
20
|
+
artifactName: string;
|
|
21
|
+
/** "/"-normalized document path (as carried on `ArtifactDocument.path`). */
|
|
22
|
+
documentPath: string;
|
|
23
|
+
/** Semantic chunk kind, e.g. "sql_table", "markdown_section". */
|
|
24
|
+
kind: string;
|
|
25
|
+
/** 0-based position among this document's chunks, so equal-kind chunks never collide. */
|
|
26
|
+
ordinal: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function assertNonEmpty(value: string, label: string): void {
|
|
30
|
+
if (value.length === 0) {
|
|
31
|
+
throw new Error(`buildArtifactChunkId: "${label}" must not be empty`);
|
|
32
|
+
}
|
|
33
|
+
if (value.includes(ID_COMPONENT_SEPARATOR)) {
|
|
34
|
+
throw new Error(`buildArtifactChunkId: "${label}" must not contain the U+001F separator`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Deterministic `ArtifactChunk.chunkId` (see module doc). */
|
|
39
|
+
export function buildArtifactChunkId(input: ArtifactChunkIdInput): string {
|
|
40
|
+
assertNonEmpty(input.artifactName, "artifactName");
|
|
41
|
+
assertNonEmpty(input.documentPath, "documentPath");
|
|
42
|
+
assertNonEmpty(input.kind, "kind");
|
|
43
|
+
if (!Number.isInteger(input.ordinal) || input.ordinal < 0) {
|
|
44
|
+
throw new Error(
|
|
45
|
+
`buildArtifactChunkId: ordinal must be a non-negative integer, got ${input.ordinal}`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
return uuidv5(
|
|
49
|
+
[
|
|
50
|
+
"artifact_chunk",
|
|
51
|
+
input.artifactName,
|
|
52
|
+
input.documentPath,
|
|
53
|
+
input.kind,
|
|
54
|
+
String(input.ordinal),
|
|
55
|
+
].join(ID_COMPONENT_SEPARATOR),
|
|
56
|
+
);
|
|
57
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @untangle-code/context-artifacts — the ONLY public export of this
|
|
3
|
+
* package.
|
|
4
|
+
*
|
|
5
|
+
* `createArtifactAdapterRegistry()` returns an `ArtifactAdapter`
|
|
6
|
+
* (`@untangle-code/domain` `ports.ts`, architecture.md §20) that dispatches
|
|
7
|
+
* to one of the built-in format adapters — Markdown/ADR, SQL schema,
|
|
8
|
+
* OpenAPI, AsyncAPI, Protobuf, Terraform/HCL, Kubernetes YAML, Docker/
|
|
9
|
+
* Compose — or the safe generic text fallback. Individual `create*Adapter()`
|
|
10
|
+
* factories are exported too, both for direct golden-testing (one per
|
|
11
|
+
* adapter, see each `src/adapters/*.test.ts`) and so a caller who only
|
|
12
|
+
* needs one format can skip the registry entirely.
|
|
13
|
+
*
|
|
14
|
+
* Also exported: cross-artifact config validation (`validateArtifactConfigs`,
|
|
15
|
+
* spec.md §19.2's project-wide unique name requirement), staleness
|
|
16
|
+
* snapshotting (`computeArtifactStalenessSnapshot`/`isArtifactStale`,
|
|
17
|
+
* spec.md §19.2's "config hash + content hash"), and the skip ledger
|
|
18
|
+
* (`buildArtifactSkipLedger`, BUILD_PLAN §12 E4) — see README.md
|
|
19
|
+
* "Entscheidungen" for why these last two exist as this package's own
|
|
20
|
+
* derived utilities rather than domain port additions.
|
|
21
|
+
*/
|
|
22
|
+
export type { ArtifactAdapterRegistry, ArtifactAdapterRegistryOptions } from "./registry.js";
|
|
23
|
+
export { createArtifactAdapterRegistry } from "./registry.js";
|
|
24
|
+
|
|
25
|
+
export type { MarkdownAdapterOptions } from "./adapters/markdown.js";
|
|
26
|
+
export { createMarkdownAdapter } from "./adapters/markdown.js";
|
|
27
|
+
export type { SqlAdapterOptions, SqlDialect } from "./adapters/sql.js";
|
|
28
|
+
export { createSqlAdapter, DEFAULT_SQL_DIALECT } from "./adapters/sql.js";
|
|
29
|
+
export type { OpenApiAdapterOptions } from "./adapters/openapi.js";
|
|
30
|
+
export { createOpenApiAdapter } from "./adapters/openapi.js";
|
|
31
|
+
export type { AsyncApiAdapterOptions } from "./adapters/asyncapi.js";
|
|
32
|
+
export { createAsyncApiAdapter } from "./adapters/asyncapi.js";
|
|
33
|
+
export type { ProtobufAdapterOptions } from "./adapters/protobuf.js";
|
|
34
|
+
export { createProtobufAdapter } from "./adapters/protobuf.js";
|
|
35
|
+
export type { TerraformAdapterOptions } from "./adapters/terraform.js";
|
|
36
|
+
export { createTerraformAdapter } from "./adapters/terraform.js";
|
|
37
|
+
export type { KubernetesAdapterOptions } from "./adapters/kubernetes.js";
|
|
38
|
+
export { createKubernetesAdapter } from "./adapters/kubernetes.js";
|
|
39
|
+
export type { DockerAdapterOptions } from "./adapters/docker.js";
|
|
40
|
+
export { createDockerAdapter } from "./adapters/docker.js";
|
|
41
|
+
export type { TextFallbackAdapterOptions } from "./adapters/text.js";
|
|
42
|
+
export { createTextFallbackAdapter, TEXT_WINDOW_LINES } from "./adapters/text.js";
|
|
43
|
+
|
|
44
|
+
export { validateArtifactConfigs } from "./config-validation.js";
|
|
45
|
+
|
|
46
|
+
export type { ArtifactStalenessSnapshot } from "./staleness.js";
|
|
47
|
+
export {
|
|
48
|
+
computeArtifactConfigHash,
|
|
49
|
+
computeArtifactContentHash,
|
|
50
|
+
computeArtifactStalenessSnapshot,
|
|
51
|
+
isArtifactStale,
|
|
52
|
+
} from "./staleness.js";
|
|
53
|
+
|
|
54
|
+
export type {
|
|
55
|
+
ArtifactSkipLedgerEntry,
|
|
56
|
+
ArtifactSkipLedgerOptions,
|
|
57
|
+
ArtifactSkipReason,
|
|
58
|
+
ChunkPlausibilityCheck,
|
|
59
|
+
SkipLedgerReporter,
|
|
60
|
+
} from "./coverage.js";
|
|
61
|
+
export { buildArtifactSkipLedger, defaultSkipLedgerReporter } from "./coverage.js";
|
|
62
|
+
|
|
63
|
+
export { buildArtifactChunkId } from "./ids.js";
|
|
64
|
+
export type { ArtifactChunkIdInput } from "./ids.js";
|
|
65
|
+
|
|
66
|
+
// Line-indexing primitives, exposed for a custom adapter registered via
|
|
67
|
+
// `createArtifactAdapterRegistry([...])` that wants the same byte-accurate
|
|
68
|
+
// SourceRange conventions this package's own adapters use.
|
|
69
|
+
export type { LineIndex } from "./text/line-index.js";
|
|
70
|
+
export {
|
|
71
|
+
buildLineIndex,
|
|
72
|
+
rangeForLines,
|
|
73
|
+
sliceBytes,
|
|
74
|
+
sliceLines,
|
|
75
|
+
splitLines,
|
|
76
|
+
} from "./text/line-index.js";
|
|
77
|
+
|
|
78
|
+
// Plausibilitätsprüfung geschnittener Fragmente (WP-118), damit ein eigener
|
|
79
|
+
// Adapter dieselbe E4-Absicherung bekommt wie die eingebauten.
|
|
80
|
+
export type {
|
|
81
|
+
BlockCommentRule,
|
|
82
|
+
BracketRule,
|
|
83
|
+
DelimiterImbalance,
|
|
84
|
+
DelimiterImbalanceKind,
|
|
85
|
+
DelimiterScanOptions,
|
|
86
|
+
HeredocStyle,
|
|
87
|
+
QuoteRule,
|
|
88
|
+
} from "./text/delimiter-balance.js";
|
|
89
|
+
export { describeDelimiterImbalances, findDelimiterImbalances } from "./text/delimiter-balance.js";
|
|
90
|
+
|
|
91
|
+
// The YAML subset parser, exposed for the same reason (e.g. a custom
|
|
92
|
+
// Kubernetes CRD-aware adapter built on top of this package's parser).
|
|
93
|
+
export type {
|
|
94
|
+
YamlDocumentSpan,
|
|
95
|
+
YamlMapEntry,
|
|
96
|
+
YamlNode,
|
|
97
|
+
YamlScalarValue,
|
|
98
|
+
} from "./text/yaml-lite.js";
|
|
99
|
+
export {
|
|
100
|
+
YamlParseError,
|
|
101
|
+
mapGet,
|
|
102
|
+
parseYamlNode,
|
|
103
|
+
scalarString,
|
|
104
|
+
splitYamlDocuments,
|
|
105
|
+
} from "./text/yaml-lite.js";
|
package/src/registry.ts
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `ArtifactAdapterRegistry` (architecture.md §20) — a composite
|
|
3
|
+
* `ArtifactAdapter` that dispatches to the right concrete format adapter
|
|
4
|
+
* and NEVER fails to produce a result: the generic text-fallback adapter
|
|
5
|
+
* is always registered last as a catch-all, so `read()`/`chunk()` always
|
|
6
|
+
* succeed for every file (spec.md §19.2's "sicherer Text-Fallback";
|
|
7
|
+
* rule 4 / BUILD_PLAN §12 E4 — nothing is silently unindexable).
|
|
8
|
+
*
|
|
9
|
+
* Dispatch order (auto-detection priority, see each adapter's own
|
|
10
|
+
* `supports()` for its exact rule):
|
|
11
|
+
*
|
|
12
|
+
* 1. OpenAPI / AsyncAPI — filename-hint + yaml/json extension
|
|
13
|
+
* 2. Docker / Compose — `Dockerfile*` / `(docker-)compose.y*ml` filename
|
|
14
|
+
* 3. Kubernetes — any other `.yaml`/`.yml`
|
|
15
|
+
* 4. Markdown, SQL, Protobuf, Terraform — unambiguous extensions
|
|
16
|
+
* 5. Text fallback — catch-all
|
|
17
|
+
*
|
|
18
|
+
* `chunk()`/`links()` reuse the SAME per-adapter `supports()` rule, called
|
|
19
|
+
* with `(document.type, document.path)` instead of the config's declared
|
|
20
|
+
* type — every non-"text" `ArtifactDocument.type` is itself one adapter's
|
|
21
|
+
* own literal (`"sql"`, `"openapi"`, …), so this always resolves
|
|
22
|
+
* correctly without needing a second identity mechanism; the Docker
|
|
23
|
+
* adapter's `supports()` additionally recognizes `type === "text"` for
|
|
24
|
+
* exactly this reason (its documents carry type `"text"`, see `docker.ts`).
|
|
25
|
+
*/
|
|
26
|
+
import type {
|
|
27
|
+
ArtifactAdapter,
|
|
28
|
+
ArtifactChunk,
|
|
29
|
+
ArtifactDocument,
|
|
30
|
+
ArtifactInput,
|
|
31
|
+
ArtifactLink,
|
|
32
|
+
PortCallOptions,
|
|
33
|
+
} from "@untangle-code/domain";
|
|
34
|
+
import { jobCancelledError } from "@untangle-code/observability";
|
|
35
|
+
|
|
36
|
+
import { createAsyncApiAdapter } from "./adapters/asyncapi.js";
|
|
37
|
+
import { createDockerAdapter } from "./adapters/docker.js";
|
|
38
|
+
import { createKubernetesAdapter } from "./adapters/kubernetes.js";
|
|
39
|
+
import { createMarkdownAdapter } from "./adapters/markdown.js";
|
|
40
|
+
import { createOpenApiAdapter } from "./adapters/openapi.js";
|
|
41
|
+
import { createProtobufAdapter } from "./adapters/protobuf.js";
|
|
42
|
+
import type { SkipLedgerReporter } from "./adapters/shared.js";
|
|
43
|
+
import { createSqlAdapter } from "./adapters/sql.js";
|
|
44
|
+
import { createTerraformAdapter } from "./adapters/terraform.js";
|
|
45
|
+
import { createTextFallbackAdapter } from "./adapters/text.js";
|
|
46
|
+
|
|
47
|
+
export interface ArtifactAdapterRegistry extends ArtifactAdapter {
|
|
48
|
+
/** Registers an additional adapter with LOWER priority than every built-in one (inserted just before the text fallback). */
|
|
49
|
+
register(adapter: ArtifactAdapter): void;
|
|
50
|
+
listAdapters(): readonly ArtifactAdapter[];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
type ArtifactFile = ArtifactInput["files"][number];
|
|
54
|
+
|
|
55
|
+
class DefaultArtifactAdapterRegistry implements ArtifactAdapterRegistry {
|
|
56
|
+
private readonly textFallback: ArtifactAdapter;
|
|
57
|
+
private adapters: ArtifactAdapter[];
|
|
58
|
+
|
|
59
|
+
constructor(builtins: ArtifactAdapter[], textFallback: ArtifactAdapter) {
|
|
60
|
+
this.textFallback = textFallback;
|
|
61
|
+
this.adapters = [...builtins, textFallback];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
register(adapter: ArtifactAdapter): void {
|
|
65
|
+
// Keep the text fallback last no matter when custom adapters are added.
|
|
66
|
+
const withoutFallback = this.adapters.filter((existing) => existing !== this.textFallback);
|
|
67
|
+
this.adapters = [...withoutFallback, adapter, this.textFallback];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
listAdapters(): readonly ArtifactAdapter[] {
|
|
71
|
+
return [...this.adapters];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
supports(type: string, path: string): boolean {
|
|
75
|
+
return this.adapters.some((adapter) => adapter.supports(type, path));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private resolve(type: string, path: string): ArtifactAdapter {
|
|
79
|
+
return this.adapters.find((adapter) => adapter.supports(type, path)) ?? this.textFallback;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async read(input: ArtifactInput, options?: PortCallOptions): Promise<ArtifactDocument[]> {
|
|
83
|
+
const declaredType = input.config.type ?? "auto";
|
|
84
|
+
const groups = new Map<ArtifactAdapter, ArtifactFile[]>();
|
|
85
|
+
for (const file of input.files) {
|
|
86
|
+
if (options?.signal?.aborted) {
|
|
87
|
+
throw jobCancelledError(`context artifact "${input.config.name}" read was aborted`, {
|
|
88
|
+
operation: "context_artifact_read",
|
|
89
|
+
context: { artifactName: input.config.name },
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
const adapter = this.resolve(declaredType, file.path);
|
|
93
|
+
const existing = groups.get(adapter);
|
|
94
|
+
if (existing) {
|
|
95
|
+
existing.push(file);
|
|
96
|
+
} else {
|
|
97
|
+
groups.set(adapter, [file]);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const documents: ArtifactDocument[] = [];
|
|
102
|
+
for (const [adapter, files] of groups) {
|
|
103
|
+
const group = await adapter.read({ config: input.config, files }, options);
|
|
104
|
+
documents.push(...group);
|
|
105
|
+
}
|
|
106
|
+
return documents;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
chunk(document: ArtifactDocument): ArtifactChunk[] {
|
|
110
|
+
return this.resolve(document.type, document.path).chunk(document);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
links(document: ArtifactDocument): ArtifactLink[] {
|
|
114
|
+
const adapter = this.resolve(document.type, document.path);
|
|
115
|
+
return adapter.links?.(document) ?? [];
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface ArtifactAdapterRegistryOptions {
|
|
120
|
+
/**
|
|
121
|
+
* BUILD_PLAN §12 E4: overrides how every BUILT-IN adapter's `chunk()`
|
|
122
|
+
* surfaces a real skip-ledger finding (default: one stderr warning line
|
|
123
|
+
* per finding — see `coverage.ts`'s `defaultSkipLedgerReporter`). Wire in
|
|
124
|
+
* a real structured logger here once, instead of passing the same
|
|
125
|
+
* `onSkipLedgerGap` to every individual `create*Adapter()` call. Does
|
|
126
|
+
* NOT apply to `extra` adapters — those are already fully constructed
|
|
127
|
+
* `ArtifactAdapter`s by the time they reach this factory.
|
|
128
|
+
*/
|
|
129
|
+
onSkipLedgerGap?: SkipLedgerReporter;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Builds a registry with every built-in format adapter registered
|
|
134
|
+
* (priority order per this module's doc comment), plus any `extra`
|
|
135
|
+
* adapters appended just before the text fallback.
|
|
136
|
+
*/
|
|
137
|
+
export function createArtifactAdapterRegistry(
|
|
138
|
+
extra: ArtifactAdapter[] = [],
|
|
139
|
+
options: ArtifactAdapterRegistryOptions = {},
|
|
140
|
+
): ArtifactAdapterRegistry {
|
|
141
|
+
const adapterOptions =
|
|
142
|
+
options.onSkipLedgerGap === undefined ? {} : { onSkipLedgerGap: options.onSkipLedgerGap };
|
|
143
|
+
const builtins: ArtifactAdapter[] = [
|
|
144
|
+
createOpenApiAdapter(adapterOptions),
|
|
145
|
+
createAsyncApiAdapter(adapterOptions),
|
|
146
|
+
createDockerAdapter(adapterOptions),
|
|
147
|
+
createKubernetesAdapter(adapterOptions),
|
|
148
|
+
createMarkdownAdapter(adapterOptions),
|
|
149
|
+
createSqlAdapter(adapterOptions),
|
|
150
|
+
createProtobufAdapter(adapterOptions),
|
|
151
|
+
createTerraformAdapter(adapterOptions),
|
|
152
|
+
...extra,
|
|
153
|
+
];
|
|
154
|
+
return new DefaultArtifactAdapterRegistry(builtins, createTextFallbackAdapter(adapterOptions));
|
|
155
|
+
}
|