@weftlet/toolkit 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +32 -26
  2. package/THIRD_PARTY_NOTICES.md +2 -0
  3. package/dist/application/hydrator.js +18 -15
  4. package/dist/application/overlay.js +9 -9
  5. package/dist/application/semantic-run.d.ts +1 -1
  6. package/dist/extraction/ontology.d.ts +2 -0
  7. package/dist/extraction/ontology.js +28 -13
  8. package/dist/graph/code-coverage.js +2 -2
  9. package/dist/graph/code-rdf.d.ts +3 -2
  10. package/dist/graph/code-rdf.js +21 -14
  11. package/dist/graph/neighborhood.js +3 -3
  12. package/dist/graph/partitioned-shacl.d.ts +14 -0
  13. package/dist/graph/partitioned-shacl.js +66 -0
  14. package/dist/graph/paths.js +3 -3
  15. package/dist/graph/rdf-partitions.d.ts +24 -0
  16. package/dist/graph/rdf-partitions.js +170 -0
  17. package/dist/graph/rdf-projection.d.ts +7 -0
  18. package/dist/graph/rdf-projection.js +27 -0
  19. package/dist/graph/rdf-records.d.ts +12 -0
  20. package/dist/graph/rdf-records.js +90 -0
  21. package/dist/graph/rdf.d.ts +10 -10
  22. package/dist/graph/rdf.js +66 -97
  23. package/dist/graph/validation-progress.d.ts +4 -1
  24. package/dist/infrastructure/json-digest.d.ts +2 -1
  25. package/dist/infrastructure/json-digest.js +2 -2
  26. package/dist/interfaces/cli.js +1 -1
  27. package/dist/storage/storage.d.ts +3 -0
  28. package/dist/storage/storage.js +37 -13
  29. package/dist/storage/streaming-json.d.ts +5 -0
  30. package/dist/storage/streaming-json.js +45 -0
  31. package/docs/1.0-WORK-QUEUE.md +18 -8
  32. package/docs/ADR-040-bounded-graph-io.md +90 -0
  33. package/docs/RELEASING.md +3 -3
  34. package/docs/VALIDATION.md +84 -0
  35. package/docs/benchmarks/large-graph-2026-11-02/result.json +156 -0
  36. package/docs/extraction-manifest.json +26 -10
  37. package/package.json +4 -2
@@ -1,11 +1,12 @@
1
1
  // Modified for standalone extraction; see docs/extraction-manifest.json.
2
- import { mkdir, readFile, writeFile, link, rm } from 'node:fs/promises';
2
+ import { mkdir, writeFile, link, rm } from 'node:fs/promises';
3
3
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
4
4
  import { atomicRename } from './atomic-rename.js';
5
5
  import { physicalArtifactKey, temporaryArtifactPath } from '../infrastructure/artifact-paths.js';
6
+ import { readJsonFile, readJsonProperty, writeJsonFile } from './streaming-json.js';
6
7
  async function readJson(path) {
7
8
  try {
8
- return JSON.parse(await readFile(path, 'utf8'));
9
+ return await readJsonFile(path);
9
10
  }
10
11
  catch (error) {
11
12
  if (error.code === 'ENOENT')
@@ -13,8 +14,8 @@ async function readJson(path) {
13
14
  throw error;
14
15
  }
15
16
  }
16
- async function readWithLegacyPath(path, legacyPath) {
17
- const current = await readJson(path);
17
+ async function readWithLegacyPath(path, legacyPath, read = (readJson)) {
18
+ const current = await read(path);
18
19
  if (current !== undefined || path === legacyPath)
19
20
  return current;
20
21
  // No old file could have this component on supported filesystems. Do not attempt an
@@ -23,7 +24,7 @@ async function readWithLegacyPath(path, legacyPath) {
23
24
  return undefined;
24
25
  if (process.platform === 'win32' && /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(basename(legacyPath)))
25
26
  return undefined;
26
- return readJson(legacyPath);
27
+ return read(legacyPath);
27
28
  }
28
29
  async function atomicJson(path, value) {
29
30
  await mkdir(dirname(path), { recursive: true });
@@ -65,9 +66,9 @@ export class FileSnapshotStore {
65
66
  }
66
67
  async write(snapshot) {
67
68
  const destination = snapshotPath(this.root, snapshot.manifest.snapshotId);
68
- const prior = await this.read(snapshot.manifest.snapshotId);
69
+ const prior = await this.readManifest(snapshot.manifest.snapshotId);
69
70
  if (prior) {
70
- if (prior.manifest.contentDigest !== snapshot.manifest.contentDigest) {
71
+ if (prior.contentDigest !== snapshot.manifest.contentDigest) {
71
72
  throw new Error(`Refusing to overwrite immutable snapshot ${snapshot.manifest.snapshotId} with different content`);
72
73
  }
73
74
  return; // Preserve an existing legacy file without copying or rewriting its bytes.
@@ -75,15 +76,15 @@ export class FileSnapshotStore {
75
76
  await mkdir(dirname(destination), { recursive: true });
76
77
  const temporary = temporaryArtifactPath(destination);
77
78
  try {
78
- await writeFile(temporary, JSON.stringify(snapshot), { flag: 'wx' });
79
+ await writeJsonFile(temporary, snapshot);
79
80
  // Linking a complete file publishes atomically without ever replacing a winner.
80
81
  await link(temporary, destination);
81
82
  }
82
83
  catch (error) {
83
84
  if (error.code !== 'EEXIST')
84
85
  throw error;
85
- const existing = await this.read(snapshot.manifest.snapshotId);
86
- if (existing?.manifest.contentDigest !== snapshot.manifest.contentDigest) {
86
+ const existing = await this.readManifest(snapshot.manifest.snapshotId);
87
+ if (existing?.contentDigest !== snapshot.manifest.contentDigest) {
87
88
  throw new Error(`Refusing to overwrite immutable snapshot ${snapshot.manifest.snapshotId} with different content`);
88
89
  }
89
90
  }
@@ -94,13 +95,36 @@ export class FileSnapshotStore {
94
95
  read(snapshotId) {
95
96
  return readWithLegacyPath(snapshotPath(this.root, snapshotId), snapshotPath(this.root, snapshotId, true));
96
97
  }
98
+ readManifest(snapshotId) {
99
+ return readWithLegacyPath(snapshotPath(this.root, snapshotId), snapshotPath(this.root, snapshotId, true), async (path) => {
100
+ try {
101
+ const manifest = await readJsonProperty(path, 'manifest');
102
+ if (!manifest || typeof manifest.contentDigest !== 'string')
103
+ throw Error('Invalid snapshot manifest');
104
+ return manifest;
105
+ }
106
+ catch (error) {
107
+ if (error.code === 'ENOENT')
108
+ return undefined;
109
+ throw error;
110
+ }
111
+ });
112
+ }
97
113
  async readHead(headName) {
114
+ const snapshotId = await this.headId(headName);
115
+ return snapshotId ? this.read(snapshotId) : undefined;
116
+ }
117
+ async readHeadManifest(headName) {
118
+ const snapshotId = await this.headId(headName);
119
+ return snapshotId ? this.readManifest(snapshotId) : undefined;
120
+ }
121
+ async headId(headName) {
98
122
  const key = safeKey(headName);
99
123
  const head = await readWithLegacyPath(join(this.root, 'heads', `${physicalArtifactKey(key)}.json`), join(this.root, 'heads', `${key}.json`));
100
- return head ? this.read(head.snapshotId) : undefined;
124
+ return head?.snapshotId;
101
125
  }
102
126
  async updateHead(headName, snapshotId) {
103
- if (!(await this.read(snapshotId)))
127
+ if (!(await this.readManifest(snapshotId)))
104
128
  throw new Error('Cannot point a head to an absent snapshot');
105
129
  return atomicJson(join(this.root, 'heads', `${physicalArtifactKey(safeKey(headName))}.json`), { snapshotId });
106
130
  }
@@ -138,7 +162,7 @@ export class SemanticArtifactStore {
138
162
  await mkdir(dirname(path), { recursive: true });
139
163
  const temporary = temporaryArtifactPath(path);
140
164
  try {
141
- await writeFile(temporary, JSON.stringify(artifact), { flag: 'wx' });
165
+ await writeJsonFile(temporary, artifact);
142
166
  await link(temporary, path);
143
167
  }
144
168
  catch (error) {
@@ -0,0 +1,5 @@
1
+ /** Ordinary JSON bytes, incrementally encoded/decoded; legacy files need no migration. */
2
+ export declare function readJsonFile<T>(path: string): Promise<T>;
3
+ /** Validate the complete document while retaining only a selected root property. */
4
+ export declare function readJsonProperty<T>(path: string, property: string): Promise<T | undefined>;
5
+ export declare function writeJsonFile(path: string, value: unknown): Promise<void>;
@@ -0,0 +1,45 @@
1
+ import { createReadStream, createWriteStream } from 'node:fs';
2
+ import { pipeline } from 'node:stream/promises';
3
+ import { stringifyChunked } from '@discoveryjs/json-ext';
4
+ import { JSONParser, TokenizerError, TokenParserError } from '@streamparser/json';
5
+ /** Ordinary JSON bytes, incrementally encoded/decoded; legacy files need no migration. */
6
+ export async function readJsonFile(path) {
7
+ return parseJsonFile(path, '$');
8
+ }
9
+ /** Validate the complete document while retaining only a selected root property. */
10
+ export async function readJsonProperty(path, property) {
11
+ if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(property))
12
+ throw Error('Invalid JSON property selector');
13
+ return parseJsonFile(path, `$.${property}`);
14
+ }
15
+ async function parseJsonFile(path, selector) {
16
+ let value;
17
+ const parser = new JSONParser({ paths: [selector], keepStack: selector === '$' });
18
+ let complete = false;
19
+ let hasInput = false;
20
+ parser.onValue = result => { value = result.value; };
21
+ parser.onEnd = () => { complete = true; };
22
+ parser.onError = error => {
23
+ // Cache inspection distinguishes invalid JSON from I/O/infrastructure errors.
24
+ // Keep that contract without including artifact contents in the public message.
25
+ // 0.0.26 reports a truncated container as plain Error, unlike invalid tokens.
26
+ const incompleteContainer = error.constructor === Error && error.message.startsWith('Parser ended in mid-parsing (state:');
27
+ if (error instanceof TokenizerError || error instanceof TokenParserError || incompleteContainer) {
28
+ throw new SyntaxError('Invalid JSON document', { cause: error });
29
+ }
30
+ throw error;
31
+ };
32
+ for await (const chunk of createReadStream(path)) {
33
+ if (!hasInput)
34
+ hasInput = chunk.some(byte => ![9, 10, 13, 32].includes(byte));
35
+ parser.write(chunk);
36
+ }
37
+ if (!parser.isEnded)
38
+ parser.end();
39
+ if (!complete || !hasInput)
40
+ throw new SyntaxError('Incomplete JSON document');
41
+ return value;
42
+ }
43
+ export async function writeJsonFile(path, value) {
44
+ await pipeline(stringifyChunked(value), createWriteStream(path, { flags: 'wx' }));
45
+ }
@@ -16,18 +16,28 @@ for this candidate. Prior consumer Codex hydration remains evidence for the esta
16
16
  flow; it does not validate the new proposal operation. No further feature milestone is scheduled
17
17
  before this candidate. B6b and the explicit official 1.0 decision remain open.
18
18
 
19
+ Consumer blocker, 2026-11-02: 0.7.0's richer code graph exceeded Node's whole-string limit before
20
+ strict SHACL/publication. [ADR 040](ADR-040-bounded-graph-io.md) tracks the hashing, persistence,
21
+ RDF/query and bounded-validation patch. Its public large-corpus lifecycle and hosted Linux/Windows checks
22
+ pass; patch publication and private acceptance remain separate steps. No 1.0 approval is implied.
23
+
24
+ All three public 554,516-fact strict SHACL passes succeeded but took about 35–37 minutes each
25
+ in this cloud. This is a capacity result, not a 30-minute hydration qualification. B6b must record
26
+ full hydration and agent-query latency on the consumer workload as well as correctness.
27
+ Any further validation optimization must retain full constraints and parity evidence.
28
+
19
29
  | Task | Status / dependency | Deliverable and check |
20
30
  | --- | --- | --- |
21
- | **B1a — Exact node neighborhoods** | Implemented; next candidate pending. Validation in ADR 029 and the change PR | `graph.neighbors` through the shared public API, CLI and MCP. Exact snapshot/node IDs, direction/relation filters, cited assertion pagination, remaining-neighbor counts, further-expansion hints and shared evidence state. Check duplicate labels, cycles, self-links, parallel evidence, missing/deleted nodes, stale/tampered cursors, dirty overlays and real extracted cross-service facts. |
22
- | **B1b — Bounded dependency paths** | Implemented; next candidate pending. Validation in ADR 030 and the change PR | `graph.paths` enumerates cited simple paths through API/CLI/MCP, with depth/work/output limits, continuation and explicit pending/depth-cut frontier. The offline viewer uses the same search function on displayed facts. Check competing branches, cycles, deep paths, scope omissions and all 4,096 directed four-node graphs against an independent oracle. |
31
+ | **B1a — Exact node neighborhoods** | Shipped in 0.7.0. Validation in ADR 029 and the change PR | `graph.neighbors` through the shared public API, CLI and MCP. Exact snapshot/node IDs, direction/relation filters, cited assertion pagination, remaining-neighbor counts, further-expansion hints and shared evidence state. Check duplicate labels, cycles, self-links, parallel evidence, missing/deleted nodes, stale/tampered cursors, dirty overlays and real extracted cross-service facts. |
32
+ | **B1b — Bounded dependency paths** | Shipped in 0.7.0. Validation in ADR 030 and the change PR | `graph.paths` enumerates cited simple paths through API/CLI/MCP, with depth/work/output limits, continuation and explicit pending/depth-cut frontier. The offline viewer uses the same search function on displayed facts. Check competing branches, cycles, deep paths, scope omissions and all 4,096 directed four-node graphs against an independent oracle. |
23
33
  | **B2a — Freeze code-graph comparison** | Frozen; verifier and fixture typechecks pass | 51 labeled code cases, three explicit gaps, unchanged A1–A3/dispatch controls, eight scenarios, pinned tools and bounded metrics in `evaluations/code-graph`. Read-only verification rejects input/label drift and supports immutable Git refs. No comparative execution or private/model inputs. |
24
34
  | **B2b — Run comparison and decide** | Measured; [ADR 031](ADR-031-code-extractor-selection.md) selects richer SCIP projection | Compare current projection, richer raw SCIP and pinned Graphify static extraction. Record correctness, citations, identity stability, cold/edit/repeat cost and packaging. Make an evidence-backed keep/improve/replace decision before B3's code-entity schema. |
25
- | **B3a — Explicit rule-to-symbol resolution** | Implemented in two increments ([ADR 032](ADR-032-scip-code-evidence.md), [ADR 033](ADR-033-document-symbol-bindings.md)); next candidate pending | Retain local/private symbols, same-file references and repeated source occurrences before resolving explicit document paths/symbol references. Include aliases, overloads, duplicate names, rename/delete and unresolved targets; retain both source spans/revisions and mapping authority. Follow the canonical [same-file acceptance boundary](PRE-1.0-BASELINE.md#same-file-exploration-acceptance). |
26
- | **B3b — Assess missing mappings** | Deterministic assessment complete ([ADR 034](ADR-034-mapping-gaps.md)); bounded opt-in implementation complete ([ADR 039](ADR-039-mapping-proposals.md)); live validation tentative and deferred by the maintainer for 0.7.0 | Identify gaps remaining after explicit resolution. Evaluate a bounded inference proposal only if justified, with an approved model budget before paid work. Never infer owner approval or silently convert proposed mappings into proven behavior. Record any scoped deferral for maintainer review. |
27
- | **B3c — Same-file call exploration** | Implemented ([ADR 035](ADR-035-static-call-evidence.md)); next candidate pending | Select and test static call analysis, preserving caller/callee/callsite and unresolved targets. Distinguish direct calls from references and callback registration; never claim observed execution. Meet the public large-file extraction/query acceptance in the canonical boundary, including recursion, unrelated functions, shadowed names, repeated sites and cross-file callers. No duplicate-behavior equivalence or mandatory second indexer. |
28
- | **B4a — Producer/consumer obligations** | Implemented for explicit declarations ([ADR 036](ADR-036-contract-obligation-declarations.md)); next candidate pending | Represent and retrieve cited guarantees/requirements on queue and stored-data contracts, with domain, stage, units, freshness and version where documented. Keep absent fields unknown. |
29
- | **B4b — Compatibility acceptance** | Implemented for selected explicit numeric declarations ([ADR 037](ADR-037-selected-contract-compatibility.md)); next candidate pending | Public queue/stored-data cases cover subset, insufficient guarantee, disjoint limits, revalidation and unknowns. Shared `graph.compatibility` retains two cited assertions and follow-up IDs. This is advisory; hard failures require approved deterministic policy. |
30
- | **B5 — Monorepo coverage** | Implemented for saved TypeScript-family inventory and package boundaries ([ADR 038](ADR-038-monorepo-code-coverage.md)); next candidate pending | `graph.coverage` pages selected/requested/indexed files and package connections; shared overview distinguishes saved scope from returned-page counts. Real barrels, aliases, TSX, independent/omitted services, declaration-only and unresolved external targets are tested. Unknown/zero observations never prove no impact. |
35
+ | **B3a — Explicit rule-to-symbol resolution** | Implemented in two increments ([ADR 032](ADR-032-scip-code-evidence.md), [ADR 033](ADR-033-document-symbol-bindings.md)); shipped in 0.7.0 | Retain local/private symbols, same-file references and repeated source occurrences before resolving explicit document paths/symbol references. Include aliases, overloads, duplicate names, rename/delete and unresolved targets; retain both source spans/revisions and mapping authority. Follow the canonical [same-file acceptance boundary](PRE-1.0-BASELINE.md#same-file-exploration-acceptance). |
36
+ | **B3b — Assess missing mappings** | Deterministic assessment complete ([ADR 034](ADR-034-mapping-gaps.md)); bounded opt-in implementation shipped in 0.7.0 ([ADR 039](ADR-039-mapping-proposals.md)); live validation tentative and deferred by the maintainer for 0.7.0 | Identify gaps remaining after explicit resolution. Evaluate a bounded inference proposal only if justified, with an approved model budget before paid work. Never infer owner approval or silently convert proposed mappings into proven behavior. Record any scoped deferral for maintainer review. |
37
+ | **B3c — Same-file call exploration** | Implemented ([ADR 035](ADR-035-static-call-evidence.md)); shipped in 0.7.0 | Select and test static call analysis, preserving caller/callee/callsite and unresolved targets. Distinguish direct calls from references and callback registration; never claim observed execution. Meet the public large-file extraction/query acceptance in the canonical boundary, including recursion, unrelated functions, shadowed names, repeated sites and cross-file callers. No duplicate-behavior equivalence or mandatory second indexer. |
38
+ | **B4a — Producer/consumer obligations** | Implemented for explicit declarations ([ADR 036](ADR-036-contract-obligation-declarations.md)); shipped in 0.7.0 | Represent and retrieve cited guarantees/requirements on queue and stored-data contracts, with domain, stage, units, freshness and version where documented. Keep absent fields unknown. |
39
+ | **B4b — Compatibility acceptance** | Implemented for selected explicit numeric declarations ([ADR 037](ADR-037-selected-contract-compatibility.md)); shipped in 0.7.0 | Public queue/stored-data cases cover subset, insufficient guarantee, disjoint limits, revalidation and unknowns. Shared `graph.compatibility` retains two cited assertions and follow-up IDs. This is advisory; hard failures require approved deterministic policy. |
40
+ | **B5 — Monorepo coverage** | Implemented for saved TypeScript-family inventory and package boundaries ([ADR 038](ADR-038-monorepo-code-coverage.md)); shipped in 0.7.0 | `graph.coverage` pages selected/requested/indexed files and package connections; shared overview distinguishes saved scope from returned-page counts. Real barrels, aliases, TSX, independent/omitted services, declaration-only and unresolved external targets are tested. Unknown/zero observations never prove no impact. |
31
41
  | **B6a — Candidate qualification** | B1–B5 decisions/increments reviewed | Freeze tasks and expected downstream obligations before measuring. Run public end-to-end, fresh-install and supported Windows/Linux release gates. Publish a minor acceptance candidate with versioned evidence and limitations. |
32
42
  | **B6b — Live consumer and 1.0 decision** | Published candidate | TradePilot runs real hydration/repeat and agent follow-up tasks; retain private requests/citations/timings locally and share a sanitized result. Assess usefulness and misses separately from pipeline success. Fix blockers, then explicitly review official 1.0. |
33
43
 
@@ -0,0 +1,90 @@
1
+ # ADR 040: Preserve evidence while removing whole-graph string and RDF copies
2
+
3
+ Status: implemented; public large-corpus lifecycle and hosted Linux/Windows checks pass.
4
+ Consumer acceptance remains open.
5
+
6
+ ## Problem
7
+
8
+ The 0.7.0 consumer report reaches 561,777 facts / 174,710 nodes, reusing 18 semantic artifacts
9
+ without model calls, then fails at `sha256(stableJson(logical))`. Node's string ceiling is
10
+ 536,870,888 UTF-16 code units; increasing the heap does not remove it. Snapshot JSON writing,
11
+ JSON reload and full RDF exports had similar allocations. The old internal SPARQL projection
12
+ also rejected more than 100,000 facts.
13
+
14
+ A generated public TypeScript corpus additionally demonstrated the memory cost of retaining a
15
+ full named N3 dataset and its validation union simultaneously. The fix must retain every assertion,
16
+ test fact, citation, confidence, authority and review state. It does not change extraction scope.
17
+
18
+ ## Decision
19
+
20
+ - `stableJsonDigest` incrementally hashes the **existing encoding**, including sorted keys,
21
+ escaping, number formatting and legacy undefined-field behavior. `persistedJsonDigest` retains
22
+ the existing JSON-normalized digest contract. Neither is a new canonicalization standard.
23
+ - Stream ordinary snapshot/artifact JSON to exclusive temporary files, then publish with the
24
+ existing atomic hard-link/head replacement rules. Read JSON incrementally with a strict parser.
25
+ Header-only checks parse the complete document but retain only the manifest. Old files need no
26
+ migration and existing immutable winners are never rewritten.
27
+ - Stream default semantic N-Triples and optional openCAESAR file digests, preserving their existing
28
+ byte encodings and hashes. No Docker or provider is required to query saved evidence.
29
+ - Use one RDF encoder for named exports, SPARQL matching and validation partitions. RDF/JS matching
30
+ streams complete groups and indexes subject/graph lookups; Comunica remains the SPARQL engine.
31
+ Cardinalities are exact, never guessed for optimizer shortcuts. The traversal record projection
32
+ has equivalence tests against SPARQL, including duplicate-ID join multiplicity, and no graph-size
33
+ response cap. Public query result limits still report truncation.
34
+ - `serializeChunks()` / `serializeTo(writable)` export complete N-Quads with backpressure. The
35
+ legacy `serialize(): Promise<string>` convenience necessarily remains limited by the JavaScript
36
+ string ceiling. N-Quads ordering can differ from 0.7.0; its quad set is unchanged. Export order
37
+ is not a snapshot identity. Snapshot JSON encoding and semantic RDF digests are preserved.
38
+
39
+ ## Strict SHACL without two complete RDF stores
40
+
41
+ Keep `rdf-validate-shacl` and every current constraint. Group all outgoing triples for a subject
42
+ together, including every occurrence of a shared identity; do not validate isolated conflicting
43
+ assertions as if each were a separate node. Current shapes inspect direct outgoing properties;
44
+ their only cross-subject constraint is `sh:class`. Supply complete graph-wide `rdf:type` membership
45
+ for those references. Determine the original targets before adding support-only type triples and
46
+ express those same targets as `sh:targetNode` for the batch. Support-only nodes are checked with
47
+ their complete records in their own batch.
48
+
49
+ Every subject and constraint is evaluated. Missing proof targets, contradictory metadata, kind
50
+ conflicts, corrupt citations and malformed code evidence still fail. Existing TypeScript proof
51
+ and provenance checks run over all facts after SHACL. No validation result is reused or inferred
52
+ from semantic cache hits. An allowlist rejects new SHACL vocabulary or non-direct paths until
53
+ partition locality is reviewed; unsupported constraints cannot silently pass. Batch progress
54
+ reports completed groups and elapsed time, not an estimated completion time.
55
+
56
+ This changes execution/storage, not asserted semantics: assembly 12, extractor identities,
57
+ semantic cache keys, schema version and logical snapshot IDs remain unchanged. Old snapshots stay
58
+ addressable; existing extraction incompatibility checks still apply.
59
+
60
+ ## Validation and limits
61
+
62
+ Tests compare against a frozen 0.7.0 RDF builder at
63
+ `073edab3f1cef875f2fc058ffbc0ddc59cf934d2`. They compare complete named and union quad sets,
64
+ positive/negative verdicts and issues, real SCIP/code-proof metadata, cross-partition class
65
+ references, aggregates and traversal projections. Hash compatibility includes Unicode, surrogate
66
+ boundaries, optional fields and an encoding larger than V8's actual string ceiling. Storage tests
67
+ cover malformed/truncated JSON, special property names, legacy files and concurrent publication.
68
+
69
+ The repeatable public workload is generated TypeScript with shared helpers, same-file and
70
+ cross-file calls, spec files and 18 prose documents. Its provider is a deterministic fixture
71
+ parser, not a live model. It exercises real SCIP, strict hydration, snapshot reload, streaming
72
+ N-Quads parsing and bounded SPARQL, unchanged repeat and clean comparison; it asserts canonical
73
+ encoding size above the actual string ceiling, exact provenance, semantic reuse, unchanged
74
+ versions and immutable snapshot/artifact bytes.
75
+
76
+ ```sh
77
+ pnpm build
78
+ pnpm exec vitest run tests/large-graph-lifecycle.test.ts tests/rdf-scaling-parity.test.ts
79
+ node --max-old-space-size=8192 --import tsx scripts/large-graph/run.ts 1100 40 --require-large
80
+ ```
81
+
82
+ The large run is deliberately separate from ordinary CI; its scaled fixture runs in CI.
83
+ The [recorded full run](benchmarks/large-graph-2026-11-02/result.json) passes with 554,516 facts,
84
+ a 1 GB snapshot and a complete 5.3 GB RDF export. Each strict SHACL pass takes about 35–37 minutes;
85
+ this does not establish a 30-minute hydration budget. Linux capacity results do not establish
86
+ native Windows consumer acceptance. See VALIDATION.md for source/checkpoint distinctions. This is a capacity regression, not a
87
+ semantic-quality or universal performance guarantee. Parsed graph objects and indexes still use
88
+ memory; a single very large record/identity can exceed a group target. Arbitrary SPARQL joins,
89
+ sorts and aggregates have their own time/memory costs. There is no persistent graph database,
90
+ incremental-validation cache or Rust validator in this change.
package/docs/RELEASING.md CHANGED
@@ -202,14 +202,14 @@ Verify the matching public version in a fresh disposable project after publicati
202
202
 
203
203
  ```sh
204
204
  npm init -y
205
- npx --yes @weftlet/toolkit@0.7.0 init --yes
206
- npx --yes @weftlet/toolkit@0.7.0 config validate
205
+ npx --yes @weftlet/toolkit@0.7.1 init --yes
206
+ npx --yes @weftlet/toolkit@0.7.1 config validate
207
207
  ```
208
208
 
209
209
  For local installation, use both prepared archives from the same reviewed release:
210
210
 
211
211
  ```sh
212
- npm install --save-dev "/path/to/weftlet/.release/weftlet-core-0.7.0.tgz" "/path/to/weftlet/.release/weftlet-toolkit-0.7.0.tgz"
212
+ npm install --save-dev "/path/to/weftlet/.release/weftlet-core-0.7.1.tgz" "/path/to/weftlet/.release/weftlet-toolkit-0.7.1.tgz"
213
213
  ```
214
214
  <!-- weftlet:release:end -->
215
215
 
@@ -1,5 +1,89 @@
1
1
  # Validation evidence
2
2
 
3
+ ## 0.7.0 large-graph field blocker follow-up, 2026-11-02
4
+
5
+ Status: public capacity qualification and hosted Linux/Windows checks pass; patch release pending.
6
+ Private consumer acceptance remains open.
7
+ [ADR 040](ADR-040-bounded-graph-io.md) records the encoding, storage and strict SHACL boundaries.
8
+ The consumer's reported 561,777-fact failure occurs before SHACL at whole-graph canonical
9
+ string construction. Its prior snapshots remain readable; no private inputs were imported here.
10
+
11
+ Local Linux checks currently establish:
12
+
13
+ - Production build and 38 core tests pass, including an independently checked canonical hash
14
+ larger than V8's actual string ceiling. Existing encoding and persisted-JSON digest semantics
15
+ are retained, including optional fields and Unicode boundaries.
16
+ - 65 focused lifecycle, RDF parity, streaming-storage and release-documentation tests pass.
17
+ The frozen 0.7.0 reference verifies named/union quad sets, strict positive/negative verdicts,
18
+ cross-subject proof checks, SPARQL aggregates and lossless traversal projection. A separate
19
+ 100,001-fact test removes the old projection ceiling without truncation.
20
+ - Fresh packed-consumer installation passes setup, preview/cache adoption, traversal hydration
21
+ and reuse, real SCIP, RDF/SPARQL/SHACL, HTML/DOT/SVG assets and MCP.
22
+ - Inventory and license checks pass; all 79 original source import path/SHA pairs are unchanged.
23
+ Architecture passes with 15 advisory findings: two additional complexity advisories in the
24
+ canonical digest visitor and RDF matching function. No baseline, exception or gate changed.
25
+
26
+ The full suite exposed a corrupt-JSON error-class compatibility regression. The streaming
27
+ reader now preserves SyntaxError classification for malformed/truncated input while retaining
28
+ I/O errors as infrastructure failures; cache-adoption CLI and strict parser regressions pass.
29
+ The full local pass before those fixes completed with 977 passes / 8 failures / 1 optional skip.
30
+ All corrupt-artifact cases pass on rerun (44 cache/preview/storage tests plus 3 CLI cases).
31
+ An overlapping local rebuild interrupted a compiled benchmark subprocess; its 6 tests pass after
32
+ the build. Seven sequential CLI launches exceeded the old aggregate 30-second setup-test budget;
33
+ the test now bounds each process to 30 seconds and the sequence to 180 seconds, retaining all
34
+ assertions (3 setup tests pass). A subsequent 29-test RDF/setup run passes. These corrections
35
+ do not change production operation deadlines. At `85361dbb` both hosted runs
36
+ [41532316889](https://github.com/jchewitt/weftlet/actions/runs/41532316889) and
37
+ [41532326444](https://github.com/jchewitt/weftlet/actions/runs/41532326444) pass Linux and Windows.
38
+ Linux passes 38 core + 985 toolkit tests (1 optional ontology integration skipped), architecture,
39
+ inventory, licenses and fresh packed-consumer smoke. Windows passes 38 core + 878 selected tests,
40
+ inventory and licenses. The final local packed-consumer rerun also passes.
41
+
42
+ At `d1498a4`, hosted push run
43
+ [41542970652](https://github.com/jchewitt/weftlet/actions/runs/41542970652) passes both platforms.
44
+ The duplicate PR run
45
+ [41542976891](https://github.com/jchewitt/weftlet/actions/runs/41542976891) passes Linux; Windows
46
+ passes 877 selected tests but times out in the four-command failure-reporting test at its
47
+ 30-second aggregate deadline, then encounters cleanup EBUSY with the child still using its cwd.
48
+ The harness now retains each 20-second command limit, allows 100 seconds for the combined
49
+ sequence and fixture setup, and aborts/waits for child closure before cleanup. All four local
50
+ failure-reporting tests pass; this changes no production timeout or assertion.
51
+ At `4bc41fd1`, both subsequent hosted runs
52
+ [41545800490](https://github.com/jchewitt/weftlet/actions/runs/41545800490) and
53
+ [41545807452](https://github.com/jchewitt/weftlet/actions/runs/41545807452) pass Linux and Windows,
54
+ including the corrected test and fresh packed-consumer installation.
55
+
56
+ The optional broader test-source TypeScript check still reports existing diagnostics outside
57
+ the production build. An isolated archive of the base commit has the same 42 diagnostics after
58
+ normalizing checkout paths and ignoring its two absent-build-output errors; this patch adds none.
59
+
60
+ The [public capacity result](benchmarks/large-graph-2026-11-02/result.json) passes the complete
61
+ hydrate → strict validation → persist/read → unchanged repeat → clean comparison lifecycle.
62
+ It uses 1,100 TypeScript modules with 40 functions each, actual SCIP, spec files and 18
63
+ deterministic prose extractions, with zero live model calls. The run produced **554,516 facts /
64
+ 143,044 nodes**, a **1,021,806,445-byte snapshot**, and a **5,331,139,680-byte N-Quads export**
65
+ containing 21,522,256 quads. Every exported record was parsed and matched by ID and canonical
66
+ digest. Bounded real SPARQL and the complete record projection pass without dropping facts.
67
+
68
+ All three full strict SHACL passes succeed: cold **2,099,768 ms**, repeat **2,225,093 ms**, and
69
+ clean **2,096,881 ms** (about 35–37 minutes each). Both repeat and clean reuse all 18 artifacts
70
+ with zero further extraction. Snapshot identity on repeat, logical/content digests, exact
71
+ provenance, extractor versions, cache keys, original snapshot bytes and all semantic artifact
72
+ bytes are retained. The complete harness took 8,841,003 ms, including three indexing/validation
73
+ runs, large-file round trips, independent assertions and fixture cleanup. This establishes
74
+ capacity/correctness, **not a 30-minute hydration budget** or private-workload performance.
75
+ Other local checks overlapped part of the run; it is not an isolated comparative benchmark.
76
+
77
+ An independent read of the saved 554,516 facts confirms that 938,267,270 characters of logical
78
+ canonical encoding hash to
79
+ `e6965407b8a8d76ed6d2d7ef8d2daa0297059b5fb327723cd49e73233865c001`
80
+ under both the legacy per-record encoding with exact array separators and `stableJsonDigest`.
81
+ That assertion is now also in the lifecycle harness; its scaled CI fixture passes.
82
+ The long-running capacity process predates the final negative-path shape/error guards and
83
+ harness corrections. Those changes pass separately in the hosted checks above; this is not
84
+ claimed as an exact-final-head large-corpus rerun. Native Windows field acceptance, live provider
85
+ quality and optional openCAESAR runtime remain separate checks.
86
+
3
87
  ## 0.7.0 acceptance release disposition, 2026-11-02
4
88
 
5
89
  The maintainer authorized publishing the accumulated B1–B5 implementation for live TradePilot
@@ -0,0 +1,156 @@
1
+ {
2
+ "schemaVersion": 1,
3
+ "workload": "Generated committed public TypeScript, actual SCIP, deterministic prose provider",
4
+ "environment": {
5
+ "os": "Linux x64",
6
+ "node": "24.19.0",
7
+ "pnpm": "10.18.2",
8
+ "nodeHeapLimitMiB": 8192,
9
+ "cpuQuota": 4,
10
+ "cgroupMemoryBytes": 34359738368
11
+ },
12
+ "command": "NODE_OPTIONS=--max-old-space-size=8192 pnpm exec tsx scripts/large-graph/run.ts 1100 40 --require-large",
13
+ "qualificationScope": "The capacity process started before the final negative-path shape/error guards and test-harness corrections. Those changes, and the newly embedded legacy-hash assertion, passed separately in hosted Linux/Windows checks at 4bc41fd1eee4169934fc8e50960bb60b7c8ff617. This is not an exact-final-head large-corpus rerun.",
14
+ "result": {
15
+ "status": "passed",
16
+ "elapsedMs": 8841003,
17
+ "modules": 1100,
18
+ "functions": 40,
19
+ "factCount": 554516,
20
+ "nodeCount": 143044,
21
+ "bytes": 1021806445,
22
+ "logicalCharacters": 938267270,
23
+ "rdfBytes": 5331139680,
24
+ "quadCount": 21522256,
25
+ "contentDigest": "2f75e3dbc16e7ca856d177264046034f40a1560b8a39e5df8b998c653c7b8e81",
26
+ "immutableDigest": "4f48702636a5499e2bfde70f2cca189d2d67836531180685515b4eb3d10ca3f2",
27
+ "providerCalls": 0,
28
+ "deterministicProviderExtractions": 18,
29
+ "repeatReused": 18,
30
+ "cleanReused": 18,
31
+ "extractorVersions": {
32
+ "assembly": "standalone-graph-v12",
33
+ "repository-contract-extractor": "1.2.0",
34
+ "scip-symbol-extractor": "2.1.0",
35
+ "typescript-call-extractor": "1.0.0",
36
+ "repository-node-identity": "1",
37
+ "document-linker": "2.0.0",
38
+ "document-code-binding-extractor": "1.0.0",
39
+ "document-obligation-extractor": "1.0.0",
40
+ "semantic": "1.1.0",
41
+ "scip": "0.4.0",
42
+ "ontology": "rdf-record-v1",
43
+ "repository-file-extractor": "2.0.0"
44
+ }
45
+ },
46
+ "legacyCanonicalDigestParity": {
47
+ "status": "passed",
48
+ "facts": 554516,
49
+ "canonicalCharacters": 938267270,
50
+ "digest": "e6965407b8a8d76ed6d2d7ef8d2daa0297059b5fb327723cd49e73233865c001",
51
+ "method": "legacy per-record stableJson plus exact array separators vs stableJsonDigest"
52
+ },
53
+ "completedPhases": [
54
+ {
55
+ "totalElapsedMs": 112233,
56
+ "phase": "validate-identities",
57
+ "status": "completed",
58
+ "factCount": 554516,
59
+ "nodeCount": 143044,
60
+ "elapsedMs": 1118
61
+ },
62
+ {
63
+ "totalElapsedMs": 138845,
64
+ "phase": "build-rdf-evidence",
65
+ "status": "completed",
66
+ "factCount": 554516,
67
+ "nodeCount": 143044,
68
+ "elapsedMs": 1
69
+ },
70
+ {
71
+ "totalElapsedMs": 344926,
72
+ "phase": "build-validation-union",
73
+ "status": "completed",
74
+ "factCount": 554516,
75
+ "elapsedMs": 206080,
76
+ "nodeCount": 143044
77
+ },
78
+ {
79
+ "totalElapsedMs": 2444694,
80
+ "phase": "validate-shacl",
81
+ "status": "completed",
82
+ "factCount": 554516,
83
+ "elapsedMs": 2099768,
84
+ "nodeCount": 143044
85
+ },
86
+ {
87
+ "totalElapsedMs": 3525345,
88
+ "phase": "validate-identities",
89
+ "status": "completed",
90
+ "factCount": 554516,
91
+ "nodeCount": 143044,
92
+ "elapsedMs": 754
93
+ },
94
+ {
95
+ "totalElapsedMs": 3555601,
96
+ "phase": "build-rdf-evidence",
97
+ "status": "completed",
98
+ "factCount": 554516,
99
+ "nodeCount": 143044,
100
+ "elapsedMs": 1
101
+ },
102
+ {
103
+ "totalElapsedMs": 3783233,
104
+ "phase": "build-validation-union",
105
+ "status": "completed",
106
+ "factCount": 554516,
107
+ "elapsedMs": 227631,
108
+ "nodeCount": 143044
109
+ },
110
+ {
111
+ "totalElapsedMs": 6008326,
112
+ "phase": "validate-shacl",
113
+ "status": "completed",
114
+ "factCount": 554516,
115
+ "elapsedMs": 2225093,
116
+ "nodeCount": 143044
117
+ },
118
+ {
119
+ "totalElapsedMs": 6340553,
120
+ "phase": "validate-identities",
121
+ "status": "completed",
122
+ "factCount": 554516,
123
+ "nodeCount": 143044,
124
+ "elapsedMs": 977
125
+ },
126
+ {
127
+ "totalElapsedMs": 6370727,
128
+ "phase": "build-rdf-evidence",
129
+ "status": "completed",
130
+ "factCount": 554516,
131
+ "nodeCount": 143044,
132
+ "elapsedMs": 0
133
+ },
134
+ {
135
+ "totalElapsedMs": 6575068,
136
+ "phase": "build-validation-union",
137
+ "status": "completed",
138
+ "factCount": 554516,
139
+ "elapsedMs": 204340,
140
+ "nodeCount": 143044
141
+ },
142
+ {
143
+ "totalElapsedMs": 8671949,
144
+ "phase": "validate-shacl",
145
+ "status": "completed",
146
+ "factCount": 554516,
147
+ "elapsedMs": 2096881,
148
+ "nodeCount": 143044
149
+ }
150
+ ],
151
+ "limitations": [
152
+ "Capacity and correctness regression; not a 30-minute hydration qualification.",
153
+ "Other local checks overlapped part of the run; this is not an isolated comparative performance benchmark.",
154
+ "No private consumer source, native Windows field acceptance, live model-quality evaluation, or optional openCAESAR runtime is claimed."
155
+ ]
156
+ }