@weftlet/toolkit 0.7.0 → 0.7.2

package/dist/graph/rdf-records.js

@@ -0,0 +1,90 @@
+import { DataFactory, Store } from 'n3';
+import { contextPackEvidenceAuthority } from './authority.js';
+import { stableJson } from '../infrastructure/util.js';
+import { CG, RDF, XSD } from './shacl-shapes.js';
+import { addBindingEvidence } from './binding-rdf.js';
+import { addCallEvidence } from './call-rdf.js';
+import { addObligationEvidence } from './obligation-rdf.js';
+import { addCodeEvidence, addCodeNodeEvidence } from './code-rdf.js';
+const { namedNode: iri, literal } = DataFactory;
+export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
+export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
+export function addFactRecord(dataset, fact, ordinal) {
+    const record = iri(factIri(fact.id));
+    const subject = iri(nodeIri(fact.subject.id));
+    const object = iri(nodeIri(fact.object.id));
+    const predicate = iri(CG + fact.predicate);
+    dataset.addQuad(record, iri(RDF + 'type'), iri(RDF + 'Statement'));
+    dataset.addQuad(record, iri(CG + 'ordinal'), literal(String(ordinal), iri(XSD + 'integer')));
+    for (const [p, value] of [['subject', subject], ['predicate', predicate], ['object', object]]) {
+        dataset.addQuad(record, iri(RDF + p), value);
+    }
+    for (const [p, value] of Object.entries({
+        authority: contextPackEvidenceAuthority(fact), path: fact.provenance.path,
+        commit: fact.provenance.commitSha, sourceDigest: fact.provenance.digest,
+        extractor: `${fact.extractor.id}@${fact.extractor.version}`,
+        snapshotId: fact.snapshotId, record: stableJson(fact),
+        ...(fact.sourceAuthority ? { sourceAuthority: fact.sourceAuthority } : {}),
+        ...(fact.extractionReview ? { extractionReview: fact.extractionReview } : {}),
+        ...(fact.sourceRole !== undefined ? { sourceRole: fact.sourceRole } : {}),
+        ...(fact.sourcePolicyDigest !== undefined ? { sourcePolicyDigest: fact.sourcePolicyDigest } : {}),
+    }))
+        dataset.addQuad(record, iri(CG + p), literal(value));
+    if (fact.sourceOwnership !== undefined) {
+        dataset.addQuad(record, iri(CG + 'ownershipStatus'), literal(fact.sourceOwnership.status));
+        for (const ownerDocument of fact.sourceOwnership.ownerDocuments) {
+            dataset.addQuad(record, iri(CG + 'ownerDocument'), literal(ownerDocument));
+        }
+        for (const mappingId of fact.sourceOwnership.mappingIds) {
+            dataset.addQuad(record, iri(CG + 'ownershipMapping'), literal(mappingId));
+        }
+    }
+    for (const [p, value] of [['startLine', fact.provenance.startLine], ['endLine', fact.provenance.endLine]]) {
+        dataset.addQuad(record, iri(CG + p), literal(String(value), iri(XSD + 'integer')));
+    }
+    dataset.addQuad(record, iri(CG + 'confidence'), literal(String(fact.confidence), iri(XSD + 'decimal')));
+    addCodeEvidence(dataset, record, fact, nodeIri, false);
+    addBindingEvidence(dataset, record, fact, factIri);
+    addCallEvidence(dataset, record, fact, factIri);
+    addObligationEvidence(dataset, record, fact);
+}
+export function addNodeRdf(dataset, node) {
+    const term = iri(nodeIri(node.id));
+    dataset.addQuad(term, iri(RDF + 'type'), iri(CG + 'Node'));
+    dataset.addQuad(term, iri(CG + 'identifier'), literal(node.id));
+    dataset.addQuad(term, iri(CG + 'kind'), literal(node.kind));
+    dataset.addQuad(term, iri(CG + 'label'), literal(node.label));
+    addCodeNodeEvidence(dataset, node, nodeIri);
+}
+export function addAttributes(dataset, node, graph = DataFactory.defaultGraph()) {
+    for (const [key, value] of Object.entries(node.attributes ?? {})) {
+        const datatype = typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'decimal' : 'string';
+        dataset.addQuad(iri(nodeIri(node.id)), iri(`${CG}attribute/${encodeURIComponent(key)}`), literal(String(value), iri(XSD + datatype)), graph);
+    }
+}
+export function addFactEdges(dataset, fact) {
+    const record = iri(factIri(fact.id));
+    dataset.addQuad(iri(nodeIri(fact.subject.id)), iri(CG + fact.predicate), iri(nodeIri(fact.object.id)), record);
+    for (const node of [fact.subject, fact.object])
+        addAttributes(dataset, node, record);
+}
+export function addManifest(dataset, snapshot) {
+    dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'manifest'), literal(stableJson(snapshot.manifest)));
+    for (const scope of snapshot.manifest.scope ?? []) {
+        dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'scope'), literal(scope));
+    }
+    if (snapshot.manifest.configurationDigest) {
+        dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'configurationDigest'), literal(snapshot.manifest.configurationDigest));
+    }
+}
+export function evidenceDataset(snapshot) {
+    const dataset = new Store();
+    for (const [ordinal, fact] of snapshot.facts.entries()) {
+        addFactRecord(dataset, fact, ordinal);
+        addFactEdges(dataset, fact);
+        for (const node of [fact.subject, fact.object])
+            addNodeRdf(dataset, node);
+    }
+    addManifest(dataset, snapshot);
+    return dataset;
+}

package/dist/graph/rdf.d.ts

@@ -1,29 +1,29 @@
+import type { Writable } from 'node:stream';
 import type { GraphFact, GraphSnapshot } from "@weftlet/core";
 import { type GraphValidationProgress } from './validation-progress.js';
 export { CG } from './shacl-shapes.js';
-export declare const nodeIri: (id: string) => string;
-export declare const factIri: (id: string) => string;
+export { factIri, nodeIri } from './rdf-records.js';
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export declare class RdfEvidenceGraph {
     readonly snapshot: GraphSnapshot;
-    private readonly dataset;
+    private indexed?;
     constructor(snapshot: GraphSnapshot);
     validate(options?: {
         readonly onProgress?: (event: GraphValidationProgress) => void;
     }): Promise<{
         conforms: boolean;
-        issues: {
-            category: "extraction-gap";
-            focus: string;
-            path: string;
-            message: string;
-        }[];
+        issues: import("./partitioned-shacl.js").ShaclIssue[];
     }>;
     select(sparql: string, maxRows?: number): Promise<{
         rows: Record<string, string>[];
         truncated: boolean;
     }>;
-    /** The traversal projection is read through SPARQL, not a separate fact source. */
+    /** Lossless optimized projection, checked for equivalence with the public SPARQL path. */
     facts(): Promise<readonly GraphFact[]>;
+    /** Complete N-Quads in bounded groups; RDF triple order is not a snapshot identity. */
+    serializeChunks(): IterableIterator<string>;
+    /** Export large datasets with backpressure, without a whole-dataset string/quad array. */
+    serializeTo(destination: Writable): Promise<void>;
+    /** Compatibility convenience for small exports; use serializeTo for large graphs. */
     serialize(): Promise<string>;
 }

package/dist/graph/rdf.js

@@ -1,105 +1,48 @@
 import { QueryEngine } from '@comunica/query-sparql';
-import { DataFactory, Store, Writer } from 'n3';
-import SHACLValidator from 'rdf-validate-shacl';
-import { contextPackEvidenceAuthority } from "./authority.js";
-import { stableJson } from "../infrastructure/util.js";
+import { constants } from 'node:buffer';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import { Writer } from 'n3';
 import { createRequire } from 'node:module';
-import { CG, RDF, XSD, evidenceShapes } from './shacl-shapes.js';
+import { CG } from './shacl-shapes.js';
 import { measureValidationPhase } from './validation-progress.js';
-import { addBindingEvidence, bindingEvidenceIssues } from './binding-rdf.js';
-import { addCallEvidence, callEvidenceIssues } from './call-rdf.js';
-import { addObligationEvidence, obligationEvidenceIssues } from './obligation-rdf.js';
-import { addCodeEvidence, codeEvidenceIssues, invalidCodeRange } from './code-rdf.js';
-const { namedNode: iri, literal } = DataFactory;
+import { bindingEvidenceIssues } from './binding-rdf.js';
+import { callEvidenceIssues } from './call-rdf.js';
+import { obligationEvidenceIssues } from './obligation-rdf.js';
+import { codeEvidenceIssues, invalidCodeRange } from './code-rdf.js';
 export { CG } from './shacl-shapes.js';
-export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
-export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
+import { factIri } from './rdf-records.js';
+import { RdfPartitions } from './rdf-partitions.js';
+import { validatePartitions } from './partitioned-shacl.js';
+import { projectRdfFacts } from './rdf-projection.js';
+export { factIri, nodeIri } from './rdf-records.js';
 const engine = new QueryEngine();
 const SparqlParser = createRequire(import.meta.url)('sparqljs').Parser;
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export class RdfEvidenceGraph {
     snapshot;
-    dataset = new Store();
+    indexed;
     constructor(snapshot) {
         this.snapshot = snapshot;
-        for (const [ordinal, fact] of snapshot.facts.entries()) {
-            const record = iri(factIri(fact.id));
-            const subject = iri(nodeIri(fact.subject.id));
-            const object = iri(nodeIri(fact.object.id));
-            const predicate = iri(CG + fact.predicate);
-            this.dataset.addQuad(subject, predicate, object, record);
-            this.dataset.addQuad(record, iri(RDF + 'type'), iri(RDF + 'Statement'));
-            this.dataset.addQuad(record, iri(CG + 'ordinal'), literal(String(ordinal), iri(XSD + 'integer')));
-            for (const [p, value] of [['subject', subject], ['predicate', predicate], ['object', object]]) {
-                this.dataset.addQuad(record, iri(RDF + p), value);
-            }
-            for (const [p, value] of Object.entries({
-                authority: contextPackEvidenceAuthority(fact), path: fact.provenance.path,
-                commit: fact.provenance.commitSha, sourceDigest: fact.provenance.digest,
-                extractor: `${fact.extractor.id}@${fact.extractor.version}`,
-                snapshotId: fact.snapshotId, record: stableJson(fact),
-                ...(fact.sourceAuthority ? { sourceAuthority: fact.sourceAuthority } : {}),
-                ...(fact.extractionReview ? { extractionReview: fact.extractionReview } : {}),
-                ...(fact.sourceRole !== undefined ? { sourceRole: fact.sourceRole } : {}),
-                ...(fact.sourcePolicyDigest !== undefined ? { sourcePolicyDigest: fact.sourcePolicyDigest } : {}),
-            }))
-                this.dataset.addQuad(record, iri(CG + p), literal(value));
-            if (fact.sourceOwnership !== undefined) {
-                this.dataset.addQuad(record, iri(CG + 'ownershipStatus'), literal(fact.sourceOwnership.status));
-                for (const ownerDocument of fact.sourceOwnership.ownerDocuments) {
-                    this.dataset.addQuad(record, iri(CG + 'ownerDocument'), literal(ownerDocument));
-                }
-                for (const mappingId of fact.sourceOwnership.mappingIds) {
-                    this.dataset.addQuad(record, iri(CG + 'ownershipMapping'), literal(mappingId));
-                }
-            }
-            for (const [p, value] of [['startLine', fact.provenance.startLine], ['endLine', fact.provenance.endLine]]) {
-                this.dataset.addQuad(record, iri(CG + p), literal(String(value), iri(XSD + 'integer')));
-            }
-            this.dataset.addQuad(record, iri(CG + 'confidence'), literal(String(fact.confidence), iri(XSD + 'decimal')));
-            addCodeEvidence(this.dataset, record, fact, nodeIri);
-            addBindingEvidence(this.dataset, record, fact, factIri);
-            addCallEvidence(this.dataset, record, fact, factIri);
-            addObligationEvidence(this.dataset, record, fact);
-            for (const node of [fact.subject, fact.object]) {
-                const term = iri(nodeIri(node.id));
-                this.dataset.addQuad(term, iri(RDF + 'type'), iri(CG + 'Node'));
-                this.dataset.addQuad(term, iri(CG + 'identifier'), literal(node.id));
-                this.dataset.addQuad(term, iri(CG + 'kind'), literal(node.kind));
-                this.dataset.addQuad(term, iri(CG + 'label'), literal(node.label));
-                for (const [key, value] of Object.entries(node.attributes ?? {})) {
-                    const datatype = typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'decimal' : 'string';
-                    this.dataset.addQuad(term, iri(`${CG}attribute/${encodeURIComponent(key)}`), literal(String(value), iri(XSD + datatype)), record);
-                }
-            }
-        }
-        this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'manifest'), literal(stableJson(snapshot.manifest)));
-        for (const scope of snapshot.manifest.scope ?? []) {
-            this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'scope'), literal(scope));
-        }
-        if (snapshot.manifest.configurationDigest) {
-            this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'configurationDigest'), literal(snapshot.manifest.configurationDigest));
-        }
     }
     async validate(options = {}) {
-        // These graph-independent shapes already read the dataset as a union. Index
-        // that union once so every property lookup avoids scanning one graph per fact.
-        // Keep every triple; the original named dataset retains assertion provenance
-        // and remains the source for queries and serialization.
         const counts = { factCount: this.snapshot.facts.length };
-        const validationGraph = await measureValidationPhase('build-validation-union', counts, options.onProgress, () => {
-            const union = new Store();
-            for (const { subject, predicate, object } of this.dataset)
-                union.addQuad(subject, predicate, object);
-            return union;
+        const { partitions, types } = await measureValidationPhase('build-validation-union', counts, options.onProgress, () => {
+            const started = performance.now();
+            const partitions = new RdfPartitions(this.snapshot);
+            return { partitions, types: partitions.types(completed => {
+                    if (completed % 10 === 0)
+                        options.onProgress?.({ phase: 'build-validation-union', status: 'progress',
+                            elapsedMs: Math.round(performance.now() - started), ...counts, completed, total: partitions.count });
+                }) };
         });
-        const report = await measureValidationPhase('validate-shacl', counts, options.onProgress, () => new SHACLValidator(new Store(evidenceShapes)).validate(validationGraph));
-        const issues = report.results.map((result) => ({
-            category: 'extraction-gap',
-            focus: result.focusNode?.value,
-            path: result.path?.value,
-            message: result.message.map((message) => message.value).join('; '),
+        const started = performance.now();
+        const report = await measureValidationPhase('validate-shacl', counts, options.onProgress, () => validatePartitions(partitions, types, completed => {
+            if (completed % 10 === 0)
+                options.onProgress?.({ phase: 'validate-shacl', status: 'progress',
+                    elapsedMs: Math.round(performance.now() - started), ...counts, completed, total: partitions.count });
         }));
+        const issues = report.issues;
         const byId = new Map(this.snapshot.facts.map(fact => [fact.id, fact]));
         for (const fact of this.snapshot.facts) {
             for (const message of obligationEvidenceIssues(fact, this.snapshot.manifest.repositoryId))
@@ -140,7 +83,12 @@ export class RdfEvidenceGraph {
         if (ast.type !== 'query' || ast.queryType !== 'SELECT' || ast.from || hasService(ast)) {
             throw new Error('Only local read queries are supported; remote datasets and updates are disabled');
         }
-        const stream = await engine.queryBindings(sparql, { sources: [this.dataset], unionDefaultGraph: true });
+        const indexed = this.indexed ??= new RdfPartitions(this.snapshot);
+        const source = {
+            match: (s, p, o, g) => Readable.from(indexed.match(s, p, o, g)),
+            countQuads: (s, p, o, g) => indexed.countMatches(s, p, o, g),
+        };
+        const stream = await engine.queryBindings(sparql, { sources: [{ type: 'rdfjs', value: source }], unionDefaultGraph: true });
         const rows = [];
         let truncated = false;
         for await (const binding of stream) {
@@ -153,18 +101,39 @@ export class RdfEvidenceGraph {
         }
         return { rows, truncated };
     }
-    /** The traversal projection is read through SPARQL, not a separate fact source. */
+    /** Lossless optimized projection, checked for equivalence with the public SPARQL path. */
     async facts() {
-        const result = await this.select(`SELECT ?record WHERE {
-      ?fact <${CG}record> ?record; <${CG}ordinal> ?ordinal
-    } ORDER BY ?ordinal`, 100_000);
-        if (result.truncated)
-            throw new Error('Graph exceeds the 100000-fact projection bound');
-        return result.rows.map((row) => JSON.parse(row.record));
+        return projectRdfFacts(this.snapshot.facts);
     }
-    serialize() {
+    /** Complete N-Quads in bounded groups; RDF triple order is not a snapshot identity. */
+    *serializeChunks() {
         const writer = new Writer({ format: 'N-Quads' });
-        writer.addQuads(this.dataset.getQuads(null, null, null, null));
-        return new Promise((resolve, reject) => writer.end((error, text) => error ? reject(error) : resolve(text)));
+        let buffer = '';
+        for (const batch of new RdfPartitions(this.snapshot).named())
+            for (const quad of batch) {
+                buffer += writer.quadToString(quad.subject, quad.predicate, quad.object, quad.graph);
+                if (buffer.length >= 64 * 1024) {
+                    yield buffer;
+                    buffer = '';
+                }
+            }
+        if (buffer)
+            yield buffer;
+    }
+    /** Export large datasets with backpressure, without a whole-dataset string/quad array. */
+    async serializeTo(destination) {
+        await pipeline(this.serializeChunks(), destination);
+    }
+    /** Compatibility convenience for small exports; use serializeTo for large graphs. */
+    async serialize() {
+        const chunks = [];
+        let length = 0;
+        for (const chunk of this.serializeChunks()) {
+            length += chunk.length;
+            if (length > constants.MAX_STRING_LENGTH)
+                throw new Error('RDF exceeds the JavaScript string limit; use serializeTo() or serializeChunks()');
+            chunks.push(chunk);
+        }
+        return chunks.join('');
     }
 }

package/dist/graph/validation-progress.d.ts

@@ -1,7 +1,10 @@
 export type GraphValidationPhase = 'validate-identities' | 'build-rdf-evidence' | 'build-validation-union' | 'validate-shacl';
 export interface GraphValidationProgress {
     readonly phase: GraphValidationPhase;
-    readonly status: 'started' | 'completed' | 'failed';
+    readonly status: 'started' | 'completed' | 'failed' | 'progress';
+    /** Completed bounded RDF subject groups, when available. */
+    readonly completed?: number;
+    readonly total?: number;
     /** Wall time for this phase, not a completion estimate or in-phase heartbeat. */
     readonly elapsedMs: number;
     readonly factCount: number;

package/dist/infrastructure/json-digest.d.ts

@@ -1,2 +1,3 @@
+import { persistedJsonDigest } from './util.js';
 /** Identity of persisted JSON, excluding undefined optional fields that JSON cannot retain. */
-export declare const jsonDigest: (value: unknown) => string;
+export declare const jsonDigest: typeof persistedJsonDigest;

package/dist/infrastructure/json-digest.js

@@ -1,3 +1,3 @@
-import { sha256, stableJson } from './util.js';
+import { persistedJsonDigest } from './util.js';
 /** Identity of persisted JSON, excluding undefined optional fields that JSON cannot retain. */
-export const jsonDigest = (value) => sha256(stableJson(JSON.parse(JSON.stringify(value))));
+export const jsonDigest = persistedJsonDigest;

package/dist/interfaces/cli.js

@@ -103,7 +103,7 @@ function formatHydrationPlan(plan) {
 async function main() {
     const args = process.argv.slice(2);
     if (args.includes('--help') || !args.length) {
-        process.stdout.write('weftlet mappings plan|mappings propose|mappings read|documents plan|init|config show|config validate|doctor|hydrate|cache adopt|overlay|impact scan|graph neighbors|graph paths|graph compatibility|graph coverage|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nmappings plan|propose --request <json-file> [--allow-model for propose] previews or runs one opt-in proposal; mappings read --snapshot <id> --key <key> reads saved unreviewed results.\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ngraph neighbors --snapshot <id> --node <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-facts <1..200>] [--continuation <token>] [--json] reads one-hop saved evidence; repeat --predicate to select multiple relations.\ngraph paths --snapshot <id> --from <exact-id> --to <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-depth <1..32>] [--max-paths <1..20>] [--max-steps <1..10000>] [--continuation <token>] [--json] explores cited simple paths; incomplete searches report frontier and limits.\ngraph coverage --snapshot <id> [--project <exact-package-root>] [--max-items <1..200>] [--continuation <token>] [--json] pages saved package/file coverage; absence or zero references is not proof of no impact.\ngraph compatibility --snapshot <id> --producer-fact <assertion-id> --consumer-fact <assertion-id> [--json] compares two documented numeric obligations; advisory only, with explicit unknowns and no implementation or CI verdict.\ndocuments plan --entry <document> [--entry <document>] [--root <path>] [--document-policy <json-file>] [--ref <commit>] [--json] previews prospective committed link coverage; it does not change hydration inputs or estimate model/cache work. Source classification/ownership completeness is separate from traversal readiness. The preview policy is an operation input. Set documentTraversal in configuration to activate traversal hydration; hydrate --plan describes its execution inputs. Safety limits: --max-read-documents, --max-read-bytes, --max-document-bytes.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Traversal coverage and role ambiguity block new model work; direct-source selection retains its existing behavior.\nhydrate --json --quiet emits structured semantic failures on stderr with no progress lines; failure stdout stays empty. hydrate --plan maps opaque source IDs to local document paths.\nhydrate|overlay --validation-details explicitly includes local node/fact identities, extractor names and source citations in identity-conflict errors; default reports use opaque references. This flag writes no diagnostic file. Use --json --quiet for structured identity-conflict stderr and empty failure stdout. Validation timings report phase boundaries, not a live heartbeat or percentage.\ncache adopt --from-config <previous.json> [--config <current.json>] [--ref <commit>] [--dry-run] returns JSON and copies only validated compatible artifacts, without model calls or snapshot writes. --dry-run performs no cache writes. Agent configurations accept the same --owner-doc, --task and --capability selection as hydrate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
+        process.stdout.write('weftlet mappings plan|mappings propose|mappings read|documents plan|init|config show|config validate|doctor|hydrate|cache adopt|overlay|impact scan|graph neighbors|graph paths|graph compatibility|graph coverage|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nmappings plan|propose --request <json-file> [--allow-model for propose] previews or runs one opt-in proposal; mappings read --snapshot <id> --key <key> reads saved unreviewed results.\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ngraph neighbors --snapshot <id> --node <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-facts <1..200>] [--continuation <token>] [--json] reads one-hop saved evidence; repeat --predicate to select multiple relations.\ngraph paths --snapshot <id> --from <exact-id> --to <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-depth <1..32>] [--max-paths <1..20>] [--max-steps <1..10000>] [--continuation <token>] [--json] explores cited simple paths; incomplete searches report frontier and limits.\ngraph coverage --snapshot <id> [--project <exact-package-root>] [--max-items <1..200>] [--continuation <token>] [--json] pages saved package/file coverage; absence or zero references is not proof of no impact.\ngraph compatibility --snapshot <id> --producer-fact <assertion-id> --consumer-fact <assertion-id> [--json] compares two documented numeric obligations; advisory only, with explicit unknowns and no implementation or CI verdict.\ndocuments plan --entry <document> [--entry <document>] [--root <path>] [--document-policy <json-file>] [--ref <commit>] [--json] previews prospective committed link coverage; it does not change hydration inputs or estimate model/cache work. Source classification/ownership completeness is separate from traversal readiness. The preview policy is an operation input. Set documentTraversal in configuration to activate traversal hydration; hydrate --plan describes its execution inputs. Safety limits: --max-read-documents, --max-read-bytes, --max-document-bytes.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Traversal coverage and role ambiguity block new model work; direct-source selection retains its existing behavior.\nhydrate --json --quiet emits structured semantic failures on stderr with no progress lines; failure stdout stays empty. hydrate --plan maps opaque source IDs to local document paths.\nhydrate|overlay --validation-details explicitly includes local node/fact identities, extractor names and source citations in identity-conflict errors; default reports use opaque references. This flag writes no diagnostic file. Use --json --quiet for structured identity-conflict stderr and empty failure stdout. Validation reports phase boundaries, elapsed time and completed groups; group counts are not completion-time estimates.\ncache adopt --from-config <previous.json> [--config <current.json>] [--ref <commit>] [--dry-run] returns JSON and copies only validated compatible artifacts, without model calls or snapshot writes. --dry-run performs no cache writes. Agent configurations accept the same --owner-doc, --task and --capability selection as hydrate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
         return;
     }
     if (await runDocumentCoverage(args, defaultConfigPath))

package/dist/storage/storage.d.ts

@@ -4,7 +4,10 @@ export declare class FileSnapshotStore implements SnapshotStore {
     constructor(root: string);
     write(snapshot: GraphSnapshot): Promise<void>;
     read(snapshotId: string): Promise<GraphSnapshot | undefined>;
+    private readManifest;
     readHead(headName: string): Promise<GraphSnapshot | undefined>;
+    readHeadManifest(headName: string): Promise<GraphSnapshot['manifest'] | undefined>;
+    private headId;
     updateHead(headName: string, snapshotId: string): Promise<void>;
 }
 export declare class SemanticArtifactStore {

package/dist/storage/storage.js

@@ -1,11 +1,12 @@
 // Modified for standalone extraction; see docs/extraction-manifest.json.
-import { mkdir, readFile, writeFile, link, rm } from 'node:fs/promises';
+import { mkdir, writeFile, link, rm } from 'node:fs/promises';
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { atomicRename } from './atomic-rename.js';
 import { physicalArtifactKey, temporaryArtifactPath } from '../infrastructure/artifact-paths.js';
+import { readJsonFile, readJsonProperty, writeJsonFile } from './streaming-json.js';
 async function readJson(path) {
     try {
-        return JSON.parse(await readFile(path, 'utf8'));
+        return await readJsonFile(path);
     }
     catch (error) {
         if (error.code === 'ENOENT')
@@ -13,8 +14,8 @@ async function readJson(path) {
         throw error;
     }
 }
-async function readWithLegacyPath(path, legacyPath) {
-    const current = await readJson(path);
+async function readWithLegacyPath(path, legacyPath, read = (readJson)) {
+    const current = await read(path);
     if (current !== undefined || path === legacyPath)
         return current;
     // No old file could have this component on supported filesystems. Do not attempt an
@@ -23,7 +24,7 @@ async function readWithLegacyPath(path, legacyPath) {
         return undefined;
     if (process.platform === 'win32' && /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(basename(legacyPath)))
         return undefined;
-    return readJson(legacyPath);
+    return read(legacyPath);
 }
 async function atomicJson(path, value) {
     await mkdir(dirname(path), { recursive: true });
@@ -65,9 +66,9 @@ export class FileSnapshotStore {
     }
     async write(snapshot) {
         const destination = snapshotPath(this.root, snapshot.manifest.snapshotId);
-        const prior = await this.read(snapshot.manifest.snapshotId);
+        const prior = await this.readManifest(snapshot.manifest.snapshotId);
         if (prior) {
-            if (prior.manifest.contentDigest !== snapshot.manifest.contentDigest) {
+            if (prior.contentDigest !== snapshot.manifest.contentDigest) {
                 throw new Error(`Refusing to overwrite immutable snapshot ${snapshot.manifest.snapshotId} with different content`);
             }
             return; // Preserve an existing legacy file without copying or rewriting its bytes.
@@ -75,15 +76,15 @@ export class FileSnapshotStore {
         await mkdir(dirname(destination), { recursive: true });
         const temporary = temporaryArtifactPath(destination);
         try {
-            await writeFile(temporary, JSON.stringify(snapshot), { flag: 'wx' });
+            await writeJsonFile(temporary, snapshot);
             // Linking a complete file publishes atomically without ever replacing a winner.
             await link(temporary, destination);
         }
         catch (error) {
             if (error.code !== 'EEXIST')
                 throw error;
-            const existing = await this.read(snapshot.manifest.snapshotId);
-            if (existing?.manifest.contentDigest !== snapshot.manifest.contentDigest) {
+            const existing = await this.readManifest(snapshot.manifest.snapshotId);
+            if (existing?.contentDigest !== snapshot.manifest.contentDigest) {
                 throw new Error(`Refusing to overwrite immutable snapshot ${snapshot.manifest.snapshotId} with different content`);
             }
         }
@@ -94,13 +95,36 @@ export class FileSnapshotStore {
     read(snapshotId) {
         return readWithLegacyPath(snapshotPath(this.root, snapshotId), snapshotPath(this.root, snapshotId, true));
     }
+    readManifest(snapshotId) {
+        return readWithLegacyPath(snapshotPath(this.root, snapshotId), snapshotPath(this.root, snapshotId, true), async (path) => {
+            try {
+                const manifest = await readJsonProperty(path, 'manifest');
+                if (!manifest || typeof manifest.contentDigest !== 'string')
+                    throw Error('Invalid snapshot manifest');
+                return manifest;
+            }
+            catch (error) {
+                if (error.code === 'ENOENT')
+                    return undefined;
+                throw error;
+            }
+        });
+    }
     async readHead(headName) {
+        const snapshotId = await this.headId(headName);
+        return snapshotId ? this.read(snapshotId) : undefined;
+    }
+    async readHeadManifest(headName) {
+        const snapshotId = await this.headId(headName);
+        return snapshotId ? this.readManifest(snapshotId) : undefined;
+    }
+    async headId(headName) {
         const key = safeKey(headName);
         const head = await readWithLegacyPath(join(this.root, 'heads', `${physicalArtifactKey(key)}.json`), join(this.root, 'heads', `${key}.json`));
-        return head ? this.read(head.snapshotId) : undefined;
+        return head?.snapshotId;
     }
     async updateHead(headName, snapshotId) {
-        if (!(await this.read(snapshotId)))
+        if (!(await this.readManifest(snapshotId)))
             throw new Error('Cannot point a head to an absent snapshot');
         return atomicJson(join(this.root, 'heads', `${physicalArtifactKey(safeKey(headName))}.json`), { snapshotId });
     }
@@ -138,7 +162,7 @@ export class SemanticArtifactStore {
         await mkdir(dirname(path), { recursive: true });
         const temporary = temporaryArtifactPath(path);
         try {
-            await writeFile(temporary, JSON.stringify(artifact), { flag: 'wx' });
+            await writeJsonFile(temporary, artifact);
             await link(temporary, path);
         }
         catch (error) {

package/dist/storage/streaming-json.d.ts

@@ -0,0 +1,5 @@
+/** Ordinary JSON bytes, incrementally encoded/decoded; legacy files need no migration. */
+export declare function readJsonFile<T>(path: string): Promise<T>;
+/** Validate the complete document while retaining only a selected root property. */
+export declare function readJsonProperty<T>(path: string, property: string): Promise<T | undefined>;
+export declare function writeJsonFile(path: string, value: unknown): Promise<void>;

package/dist/storage/streaming-json.js

@@ -0,0 +1,45 @@
+import { createReadStream, createWriteStream } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { stringifyChunked } from '@discoveryjs/json-ext';
+import { JSONParser, TokenizerError, TokenParserError } from '@streamparser/json';
+/** Ordinary JSON bytes, incrementally encoded/decoded; legacy files need no migration. */
+export async function readJsonFile(path) {
+    return parseJsonFile(path, '$');
+}
+/** Validate the complete document while retaining only a selected root property. */
+export async function readJsonProperty(path, property) {
+    if (!/^[a-zA-Z][a-zA-Z0-9]*$/.test(property))
+        throw Error('Invalid JSON property selector');
+    return parseJsonFile(path, `$.${property}`);
+}
+async function parseJsonFile(path, selector) {
+    let value;
+    const parser = new JSONParser({ paths: [selector], keepStack: selector === '$' });
+    let complete = false;
+    let hasInput = false;
+    parser.onValue = result => { value = result.value; };
+    parser.onEnd = () => { complete = true; };
+    parser.onError = error => {
+        // Cache inspection distinguishes invalid JSON from I/O/infrastructure errors.
+        // Keep that contract without including artifact contents in the public message.
+        // 0.0.26 reports a truncated container as plain Error, unlike invalid tokens.
+        const incompleteContainer = error.constructor === Error && error.message.startsWith('Parser ended in mid-parsing (state:');
+        if (error instanceof TokenizerError || error instanceof TokenParserError || incompleteContainer) {
+            throw new SyntaxError('Invalid JSON document', { cause: error });
+        }
+        throw error;
+    };
+    for await (const chunk of createReadStream(path)) {
+        if (!hasInput)
+            hasInput = chunk.some(byte => ![9, 10, 13, 32].includes(byte));
+        parser.write(chunk);
+    }
+    if (!parser.isEnded)
+        parser.end();
+    if (!complete || !hasInput)
+        throw new SyntaxError('Incomplete JSON document');
+    return value;
+}
+export async function writeJsonFile(path, value) {
+    await pipeline(stringifyChunked(value), createWriteStream(path, { flags: 'wx' }));
+}

package/dist/visualization/viewer-interaction.js

@@ -102,7 +102,7 @@ export function viewerScript() {
           if (!nextPath.disabled) status.textContent += ' Continue to examine remaining branches.';
           if (result.search.depthLimitReached) status.textContent += ' Depth limit reached; increase depth and restart.';
           if (result.search.stopReason === 'search-limit') status.textContent += ' Narrow the selection or choose an intermediate node.';
-          status.textContent += ' Pending prefixes: ' + result.frontier.pending.length + '; depth-limited extensions: ' + result.frontier.depthLimited.count + '.';
+          status.textContent += ' Pending prefixes: ' + result.frontier.pendingCount + ' (' + result.frontier.omittedPending + ' examples omitted); depth-limited extensions: ' + result.frontier.depthLimited.count + '.';
           search.value = ''; cluster.value = ''; document.getElementById('neighbors-only').checked = false;
           update();
         } catch (error) { clearPath(); status.textContent = error.message; update(); }

package/docs/1.0-WORK-QUEUE.md

@@ -16,18 +16,35 @@ for this candidate. Prior consumer Codex hydration remains evidence for the esta
 flow; it does not validate the new proposal operation. No further feature milestone is scheduled
 before this candidate. B6b and the explicit official 1.0 decision remain open.
 
+Consumer blocker, 2026-11-02: 0.7.0's richer code graph exceeded Node's whole-string limit before
+strict SHACL/publication. [ADR 040](ADR-040-bounded-graph-io.md) tracks the hashing, persistence,
+RDF/query and bounded-validation patch. Its public large-corpus lifecycle and hosted Linux/Windows checks
+pass; patch publication and private acceptance remain separate steps. No 1.0 approval is implied.
+
+Consumer follow-up: **0.7.1 is published**. The maintainer reports successful strict hydration,
+persistence/reload and unchanged repeat of 561,777 facts / 174,710 nodes with all 18 cached
+artifacts preserved. That capacity blocker is resolved in the reported run. B6b remains open:
+known direct paths and specific worker `references`/`consumes` assertions were displaced outside
+fixed retrieval budgets. [ADR 041](ADR-041-bounded-relationship-retrieval.md) tracks the bounded
+query-order patch and its public regression; no extraction refresh is needed for that patch.
+
+All three public 554,516-fact strict SHACL passes succeeded but took about 35–37 minutes each
+in this cloud. This is a capacity result, not a 30-minute hydration qualification. B6b must record
+full hydration and agent-query latency on the consumer workload as well as correctness.
+Any further validation optimization must retain full constraints and parity evidence.
+
 | Task | Status / dependency | Deliverable and check |
 | --- | --- | --- |
-| **B1a — Exact node neighborhoods** | Implemented; next candidate pending. Validation in ADR 029 and the change PR | `graph.neighbors` through the shared public API, CLI and MCP. Exact snapshot/node IDs, direction/relation filters, cited assertion pagination, remaining-neighbor counts, further-expansion hints and shared evidence state. Check duplicate labels, cycles, self-links, parallel evidence, missing/deleted nodes, stale/tampered cursors, dirty overlays and real extracted cross-service facts. |
-| **B1b — Bounded dependency paths** | Implemented; next candidate pending. Validation in ADR 030 and the change PR | `graph.paths` enumerates cited simple paths through API/CLI/MCP, with depth/work/output limits, continuation and explicit pending/depth-cut frontier. The offline viewer uses the same search function on displayed facts. Check competing branches, cycles, deep paths, scope omissions and all 4,096 directed four-node graphs against an independent oracle. |
+| **B1a — Exact node neighborhoods** | Shipped in 0.7.0. Validation in ADR 029 and the change PR | `graph.neighbors` through the shared public API, CLI and MCP. Exact snapshot/node IDs, direction/relation filters, cited assertion pagination, remaining-neighbor counts, further-expansion hints and shared evidence state. Check duplicate labels, cycles, self-links, parallel evidence, missing/deleted nodes, stale/tampered cursors, dirty overlays and real extracted cross-service facts. |
+| **B1b — Bounded dependency paths** | Shipped in 0.7.0. Validation in ADR 030 and the change PR | `graph.paths` enumerates cited simple paths through API/CLI/MCP, with depth/work/output limits, continuation and explicit pending/depth-cut frontier. The offline viewer uses the same search function on displayed facts. Check competing branches, cycles, deep paths, scope omissions and all 4,096 directed four-node graphs against an independent oracle. |
 | **B2a — Freeze code-graph comparison** | Frozen; verifier and fixture typechecks pass | 51 labeled code cases, three explicit gaps, unchanged A1–A3/dispatch controls, eight scenarios, pinned tools and bounded metrics in `evaluations/code-graph`. Read-only verification rejects input/label drift and supports immutable Git refs. No comparative execution or private/model inputs. |
 | **B2b — Run comparison and decide** | Measured; [ADR 031](ADR-031-code-extractor-selection.md) selects richer SCIP projection | Compare current projection, richer raw SCIP and pinned Graphify static extraction. Record correctness, citations, identity stability, cold/edit/repeat cost and packaging. Make an evidence-backed keep/improve/replace decision before B3's code-entity schema. |
-| **B3a — Explicit rule-to-symbol resolution** | Implemented in two increments ([ADR 032](ADR-032-scip-code-evidence.md), [ADR 033](ADR-033-document-symbol-bindings.md)); next candidate pending | Retain local/private symbols, same-file references and repeated source occurrences before resolving explicit document paths/symbol references. Include aliases, overloads, duplicate names, rename/delete and unresolved targets; retain both source spans/revisions and mapping authority. Follow the canonical [same-file acceptance boundary](PRE-1.0-BASELINE.md#same-file-exploration-acceptance). |
-| **B3b — Assess missing mappings** | Deterministic assessment complete ([ADR 034](ADR-034-mapping-gaps.md)); bounded opt-in implementation complete ([ADR 039](ADR-039-mapping-proposals.md)); live validation tentative and deferred by the maintainer for 0.7.0 | Identify gaps remaining after explicit resolution. Evaluate a bounded inference proposal only if justified, with an approved model budget before paid work. Never infer owner approval or silently convert proposed mappings into proven behavior. Record any scoped deferral for maintainer review. |
-| **B3c — Same-file call exploration** | Implemented ([ADR 035](ADR-035-static-call-evidence.md)); next candidate pending | Select and test static call analysis, preserving caller/callee/callsite and unresolved targets. Distinguish direct calls from references and callback registration; never claim observed execution. Meet the public large-file extraction/query acceptance in the canonical boundary, including recursion, unrelated functions, shadowed names, repeated sites and cross-file callers. No duplicate-behavior equivalence or mandatory second indexer. |
-| **B4a — Producer/consumer obligations** | Implemented for explicit declarations ([ADR 036](ADR-036-contract-obligation-declarations.md)); next candidate pending | Represent and retrieve cited guarantees/requirements on queue and stored-data contracts, with domain, stage, units, freshness and version where documented. Keep absent fields unknown. |
-| **B4b — Compatibility acceptance** | Implemented for selected explicit numeric declarations ([ADR 037](ADR-037-selected-contract-compatibility.md)); next candidate pending | Public queue/stored-data cases cover subset, insufficient guarantee, disjoint limits, revalidation and unknowns. Shared `graph.compatibility` retains two cited assertions and follow-up IDs. This is advisory; hard failures require approved deterministic policy. |
-| **B5 — Monorepo coverage** | Implemented for saved TypeScript-family inventory and package boundaries ([ADR 038](ADR-038-monorepo-code-coverage.md)); next candidate pending | `graph.coverage` pages selected/requested/indexed files and package connections; shared overview distinguishes saved scope from returned-page counts. Real barrels, aliases, TSX, independent/omitted services, declaration-only and unresolved external targets are tested. Unknown/zero observations never prove no impact. |
+| **B3a — Explicit rule-to-symbol resolution** | Implemented in two increments ([ADR 032](ADR-032-scip-code-evidence.md), [ADR 033](ADR-033-document-symbol-bindings.md)); shipped in 0.7.0 | Retain local/private symbols, same-file references and repeated source occurrences before resolving explicit document paths/symbol references. Include aliases, overloads, duplicate names, rename/delete and unresolved targets; retain both source spans/revisions and mapping authority. Follow the canonical [same-file acceptance boundary](PRE-1.0-BASELINE.md#same-file-exploration-acceptance). |
+| **B3b — Assess missing mappings** | Deterministic assessment complete ([ADR 034](ADR-034-mapping-gaps.md)); bounded opt-in implementation shipped in 0.7.0 ([ADR 039](ADR-039-mapping-proposals.md)); live validation tentative and deferred by the maintainer for 0.7.0 | Identify gaps remaining after explicit resolution. Evaluate a bounded inference proposal only if justified, with an approved model budget before paid work. Never infer owner approval or silently convert proposed mappings into proven behavior. Record any scoped deferral for maintainer review. |
+| **B3c — Same-file call exploration** | Implemented ([ADR 035](ADR-035-static-call-evidence.md)); shipped in 0.7.0 | Select and test static call analysis, preserving caller/callee/callsite and unresolved targets. Distinguish direct calls from references and callback registration; never claim observed execution. Meet the public large-file extraction/query acceptance in the canonical boundary, including recursion, unrelated functions, shadowed names, repeated sites and cross-file callers. No duplicate-behavior equivalence or mandatory second indexer. |
+| **B4a — Producer/consumer obligations** | Implemented for explicit declarations ([ADR 036](ADR-036-contract-obligation-declarations.md)); shipped in 0.7.0 | Represent and retrieve cited guarantees/requirements on queue and stored-data contracts, with domain, stage, units, freshness and version where documented. Keep absent fields unknown. |
+| **B4b — Compatibility acceptance** | Implemented for selected explicit numeric declarations ([ADR 037](ADR-037-selected-contract-compatibility.md)); shipped in 0.7.0 | Public queue/stored-data cases cover subset, insufficient guarantee, disjoint limits, revalidation and unknowns. Shared `graph.compatibility` retains two cited assertions and follow-up IDs. This is advisory; hard failures require approved deterministic policy. |
+| **B5 — Monorepo coverage** | Implemented for saved TypeScript-family inventory and package boundaries ([ADR 038](ADR-038-monorepo-code-coverage.md)); shipped in 0.7.0 | `graph.coverage` pages selected/requested/indexed files and package connections; shared overview distinguishes saved scope from returned-page counts. Real barrels, aliases, TSX, independent/omitted services, declaration-only and unresolved external targets are tested. Unknown/zero observations never prove no impact. |
 | **B6a — Candidate qualification** | B1–B5 decisions/increments reviewed | Freeze tasks and expected downstream obligations before measuring. Run public end-to-end, fresh-install and supported Windows/Linux release gates. Publish a minor acceptance candidate with versioned evidence and limitations. |
 | **B6b — Live consumer and 1.0 decision** | Published candidate | TradePilot runs real hydration/repeat and agent follow-up tasks; retain private requests/citations/timings locally and share a sanitized result. Assess usefulness and misses separately from pipeline success. Fix blockers, then explicitly review official 1.0. |