@weftlet/toolkit 0.7.0 → 0.7.1

package/dist/graph/rdf-partitions.d.ts

@@ -0,0 +1,24 @@
+import { Store, type Term } from 'n3';
+import type { GraphSnapshot } from '@weftlet/core';
+type Quad = Parameters<Store['add']>[0];
+/** Complete subject groups: repeated identities are joined before validation, never split. */
+export declare class RdfPartitions {
+    private readonly snapshot;
+    private readonly batchSize;
+    private readonly records;
+    private readonly nodes;
+    constructor(snapshot: GraphSnapshot, batchSize?: number);
+    /** RDF/JS matching over the same records, without a second whole-graph quad store. */
+    match(subject?: Term | null, predicate?: Term | null, object?: Term | null, graph?: Term | null): IterableIterator<Quad>;
+    /** Exact cardinalities for the record projection; other patterns count the same iterator. */
+    countMatches(subject?: Term | null, predicate?: Term | null, object?: Term | null, graph?: Term | null): number;
+    /** Each union subject occurs in exactly one partition, with all its outgoing triples. */
+    validation(): IterableIterator<Store>;
+    /** Original named dataset, in bounded groups. Triple order is not a content identity. */
+    named(): IterableIterator<Store>;
+    private recordPartitions;
+    /** Only cross-subject information used by the admitted SHACL fragment is rdf:type. */
+    get count(): number;
+    types(progress?: (completed: number) => void): Map<string, Set<string>>;
+}
+export {};

package/dist/graph/rdf-partitions.js

@@ -0,0 +1,170 @@
+import { DataFactory, Store } from 'n3';
+import { stableJson } from '../infrastructure/util.js';
+import { addAttributes, addFactEdges, addFactRecord, addManifest, addNodeRdf, nodeIri } from './rdf-records.js';
+import { CG, RDF } from './shacl-shapes.js';
+const { namedNode: iri } = DataFactory;
+/** Complete subject groups: repeated identities are joined before validation, never split. */
+export class RdfPartitions {
+    snapshot;
+    batchSize;
+    records = new Map();
+    nodes = new Map();
+    constructor(snapshot, batchSize = 256) {
+        this.snapshot = snapshot;
+        this.batchSize = batchSize;
+        if (!Number.isSafeInteger(batchSize) || batchSize < 1)
+            throw Error('Invalid RDF partition size');
+        for (const [ordinal, fact] of snapshot.facts.entries()) {
+            const records = this.records.get(fact.id) ?? [];
+            records.push({ fact, ordinal });
+            this.records.set(fact.id, records);
+            for (const node of [fact.subject, fact.object]) {
+                const group = this.nodes.get(node.id) ?? { variants: new Map(), outgoing: [], attributes: [] };
+                group.variants.set(stableJson(node), node);
+                this.nodes.set(node.id, group);
+                if (Object.keys(node.attributes ?? {}).length)
+                    group.attributes.push({ node, factId: fact.id });
+            }
+            this.nodes.get(fact.subject.id).outgoing.push(fact);
+        }
+    }
+    /** RDF/JS matching over the same records, without a second whole-graph quad store. */
+    *match(subject, predicate, object, graph) {
+        const decode = (term, prefix, suffix = false) => {
+            if (term?.termType !== 'NamedNode' || !term.value.startsWith(prefix))
+                return undefined;
+            try {
+                return decodeURIComponent(suffix ? term.value.slice(prefix.length).split(':')[0] : term.value.slice(prefix.length));
+            }
+            catch {
+                return undefined;
+            }
+        };
+        const recordId = decode(subject, 'urn:context-graph:fact:', true);
+        const graphId = decode(graph, 'urn:context-graph:fact:');
+        const nodeId = decode(subject, 'urn:context-graph:node:');
+        let groups;
+        if (recordId !== undefined || graphId !== undefined) {
+            const batch = new Store();
+            for (const { fact, ordinal } of this.records.get(recordId ?? graphId) ?? []) {
+                addFactRecord(batch, fact, ordinal);
+                addFactEdges(batch, fact);
+            }
+            groups = [batch];
+        }
+        else if (nodeId !== undefined) {
+            const batch = new Store(), group = this.nodes.get(nodeId);
+            if (group) {
+                for (const node of group.variants.values())
+                    addNodeRdf(batch, node);
+                for (const fact of group.outgoing)
+                    addFactEdges(batch, fact);
+                for (const { node, factId } of group.attributes)
+                    addAttributes(batch, node, iri(`urn:context-graph:fact:${encodeURIComponent(factId)}`));
+            }
+            groups = [batch];
+        }
+        else if (subject?.value === 'urn:context-graph:snapshot') {
+            const batch = new Store();
+            addManifest(batch, this.snapshot);
+            groups = [batch];
+        }
+        else if (subject)
+            groups = [];
+        else
+            groups = this.named();
+        for (const batch of groups)
+            for (const quad of batch)
+                if ((!subject || subject.equals(quad.subject)) && (!predicate || predicate.equals(quad.predicate)) &&
+                    (!object || object.equals(quad.object)) && (!graph || graph.equals(quad.graph)))
+                    yield quad;
+    }
+    /** Exact cardinalities for the record projection; other patterns count the same iterator. */
+    countMatches(subject, predicate, object, graph) {
+        if (!subject && !object && [CG + 'record', CG + 'ordinal'].includes(predicate?.value ?? '')) {
+            if (graph && graph.termType !== 'DefaultGraph')
+                return 0;
+            if (predicate.value === CG + 'ordinal')
+                return this.snapshot.facts.length;
+            let count = 0;
+            for (const group of this.records.values())
+                count += group.length === 1 ? 1 : new Set(group.map(({ fact }) => stableJson(fact))).size;
+            return count;
+        }
+        let count = 0;
+        for (const _quad of this.match(subject, predicate, object, graph))
+            count++;
+        return count;
+    }
+    /** Each union subject occurs in exactly one partition, with all its outgoing triples. */
+    *validation() {
+        yield* this.recordPartitions(false);
+        let batch = new Store(), count = 0;
+        for (const [id, group] of this.nodes) {
+            for (const node of group.variants.values()) {
+                addNodeRdf(batch, node);
+                addAttributes(batch, node);
+            }
+            for (const fact of group.outgoing)
+                batch.addQuad(iri(nodeIri(id)), iri(CG + fact.predicate), iri(nodeIri(fact.object.id)));
+            if (++count % this.batchSize === 0) {
+                yield batch;
+                batch = new Store();
+            }
+        }
+        if (batch.size)
+            yield batch;
+        const manifest = new Store();
+        addManifest(manifest, this.snapshot);
+        yield manifest;
+    }
+    /** Original named dataset, in bounded groups. Triple order is not a content identity. */
+    *named() {
+        yield* this.recordPartitions(true);
+        let batch = new Store(), count = 0;
+        for (const group of this.nodes.values()) {
+            for (const node of group.variants.values())
+                addNodeRdf(batch, node);
+            if (++count % this.batchSize === 0) {
+                yield batch;
+                batch = new Store();
+            }
+        }
+        if (batch.size)
+            yield batch;
+        const manifest = new Store();
+        addManifest(manifest, this.snapshot);
+        yield manifest;
+    }
+    *recordPartitions(edges) {
+        let batch = new Store(), count = 0;
+        for (const records of this.records.values()) {
+            for (const { fact, ordinal } of records) {
+                addFactRecord(batch, fact, ordinal);
+                if (edges)
+                    addFactEdges(batch, fact);
+            }
+            if (++count % this.batchSize === 0) {
+                yield batch;
+                batch = new Store();
+            }
+        }
+        if (batch.size)
+            yield batch;
+    }
+    /** Only cross-subject information used by the admitted SHACL fragment is rdf:type. */
+    get count() { return Math.ceil(this.records.size / this.batchSize) + Math.ceil(this.nodes.size / this.batchSize) + 1; }
+    types(progress) {
+        const result = new Map();
+        let completed = 0;
+        for (const batch of this.validation()) {
+            for (const quad of batch.getQuads(null, iri(RDF + 'type'), null, null)) {
+                const values = result.get(quad.subject.value) ?? new Set();
+                values.add(quad.object.value);
+                result.set(quad.subject.value, values);
+            }
+            progress?.(++completed);
+        }
+        return result;
+    }
+}

package/dist/graph/rdf-projection.d.ts

@@ -0,0 +1,7 @@
+import type { GraphFact } from '@weftlet/core';
+/**
+ * Equivalent to SELECT cg:record JOIN cg:ordinal ORDER BY ordinal. The record
+ * literal uses the canonical RDF encoder, not a separate interpretation of facts.
+ * Preserve duplicate-ID join multiplicity too; callers can reject corrupt IDs.
+ */
+export declare function projectRdfFacts(facts: readonly GraphFact[]): readonly GraphFact[];

package/dist/graph/rdf-projection.js

@@ -0,0 +1,27 @@
+import { stableJson } from '../infrastructure/util.js';
+/**
+ * Equivalent to SELECT cg:record JOIN cg:ordinal ORDER BY ordinal. The record
+ * literal uses the canonical RDF encoder, not a separate interpretation of facts.
+ * Preserve duplicate-ID join multiplicity too; callers can reject corrupt IDs.
+ */
+export function projectRdfFacts(facts) {
+    const counts = new Map();
+    for (const fact of facts)
+        counts.set(fact.id, (counts.get(fact.id) ?? 0) + 1);
+    const duplicates = new Map();
+    for (const fact of facts)
+        if (counts.get(fact.id) > 1) {
+            const records = duplicates.get(fact.id) ?? new Set();
+            records.add(stableJson(fact));
+            duplicates.set(fact.id, records);
+        }
+    const result = [];
+    for (const fact of facts) {
+        if (duplicates.has(fact.id))
+            for (const record of duplicates.get(fact.id))
+                result.push(JSON.parse(record));
+        else
+            result.push(JSON.parse(stableJson(fact)));
+    }
+    return result;
+}

package/dist/graph/rdf-records.d.ts

@@ -0,0 +1,12 @@
+import { DataFactory, Store } from 'n3';
+import type { GraphFact, GraphSnapshot, GraphNode } from '@weftlet/core';
+declare const iri: <Iri extends string = string>(value: Iri) => import("n3").NamedNode<Iri>;
+export declare const nodeIri: (id: string) => string;
+export declare const factIri: (id: string) => string;
+export declare function addFactRecord(dataset: Store, fact: GraphFact, ordinal: number): void;
+export declare function addNodeRdf(dataset: Store, node: GraphNode): void;
+export declare function addAttributes(dataset: Store, node: GraphNode, graph?: ReturnType<typeof iri> | ReturnType<typeof DataFactory.defaultGraph>): void;
+export declare function addFactEdges(dataset: Store, fact: GraphFact): void;
+export declare function addManifest(dataset: Store, snapshot: GraphSnapshot): void;
+export declare function evidenceDataset(snapshot: GraphSnapshot): Store;
+export {};

package/dist/graph/rdf-records.js

@@ -0,0 +1,90 @@
+import { DataFactory, Store } from 'n3';
+import { contextPackEvidenceAuthority } from './authority.js';
+import { stableJson } from '../infrastructure/util.js';
+import { CG, RDF, XSD } from './shacl-shapes.js';
+import { addBindingEvidence } from './binding-rdf.js';
+import { addCallEvidence } from './call-rdf.js';
+import { addObligationEvidence } from './obligation-rdf.js';
+import { addCodeEvidence, addCodeNodeEvidence } from './code-rdf.js';
+const { namedNode: iri, literal } = DataFactory;
+export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
+export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
+export function addFactRecord(dataset, fact, ordinal) {
+    const record = iri(factIri(fact.id));
+    const subject = iri(nodeIri(fact.subject.id));
+    const object = iri(nodeIri(fact.object.id));
+    const predicate = iri(CG + fact.predicate);
+    dataset.addQuad(record, iri(RDF + 'type'), iri(RDF + 'Statement'));
+    dataset.addQuad(record, iri(CG + 'ordinal'), literal(String(ordinal), iri(XSD + 'integer')));
+    for (const [p, value] of [['subject', subject], ['predicate', predicate], ['object', object]]) {
+        dataset.addQuad(record, iri(RDF + p), value);
+    }
+    for (const [p, value] of Object.entries({
+        authority: contextPackEvidenceAuthority(fact), path: fact.provenance.path,
+        commit: fact.provenance.commitSha, sourceDigest: fact.provenance.digest,
+        extractor: `${fact.extractor.id}@${fact.extractor.version}`,
+        snapshotId: fact.snapshotId, record: stableJson(fact),
+        ...(fact.sourceAuthority ? { sourceAuthority: fact.sourceAuthority } : {}),
+        ...(fact.extractionReview ? { extractionReview: fact.extractionReview } : {}),
+        ...(fact.sourceRole !== undefined ? { sourceRole: fact.sourceRole } : {}),
+        ...(fact.sourcePolicyDigest !== undefined ? { sourcePolicyDigest: fact.sourcePolicyDigest } : {}),
+    }))
+        dataset.addQuad(record, iri(CG + p), literal(value));
+    if (fact.sourceOwnership !== undefined) {
+        dataset.addQuad(record, iri(CG + 'ownershipStatus'), literal(fact.sourceOwnership.status));
+        for (const ownerDocument of fact.sourceOwnership.ownerDocuments) {
+            dataset.addQuad(record, iri(CG + 'ownerDocument'), literal(ownerDocument));
+        }
+        for (const mappingId of fact.sourceOwnership.mappingIds) {
+            dataset.addQuad(record, iri(CG + 'ownershipMapping'), literal(mappingId));
+        }
+    }
+    for (const [p, value] of [['startLine', fact.provenance.startLine], ['endLine', fact.provenance.endLine]]) {
+        dataset.addQuad(record, iri(CG + p), literal(String(value), iri(XSD + 'integer')));
+    }
+    dataset.addQuad(record, iri(CG + 'confidence'), literal(String(fact.confidence), iri(XSD + 'decimal')));
+    addCodeEvidence(dataset, record, fact, nodeIri, false);
+    addBindingEvidence(dataset, record, fact, factIri);
+    addCallEvidence(dataset, record, fact, factIri);
+    addObligationEvidence(dataset, record, fact);
+}
+export function addNodeRdf(dataset, node) {
+    const term = iri(nodeIri(node.id));
+    dataset.addQuad(term, iri(RDF + 'type'), iri(CG + 'Node'));
+    dataset.addQuad(term, iri(CG + 'identifier'), literal(node.id));
+    dataset.addQuad(term, iri(CG + 'kind'), literal(node.kind));
+    dataset.addQuad(term, iri(CG + 'label'), literal(node.label));
+    addCodeNodeEvidence(dataset, node, nodeIri);
+}
+export function addAttributes(dataset, node, graph = DataFactory.defaultGraph()) {
+    for (const [key, value] of Object.entries(node.attributes ?? {})) {
+        const datatype = typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'decimal' : 'string';
+        dataset.addQuad(iri(nodeIri(node.id)), iri(`${CG}attribute/${encodeURIComponent(key)}`), literal(String(value), iri(XSD + datatype)), graph);
+    }
+}
+export function addFactEdges(dataset, fact) {
+    const record = iri(factIri(fact.id));
+    dataset.addQuad(iri(nodeIri(fact.subject.id)), iri(CG + fact.predicate), iri(nodeIri(fact.object.id)), record);
+    for (const node of [fact.subject, fact.object])
+        addAttributes(dataset, node, record);
+}
+export function addManifest(dataset, snapshot) {
+    dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'manifest'), literal(stableJson(snapshot.manifest)));
+    for (const scope of snapshot.manifest.scope ?? []) {
+        dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'scope'), literal(scope));
+    }
+    if (snapshot.manifest.configurationDigest) {
+        dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'configurationDigest'), literal(snapshot.manifest.configurationDigest));
+    }
+}
+export function evidenceDataset(snapshot) {
+    const dataset = new Store();
+    for (const [ordinal, fact] of snapshot.facts.entries()) {
+        addFactRecord(dataset, fact, ordinal);
+        addFactEdges(dataset, fact);
+        for (const node of [fact.subject, fact.object])
+            addNodeRdf(dataset, node);
+    }
+    addManifest(dataset, snapshot);
+    return dataset;
+}

package/dist/graph/rdf.d.ts

@@ -1,29 +1,29 @@
+import type { Writable } from 'node:stream';
 import type { GraphFact, GraphSnapshot } from "@weftlet/core";
 import { type GraphValidationProgress } from './validation-progress.js';
 export { CG } from './shacl-shapes.js';
-export declare const nodeIri: (id: string) => string;
-export declare const factIri: (id: string) => string;
+export { factIri, nodeIri } from './rdf-records.js';
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export declare class RdfEvidenceGraph {
     readonly snapshot: GraphSnapshot;
-    private readonly dataset;
+    private indexed?;
     constructor(snapshot: GraphSnapshot);
     validate(options?: {
         readonly onProgress?: (event: GraphValidationProgress) => void;
     }): Promise<{
         conforms: boolean;
-        issues: {
-            category: "extraction-gap";
-            focus: string;
-            path: string;
-            message: string;
-        }[];
+        issues: import("./partitioned-shacl.js").ShaclIssue[];
     }>;
     select(sparql: string, maxRows?: number): Promise<{
         rows: Record<string, string>[];
         truncated: boolean;
     }>;
-    /** The traversal projection is read through SPARQL, not a separate fact source. */
+    /** Lossless optimized projection, checked for equivalence with the public SPARQL path. */
     facts(): Promise<readonly GraphFact[]>;
+    /** Complete N-Quads in bounded groups; RDF triple order is not a snapshot identity. */
+    serializeChunks(): IterableIterator<string>;
+    /** Export large datasets with backpressure, without a whole-dataset string/quad array. */
+    serializeTo(destination: Writable): Promise<void>;
+    /** Compatibility convenience for small exports; use serializeTo for large graphs. */
     serialize(): Promise<string>;
 }

package/dist/graph/rdf.js

@@ -1,105 +1,48 @@
 import { QueryEngine } from '@comunica/query-sparql';
-import { DataFactory, Store, Writer } from 'n3';
-import SHACLValidator from 'rdf-validate-shacl';
-import { contextPackEvidenceAuthority } from "./authority.js";
-import { stableJson } from "../infrastructure/util.js";
+import { constants } from 'node:buffer';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+import { Writer } from 'n3';
 import { createRequire } from 'node:module';
-import { CG, RDF, XSD, evidenceShapes } from './shacl-shapes.js';
+import { CG } from './shacl-shapes.js';
 import { measureValidationPhase } from './validation-progress.js';
-import { addBindingEvidence, bindingEvidenceIssues } from './binding-rdf.js';
-import { addCallEvidence, callEvidenceIssues } from './call-rdf.js';
-import { addObligationEvidence, obligationEvidenceIssues } from './obligation-rdf.js';
-import { addCodeEvidence, codeEvidenceIssues, invalidCodeRange } from './code-rdf.js';
-const { namedNode: iri, literal } = DataFactory;
+import { bindingEvidenceIssues } from './binding-rdf.js';
+import { callEvidenceIssues } from './call-rdf.js';
+import { obligationEvidenceIssues } from './obligation-rdf.js';
+import { codeEvidenceIssues, invalidCodeRange } from './code-rdf.js';
 export { CG } from './shacl-shapes.js';
-export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
-export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
+import { factIri } from './rdf-records.js';
+import { RdfPartitions } from './rdf-partitions.js';
+import { validatePartitions } from './partitioned-shacl.js';
+import { projectRdfFacts } from './rdf-projection.js';
+export { factIri, nodeIri } from './rdf-records.js';
 const engine = new QueryEngine();
 const SparqlParser = createRequire(import.meta.url)('sparqljs').Parser;
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export class RdfEvidenceGraph {
     snapshot;
-    dataset = new Store();
+    indexed;
     constructor(snapshot) {
         this.snapshot = snapshot;
-        for (const [ordinal, fact] of snapshot.facts.entries()) {
-            const record = iri(factIri(fact.id));
-            const subject = iri(nodeIri(fact.subject.id));
-            const object = iri(nodeIri(fact.object.id));
-            const predicate = iri(CG + fact.predicate);
-            this.dataset.addQuad(subject, predicate, object, record);
-            this.dataset.addQuad(record, iri(RDF + 'type'), iri(RDF + 'Statement'));
-            this.dataset.addQuad(record, iri(CG + 'ordinal'), literal(String(ordinal), iri(XSD + 'integer')));
-            for (const [p, value] of [['subject', subject], ['predicate', predicate], ['object', object]]) {
-                this.dataset.addQuad(record, iri(RDF + p), value);
-            }
-            for (const [p, value] of Object.entries({
-                authority: contextPackEvidenceAuthority(fact), path: fact.provenance.path,
-                commit: fact.provenance.commitSha, sourceDigest: fact.provenance.digest,
-                extractor: `${fact.extractor.id}@${fact.extractor.version}`,
-                snapshotId: fact.snapshotId, record: stableJson(fact),
-                ...(fact.sourceAuthority ? { sourceAuthority: fact.sourceAuthority } : {}),
-                ...(fact.extractionReview ? { extractionReview: fact.extractionReview } : {}),
-                ...(fact.sourceRole !== undefined ? { sourceRole: fact.sourceRole } : {}),
-                ...(fact.sourcePolicyDigest !== undefined ? { sourcePolicyDigest: fact.sourcePolicyDigest } : {}),
-            }))
-                this.dataset.addQuad(record, iri(CG + p), literal(value));
-            if (fact.sourceOwnership !== undefined) {
-                this.dataset.addQuad(record, iri(CG + 'ownershipStatus'), literal(fact.sourceOwnership.status));
-                for (const ownerDocument of fact.sourceOwnership.ownerDocuments) {
-                    this.dataset.addQuad(record, iri(CG + 'ownerDocument'), literal(ownerDocument));
-                }
-                for (const mappingId of fact.sourceOwnership.mappingIds) {
-                    this.dataset.addQuad(record, iri(CG + 'ownershipMapping'), literal(mappingId));
-                }
-            }
-            for (const [p, value] of [['startLine', fact.provenance.startLine], ['endLine', fact.provenance.endLine]]) {
-                this.dataset.addQuad(record, iri(CG + p), literal(String(value), iri(XSD + 'integer')));
-            }
-            this.dataset.addQuad(record, iri(CG + 'confidence'), literal(String(fact.confidence), iri(XSD + 'decimal')));
-            addCodeEvidence(this.dataset, record, fact, nodeIri);
-            addBindingEvidence(this.dataset, record, fact, factIri);
-            addCallEvidence(this.dataset, record, fact, factIri);
-            addObligationEvidence(this.dataset, record, fact);
-            for (const node of [fact.subject, fact.object]) {
-                const term = iri(nodeIri(node.id));
-                this.dataset.addQuad(term, iri(RDF + 'type'), iri(CG + 'Node'));
-                this.dataset.addQuad(term, iri(CG + 'identifier'), literal(node.id));
-                this.dataset.addQuad(term, iri(CG + 'kind'), literal(node.kind));
-                this.dataset.addQuad(term, iri(CG + 'label'), literal(node.label));
-                for (const [key, value] of Object.entries(node.attributes ?? {})) {
-                    const datatype = typeof value === 'boolean' ? 'boolean' : typeof value === 'number' ? 'decimal' : 'string';
-                    this.dataset.addQuad(term, iri(`${CG}attribute/${encodeURIComponent(key)}`), literal(String(value), iri(XSD + datatype)), record);
-                }
-            }
-        }
-        this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'manifest'), literal(stableJson(snapshot.manifest)));
-        for (const scope of snapshot.manifest.scope ?? []) {
-            this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'scope'), literal(scope));
-        }
-        if (snapshot.manifest.configurationDigest) {
-            this.dataset.addQuad(iri('urn:context-graph:snapshot'), iri(CG + 'configurationDigest'), literal(snapshot.manifest.configurationDigest));
-        }
     }
     async validate(options = {}) {
-        // These graph-independent shapes already read the dataset as a union. Index
-        // that union once so every property lookup avoids scanning one graph per fact.
-        // Keep every triple; the original named dataset retains assertion provenance
-        // and remains the source for queries and serialization.
         const counts = { factCount: this.snapshot.facts.length };
-        const validationGraph = await measureValidationPhase('build-validation-union', counts, options.onProgress, () => {
-            const union = new Store();
-            for (const { subject, predicate, object } of this.dataset)
-                union.addQuad(subject, predicate, object);
-            return union;
+        const { partitions, types } = await measureValidationPhase('build-validation-union', counts, options.onProgress, () => {
+            const started = performance.now();
+            const partitions = new RdfPartitions(this.snapshot);
+            return { partitions, types: partitions.types(completed => {
+                    if (completed % 10 === 0)
+                        options.onProgress?.({ phase: 'build-validation-union', status: 'progress',
+                            elapsedMs: Math.round(performance.now() - started), ...counts, completed, total: partitions.count });
+                }) };
         });
-        const report = await measureValidationPhase('validate-shacl', counts, options.onProgress, () => new SHACLValidator(new Store(evidenceShapes)).validate(validationGraph));
-        const issues = report.results.map((result) => ({
-            category: 'extraction-gap',
-            focus: result.focusNode?.value,
-            path: result.path?.value,
-            message: result.message.map((message) => message.value).join('; '),
+        const started = performance.now();
+        const report = await measureValidationPhase('validate-shacl', counts, options.onProgress, () => validatePartitions(partitions, types, completed => {
+            if (completed % 10 === 0)
+                options.onProgress?.({ phase: 'validate-shacl', status: 'progress',
+                    elapsedMs: Math.round(performance.now() - started), ...counts, completed, total: partitions.count });
         }));
+        const issues = report.issues;
         const byId = new Map(this.snapshot.facts.map(fact => [fact.id, fact]));
         for (const fact of this.snapshot.facts) {
             for (const message of obligationEvidenceIssues(fact, this.snapshot.manifest.repositoryId))
@@ -140,7 +83,12 @@ export class RdfEvidenceGraph {
         if (ast.type !== 'query' || ast.queryType !== 'SELECT' || ast.from || hasService(ast)) {
             throw new Error('Only local read queries are supported; remote datasets and updates are disabled');
         }
-        const stream = await engine.queryBindings(sparql, { sources: [this.dataset], unionDefaultGraph: true });
+        const indexed = this.indexed ??= new RdfPartitions(this.snapshot);
+        const source = {
+            match: (s, p, o, g) => Readable.from(indexed.match(s, p, o, g)),
+            countQuads: (s, p, o, g) => indexed.countMatches(s, p, o, g),
+        };
+        const stream = await engine.queryBindings(sparql, { sources: [{ type: 'rdfjs', value: source }], unionDefaultGraph: true });
         const rows = [];
         let truncated = false;
         for await (const binding of stream) {
@@ -153,18 +101,39 @@ export class RdfEvidenceGraph {
         }
         return { rows, truncated };
     }
-    /** The traversal projection is read through SPARQL, not a separate fact source. */
+    /** Lossless optimized projection, checked for equivalence with the public SPARQL path. */
     async facts() {
-        const result = await this.select(`SELECT ?record WHERE {
-      ?fact <${CG}record> ?record; <${CG}ordinal> ?ordinal
-    } ORDER BY ?ordinal`, 100_000);
-        if (result.truncated)
-            throw new Error('Graph exceeds the 100000-fact projection bound');
-        return result.rows.map((row) => JSON.parse(row.record));
+        return projectRdfFacts(this.snapshot.facts);
     }
-    serialize() {
+    /** Complete N-Quads in bounded groups; RDF triple order is not a snapshot identity. */
+    *serializeChunks() {
         const writer = new Writer({ format: 'N-Quads' });
-        writer.addQuads(this.dataset.getQuads(null, null, null, null));
-        return new Promise((resolve, reject) => writer.end((error, text) => error ? reject(error) : resolve(text)));
+        let buffer = '';
+        for (const batch of new RdfPartitions(this.snapshot).named())
+            for (const quad of batch) {
+                buffer += writer.quadToString(quad.subject, quad.predicate, quad.object, quad.graph);
+                if (buffer.length >= 64 * 1024) {
+                    yield buffer;
+                    buffer = '';
+                }
+            }
+        if (buffer)
+            yield buffer;
+    }
+    /** Export large datasets with backpressure, without a whole-dataset string/quad array. */
+    async serializeTo(destination) {
+        await pipeline(this.serializeChunks(), destination);
+    }
+    /** Compatibility convenience for small exports; use serializeTo for large graphs. */
+    async serialize() {
+        const chunks = [];
+        let length = 0;
+        for (const chunk of this.serializeChunks()) {
+            length += chunk.length;
+            if (length > constants.MAX_STRING_LENGTH)
+                throw new Error('RDF exceeds the JavaScript string limit; use serializeTo() or serializeChunks()');
+            chunks.push(chunk);
+        }
+        return chunks.join('');
     }
 }

package/dist/graph/validation-progress.d.ts

@@ -1,7 +1,10 @@
 export type GraphValidationPhase = 'validate-identities' | 'build-rdf-evidence' | 'build-validation-union' | 'validate-shacl';
 export interface GraphValidationProgress {
     readonly phase: GraphValidationPhase;
-    readonly status: 'started' | 'completed' | 'failed';
+    readonly status: 'started' | 'completed' | 'failed' | 'progress';
+    /** Completed bounded RDF subject groups, when available. */
+    readonly completed?: number;
+    readonly total?: number;
     /** Wall time for this phase, not a completion estimate or in-phase heartbeat. */
     readonly elapsedMs: number;
     readonly factCount: number;

package/dist/infrastructure/json-digest.d.ts

@@ -1,2 +1,3 @@
+import { persistedJsonDigest } from './util.js';
 /** Identity of persisted JSON, excluding undefined optional fields that JSON cannot retain. */
-export declare const jsonDigest: (value: unknown) => string;
+export declare const jsonDigest: typeof persistedJsonDigest;

package/dist/infrastructure/json-digest.js

@@ -1,3 +1,3 @@
-import { sha256, stableJson } from './util.js';
+import { persistedJsonDigest } from './util.js';
 /** Identity of persisted JSON, excluding undefined optional fields that JSON cannot retain. */
-export const jsonDigest = (value) => sha256(stableJson(JSON.parse(JSON.stringify(value))));
+export const jsonDigest = persistedJsonDigest;

package/dist/interfaces/cli.js

@@ -103,7 +103,7 @@ function formatHydrationPlan(plan) {
 async function main() {
     const args = process.argv.slice(2);
     if (args.includes('--help') || !args.length) {
-        process.stdout.write('weftlet mappings plan|mappings propose|mappings read|documents plan|init|config show|config validate|doctor|hydrate|cache adopt|overlay|impact scan|graph neighbors|graph paths|graph compatibility|graph coverage|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nmappings plan|propose --request <json-file> [--allow-model for propose] previews or runs one opt-in proposal; mappings read --snapshot <id> --key <key> reads saved unreviewed results.\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ngraph neighbors --snapshot <id> --node <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-facts <1..200>] [--continuation <token>] [--json] reads one-hop saved evidence; repeat --predicate to select multiple relations.\ngraph paths --snapshot <id> --from <exact-id> --to <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-depth <1..32>] [--max-paths <1..20>] [--max-steps <1..10000>] [--continuation <token>] [--json] explores cited simple paths; incomplete searches report frontier and limits.\ngraph coverage --snapshot <id> [--project <exact-package-root>] [--max-items <1..200>] [--continuation <token>] [--json] pages saved package/file coverage; absence or zero references is not proof of no impact.\ngraph compatibility --snapshot <id> --producer-fact <assertion-id> --consumer-fact <assertion-id> [--json] compares two documented numeric obligations; advisory only, with explicit unknowns and no implementation or CI verdict.\ndocuments plan --entry <document> [--entry <document>] [--root <path>] [--document-policy <json-file>] [--ref <commit>] [--json] previews prospective committed link coverage; it does not change hydration inputs or estimate model/cache work. Source classification/ownership completeness is separate from traversal readiness. The preview policy is an operation input. Set documentTraversal in configuration to activate traversal hydration; hydrate --plan describes its execution inputs. Safety limits: --max-read-documents, --max-read-bytes, --max-document-bytes.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Traversal coverage and role ambiguity block new model work; direct-source selection retains its existing behavior.\nhydrate --json --quiet emits structured semantic failures on stderr with no progress lines; failure stdout stays empty. hydrate --plan maps opaque source IDs to local document paths.\nhydrate|overlay --validation-details explicitly includes local node/fact identities, extractor names and source citations in identity-conflict errors; default reports use opaque references. This flag writes no diagnostic file. Use --json --quiet for structured identity-conflict stderr and empty failure stdout. Validation timings report phase boundaries, not a live heartbeat or percentage.\ncache adopt --from-config <previous.json> [--config <current.json>] [--ref <commit>] [--dry-run] returns JSON and copies only validated compatible artifacts, without model calls or snapshot writes. --dry-run performs no cache writes. Agent configurations accept the same --owner-doc, --task and --capability selection as hydrate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
+        process.stdout.write('weftlet mappings plan|mappings propose|mappings read|documents plan|init|config show|config validate|doctor|hydrate|cache adopt|overlay|impact scan|graph neighbors|graph paths|graph compatibility|graph coverage|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nmappings plan|propose --request <json-file> [--allow-model for propose] previews or runs one opt-in proposal; mappings read --snapshot <id> --key <key> reads saved unreviewed results.\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ngraph neighbors --snapshot <id> --node <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-facts <1..200>] [--continuation <token>] [--json] reads one-hop saved evidence; repeat --predicate to select multiple relations.\ngraph paths --snapshot <id> --from <exact-id> --to <exact-id> [--direction incoming|outgoing|both] [--predicate <relation>] [--max-depth <1..32>] [--max-paths <1..20>] [--max-steps <1..10000>] [--continuation <token>] [--json] explores cited simple paths; incomplete searches report frontier and limits.\ngraph coverage --snapshot <id> [--project <exact-package-root>] [--max-items <1..200>] [--continuation <token>] [--json] pages saved package/file coverage; absence or zero references is not proof of no impact.\ngraph compatibility --snapshot <id> --producer-fact <assertion-id> --consumer-fact <assertion-id> [--json] compares two documented numeric obligations; advisory only, with explicit unknowns and no implementation or CI verdict.\ndocuments plan --entry <document> [--entry <document>] [--root <path>] [--document-policy <json-file>] [--ref <commit>] [--json] previews prospective committed link coverage; it does not change hydration inputs or estimate model/cache work. Source classification/ownership completeness is separate from traversal readiness. The preview policy is an operation input. Set documentTraversal in configuration to activate traversal hydration; hydrate --plan describes its execution inputs. Safety limits: --max-read-documents, --max-read-bytes, --max-document-bytes.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Traversal coverage and role ambiguity block new model work; direct-source selection retains its existing behavior.\nhydrate --json --quiet emits structured semantic failures on stderr with no progress lines; failure stdout stays empty. hydrate --plan maps opaque source IDs to local document paths.\nhydrate|overlay --validation-details explicitly includes local node/fact identities, extractor names and source citations in identity-conflict errors; default reports use opaque references. This flag writes no diagnostic file. Use --json --quiet for structured identity-conflict stderr and empty failure stdout. Validation reports phase boundaries, elapsed time and completed groups; group counts are not completion-time estimates.\ncache adopt --from-config <previous.json> [--config <current.json>] [--ref <commit>] [--dry-run] returns JSON and copies only validated compatible artifacts, without model calls or snapshot writes. --dry-run performs no cache writes. Agent configurations accept the same --owner-doc, --task and --capability selection as hydrate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
         return;
     }
     if (await runDocumentCoverage(args, defaultConfigPath))

package/dist/storage/storage.d.ts

@@ -4,7 +4,10 @@ export declare class FileSnapshotStore implements SnapshotStore {
     constructor(root: string);
     write(snapshot: GraphSnapshot): Promise<void>;
     read(snapshotId: string): Promise<GraphSnapshot | undefined>;
+    private readManifest;
     readHead(headName: string): Promise<GraphSnapshot | undefined>;
+    readHeadManifest(headName: string): Promise<GraphSnapshot['manifest'] | undefined>;
+    private headId;
     updateHead(headName: string, snapshotId: string): Promise<void>;
 }
 export declare class SemanticArtifactStore {