@weftlet/toolkit 0.2.1 → 0.3.0

package/dist/graph/rdf.js

@@ -1,39 +1,16 @@
 import { QueryEngine } from '@comunica/query-sparql';
-import { DataFactory, Parser, Store, Writer } from 'n3';
+import { DataFactory, Store, Writer } from 'n3';
 import SHACLValidator from 'rdf-validate-shacl';
 import { contextPackEvidenceAuthority } from "./authority.js";
 import { stableJson } from "../infrastructure/util.js";
 import { createRequire } from 'node:module';
-const { namedNode: iri, literal, quad } = DataFactory;
-export const CG = 'https://context-graph.dev/record#';
-const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
-const XSD = 'http://www.w3.org/2001/XMLSchema#';
+import { CG, RDF, XSD, evidenceShapes } from './shacl-shapes.js';
+const { namedNode: iri, literal } = DataFactory;
+export { CG } from './shacl-shapes.js';
 export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
 export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
 const engine = new QueryEngine();
 const SparqlParser = createRequire(import.meta.url)('sparqljs').Parser;
-const shapes = new Parser().parse(`
-@prefix cg: <${CG}>.
-@prefix rdf: <${RDF}>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <${XSD}>.
-cg:FactShape a sh:NodeShape; sh:targetClass rdf:Statement;
- sh:property [ sh:path cg:record; sh:minCount 1; sh:maxCount 1; sh:datatype xsd:string ];
- sh:property [ sh:path rdf:subject; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path rdf:predicate; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path rdf:object; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path cg:authority; sh:minCount 1; sh:maxCount 1;
-   sh:in ("declared" "implemented" "validated" "observed" "inferred" "draft") ];
- sh:property [ sh:path cg:path; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:commit; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:sourceDigest; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:startLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
- sh:property [ sh:path cg:endLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
- sh:property [ sh:path cg:confidence; sh:minCount 1; sh:datatype xsd:decimal; sh:minInclusive 0; sh:maxInclusive 1 ];
- sh:property [ sh:path cg:extractor; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ].
-cg:NodeShape a sh:NodeShape; sh:targetClass cg:Node;
- sh:property [ sh:path cg:kind; sh:minCount 1; sh:maxCount 1 ].
-`);
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export class RdfEvidenceGraph {
     snapshot;
@@ -85,7 +62,15 @@ export class RdfEvidenceGraph {
         }
     }
     async validate() {
-        const report = await new SHACLValidator(new Store(shapes)).validate(this.dataset);
+        // These graph-independent shapes already read the dataset as a union. Index
+        // that union once so every property lookup avoids scanning one graph per fact.
+        // Keep every triple; the original named dataset retains assertion provenance
+        // and remains the source for queries and serialization.
+        const validationGraph = new Store();
+        for (const { subject, predicate, object } of this.dataset) {
+            validationGraph.addQuad(subject, predicate, object);
+        }
+        const report = await new SHACLValidator(new Store(evidenceShapes)).validate(validationGraph);
         const issues = report.results.map((result) => ({
             category: 'extraction-gap',
             focus: result.focusNode?.value,

package/dist/graph/shacl-shapes.d.ts

@@ -0,0 +1,5 @@
+export declare const CG = "https://context-graph.dev/record#";
+export declare const RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+export declare const XSD = "http://www.w3.org/2001/XMLSchema#";
+/** Graph-independent evidence constraints, evaluated over the union of all assertion graphs. */
+export declare const evidenceShapes: import("n3").Quad[];

package/dist/graph/shacl-shapes.js

@@ -0,0 +1,27 @@
+import { Parser } from 'n3';
+export const CG = 'https://context-graph.dev/record#';
+export const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
+export const XSD = 'http://www.w3.org/2001/XMLSchema#';
+/** Graph-independent evidence constraints, evaluated over the union of all assertion graphs. */
+export const evidenceShapes = new Parser().parse(`
+@prefix cg: <${CG}>.
+@prefix rdf: <${RDF}>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <${XSD}>.
+cg:FactShape a sh:NodeShape; sh:targetClass rdf:Statement;
+ sh:property [ sh:path cg:record; sh:minCount 1; sh:maxCount 1; sh:datatype xsd:string ];
+ sh:property [ sh:path rdf:subject; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path rdf:predicate; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path rdf:object; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path cg:authority; sh:minCount 1; sh:maxCount 1;
+   sh:in ("declared" "implemented" "validated" "observed" "inferred" "draft") ];
+ sh:property [ sh:path cg:path; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:commit; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:sourceDigest; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:startLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
+ sh:property [ sh:path cg:endLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
+ sh:property [ sh:path cg:confidence; sh:minCount 1; sh:datatype xsd:decimal; sh:minInclusive 0; sh:maxInclusive 1 ];
+ sh:property [ sh:path cg:extractor; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ].
+cg:NodeShape a sh:NodeShape; sh:targetClass cg:Node;
+ sh:property [ sh:path cg:kind; sh:minCount 1; sh:maxCount 1 ].
+`);

package/dist/index.d.ts

@@ -9,3 +9,4 @@ export * from "./visualization/visualization.js";
 export * from "./application/overlay.js";
 export * from "./graph/rdf.js";
 export * from "./extraction/providers.js";
+export * from './application/hydration-plan.js';

package/dist/index.js

@@ -10,3 +10,4 @@ export * from "./visualization/visualization.js";
 export * from "./application/overlay.js";
 export * from "./graph/rdf.js";
 export * from "./extraction/providers.js";
+export * from './application/hydration-plan.js';

package/dist/interfaces/cli.js

@@ -15,6 +15,7 @@ import { FileSnapshotStore } from "../storage/storage.js";
 import { diagnoseSetup } from "./doctor.js";
 import { runInit } from "./init.js";
 import { loadConfig } from "../configuration/config.js";
+import { planHydration } from '../application/hydration-plan.js';
 function valueAfter(args, name) {
     const index = args.indexOf(name);
     return index >= 0 ? args[index + 1] : undefined;
@@ -45,10 +46,41 @@ export function impactRequestFromArgs(args) {
         continuation: valueAfter(args, '--continuation'),
     };
 }
+function formatHydrationPlan(plan) {
+    const readiness = plan.issues.some((issue) => issue.severity === 'error')
+        ? 'blocked' : plan.ready ? 'ready' : 'needs review';
+    const lines = [
+        `Hydration plan: ${plan.repositoryId}`,
+        `Commit: ${plan.commitSha} (ref ${plan.ref})`,
+        `Configuration: ${plan.configurationDigest}`,
+        `Scope: ${plan.scope.mode}, ${plan.scope.complete ? 'complete' : 'incomplete'}; head ${plan.scope.headName}`,
+        `Documents: ${plan.budgets.selectedDocuments}/${plan.budgets.maxDocuments}; ${plan.budgets.totalSourceBytes} bytes total; ${plan.budgets.maxSourceBytes} bytes maximum per document`,
+        `Expected cache reuse: ${plan.estimates.reusableArtifacts}; extractions: ${plan.estimates.extractions}; unknown cache entries: ${plan.estimates.unknownArtifacts}`,
+        `Source/cache/budget checks: ${readiness}`,
+    ];
+    for (const document of plan.documents) {
+        lines.push(`- ${JSON.stringify(document.path)}: ${document.bytes} bytes, ${document.lines} lines; ${document.role}; source ${document.sourceAuthority}, extraction ${document.extractionReview}; cache ${document.cache.status}; selected by ${document.selectedBy.map((selector) => JSON.stringify(selector)).join(', ')}`);
+    }
+    for (const selection of plan.selections.filter((item) => item.status !== 'matched')) {
+        lines.push(`Selection ${JSON.stringify(selection.selector)}: ${selection.status}`);
+    }
+    for (const issue of plan.issues)
+        lines.push(`${issue.severity.toUpperCase()} [${issue.code}]: ${issue.message}`);
+    if (plan.scope.omittedDocuments.length) {
+        lines.push(`Documents omitted by scope limits: ${plan.scope.omittedDocuments.map((path) => JSON.stringify(path)).join(', ')}`);
+    }
+    if (plan.scope.ignoredSearchTerms)
+        lines.push(`Search terms omitted by scope limits: ${plan.scope.ignoredSearchTerms}`);
+    if (plan.excludedWorkingChanges.length) {
+        lines.push(`Working changes excluded: ${plan.excludedWorkingChanges.map((path) => JSON.stringify(path)).join(', ')}`);
+    }
+    lines.push(...plan.assumptions.map((assumption) => `Note: ${assumption}`));
+    return `${lines.join('\n')}\n`;
+}
 async function main() {
     const args = process.argv.slice(2);
     if (args.includes('--help') || !args.length) {
-        process.stdout.write('weftlet init|config show|config validate|doctor|hydrate|overlay|impact scan|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
+        process.stdout.write('weftlet init|config show|config validate|doctor|hydrate|overlay|impact scan|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Preview readiness is advisory and does not add a hydration gate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
         return;
     }
     if (args[0] === 'init') {
@@ -85,6 +117,17 @@ async function main() {
         process.stdout.write(`${JSON.stringify(await diagnoseSetup(configPath), null, 2)}\n`);
         return;
     }
+    if (args[0] === 'hydrate' && args.includes('--plan')) {
+        const plan = await planHydration(configPath, {
+            ref: valueAfter(args, '--ref') ?? 'HEAD',
+            forceSemantic: args.includes('--force-semantic'),
+            request: impactRequestFromArgs(args),
+        });
+        process.stdout.write(args.includes('--json') ? `${JSON.stringify(plan, null, 2)}\n` : formatHydrationPlan(plan));
+        if (!plan.ready)
+            process.exitCode = 1;
+        return;
+    }
     const application = await createContextGraphApplication(configPath);
     const [command, subcommand] = args;
     if (command === 'sparql' || command === 'validate') {
@@ -120,9 +163,10 @@ async function main() {
         const controller = new AbortController();
         const cancel = () => controller.abort();
         process.once('SIGINT', cancel);
-        const scope = await application.scopeResolver?.resolve(impactRequestFromArgs(args));
+        const ref = valueAfter(args, '--ref') ?? 'HEAD';
+        const scope = await application.scopeResolver?.resolve(impactRequestFromArgs(args), ref);
         const result = await application.hydrator.hydrate({
-            ref: valueAfter(args, '--ref') ?? 'HEAD',
+            ref: scope?.commitSha ?? ref,
             headName: scope?.headName ?? application.headName,
             semanticSources: scope?.semanticSources,
             forceSemantic: args.includes('--force-semantic'),

package/dist/repository/git.d.ts

@@ -22,5 +22,5 @@ export declare class GitRepository {
     readFileAt(commitSha: string, path: string): Promise<string>;
     findTextAt(commitSha: string, needles: readonly string[], roots: readonly string[]): Promise<readonly GitTextMatch[]>;
     delta(baseSha: string | undefined, headSha: string, roots: readonly string[]): Promise<GitDelta>;
-    semanticSources(context: ExtractionContext, roots: readonly string[]): Promise<readonly SemanticSource[]>;
+    semanticSources(context: Pick<ExtractionContext, 'commitSha'>, roots: readonly string[]): Promise<readonly SemanticSource[]>;
 }

package/dist/repository/git.js

@@ -26,7 +26,7 @@ export class GitRepository {
     }
     async workingChanges() {
         const prefix = await this.run(['rev-parse', '--show-prefix']);
-        const output = await this.execute(['status', '--porcelain=v1', '-z', '--untracked-files=all'], false);
+        const output = await this.execute(['--no-optional-locks', 'status', '--porcelain=v1', '-z', '--untracked-files=all'], false);
         const records = output.split('\0');
         const paths = new Set();
         for (let index = 0; index < records.length; index++) {
@@ -65,6 +65,8 @@ export class GitRepository {
         }
     }
     async listFilesAt(commitSha, roots) {
+        if (roots.length === 0)
+            return [];
         const output = await this.run(['ls-tree', '-r', '--name-only', commitSha, '--', ...roots]);
         return output.split('\n').filter(Boolean).sort();
     }
@@ -107,6 +109,8 @@ export class GitRepository {
         return [...matches.values()].sort((left, right) => left.path.localeCompare(right.path));
     }
     async delta(baseSha, headSha, roots) {
+        if (roots.length === 0)
+            return { changedPaths: [], deletedPaths: [], renamedPaths: {} };
         if (!baseSha) {
             return {
                 changedPaths: await this.listFilesAt(headSha, roots),

package/docs/ADR-016-hydration-preview.md

@@ -0,0 +1,91 @@
+# ADR-016: Read-only hydration planning
+
+Status: accepted and implemented for the 0.3.0 release plan; absent from 0.2.1.
+See [validation](VALIDATION.md) for recorded publication evidence.
+
+## Problem
+
+Consumers need to inspect document selection, extraction work and configured limits before
+invoking a model. Constructing the execution application is unsuitable for that preview because
+it loads configured repository adapters. Independently calculating source selection or cache keys
+would also allow the preview to disagree with hydration.
+
+## Decision
+
+Provide `hydrate --plan` with a human summary and `hydrate --plan --json` with a structured
+`hydration-plan` record. The public `planHydration(configPath, options)` function uses the same
+planner. `--ref` resolves once to a commit SHA, including for agent scope. Configuration and
+artifact storage are read from the current local environment, and the plan reports excluded
+working changes rather than using their bytes as committed evidence.
+
+The planner reads configuration, committed Git objects and existing semantic artifacts. It never
+constructs an execution application or loads a configured provider/adapter. It does not probe
+credentials, index TypeScript, collect runtime evidence, compile ontology artifacts, validate RDF,
+publish snapshots or create cache directories. Git status disables optional locks so the preview
+does not refresh the index. Source bodies and raw model output are not included in the result.
+
+Committed source selection, duplicate semantic-ID checks and budget assessment are shared with
+hydration. Provider identity and semantic artifact keys are pure functions shared with execution.
+Per-source entries identify exact path/content digest, stable semantic identity, inclusion selectors,
+line and UTF-8 byte counts, configured evidence role and matched source-approval patterns.
+
+Classification follows configuration, not an inference about prose meaning. `sourceAuthority`
+can be declared by an approved path while `extractionReview` remains unreviewed. A source matching
+`evidenceRoots` is labeled validation evidence, but selecting it still schedules semantic extraction.
+Recursive SOT traversal and default separation of validation documents belong to T07.
+
+## Cache and readiness semantics
+
+Cache inspection distinguishes these states without modifying evidence:
+
+| State | Planned meaning |
+| --- | --- |
+| `hit` | Readable artifact matches source identity/content, extraction contract, structure and output digest |
+| `miss` | No artifact exists for the current key; extraction is expected |
+| `forced` | Explicit force requests fresh extraction; no forced key or artifact is created by planning |
+| `corrupt` | Stored JSON, identity, structure or content digest is invalid; reuse is not claimed |
+| `unavailable` | The artifact could not be read; its reuse state is unknown |
+
+Corrupt/unavailable entries are not ordinary misses and are never silently repaired or overwritten.
+An artifact can remain reusable after a source rename or a different commit with unchanged content;
+its original path/commit need not equal the current path/commit. Hydration rebinds current provenance.
+The preview's estimate is conditional on unchanged inputs and storage state, not a guarantee against
+concurrent changes or a successful provider run.
+
+Limits apply to every selected document, including cache hits: `maxDocuments` bounds the document
+count and `maxSourceBytes` bounds each document's UTF-8 bytes. Total source bytes are reported but
+are not a token or monetary budget. Selection is never trimmed to make these limits pass.
+
+Budget, duplicate-ID and cache errors produce a blocked plan. Missing/no-document selectors and
+omissions from existing bounded agent selection produce warnings and an incomplete scope that
+needs review. Both yield `ready: false` and CLI exit 1, while retaining the full plan. `ready: true`
+and exit 0 mean only that these planning checks passed; execution prerequisites and product-intent
+completeness have not been established.
+
+Warnings do not create new hard hydration gates. Existing hydration behavior can still publish a
+new graph after deleting the last selected document, removing its old relationships. The preview
+exposes that empty match for review. An explicitly empty source list is complete and selects none.
+
+## Compatibility
+
+The shared key/output-digest implementation preserves the released 0.2.1 semantic artifact contract,
+including document identity-policy invalidation. Valid existing artifacts remain reusable; no
+semantic re-extraction is required merely by this refactor. Operational configuration changes can
+still affect cache identity/storage profiles; that is separate T03 work.
+
+The empty-source fix changes graph assembly behavior: empty roots previously could enumerate all
+repository documents. Assembly advances to `graph-v6` so corrected snapshots cannot collide with
+immutable older snapshots at the same commit/configuration. Old snapshots remain readable and
+unchanged. The requested ref also controls agent source selection instead of always using HEAD.
+
+## Validation and next work
+
+Public regressions cover committed and dirty inputs, unchanged index/files, absent or side-effecting
+provider/adapter modules, empty scope, warm/cold/forced cache, malformed and unreadable artifacts,
+released-key compatibility, budgets, duplicate IDs, missing selectors and historical agent scope
+with explicit omissions. Existing rename/delete lifecycle checks remain applicable. Full validation
+and release evidence are recorded separately in [VALIDATION.md](VALIDATION.md).
+
+This increment makes no SHACL performance change. T05 will measure graph construction and full
+validation before selecting caching, incremental validation or backend changes. See the
+[1.0 plan](RELEASE-1.0-PLAN.md) for bounded follow-ups and the separate 2.0 platform direction.

package/docs/ADR-017-shacl-validation-union.md

@@ -0,0 +1,102 @@
+# ADR-017: Validate a temporary RDF union without changing canonical evidence
+
+Status: accepted and implemented for the focused 0.3.0 release plan; absent from 0.2.1.
+See [validation](VALIDATION.md) for checks and recorded publication evidence.
+
+## Problem and measured mechanism
+
+The canonical RDF dataset has one named relationship graph per assertion, with assertion records,
+node metadata and provenance in the default graph. The JavaScript `rdf-validate-shacl` validator
+uses graph-wildcard dataset lookups through clownface. N3 indexes graphs first, so a lookup with
+no graph restriction visits each graph before consulting its subject/predicate/object indexes.
+Repeating those lookups for each focus node and shape property becomes expensive as assertion
+graphs grow.
+
+Bounded public experiments measured this cost in the current layout and compared an indexed union
+using the same validator and shapes. They establish a synthetic scaling problem and a useful
+optimization; they do not reproduce the private consumer's complete generation or diagnose its
+reported 30-minute timeout. The consumer's total graph size and other phase costs remain unknown.
+
+## Decision
+
+For each `RdfEvidenceGraph.validate()` call, build a temporary single-default-graph N3 store from
+every subject/predicate/object triple in the canonical dataset. Run the unchanged SHACL shapes
+and validator on that store, and retain the existing explicit citation-order check. The canonical
+named dataset remains unchanged and continues to serve SPARQL, fact projection and serialization.
+
+This is still full-graph validation on every refresh. It introduces no GraphDB service, native/Rust
+validator, incremental validation or validation-result cache. The short-lived union is not retained
+as another persistent graph or used to replace canonical evidence.
+
+RDF graphs are sets: identical SPO triples from multiple named graphs collapse in the validation
+projection. The present shapes already use union lookups and node/value sets, and do not inspect
+graph identity. Separate assertion records and their cited provenance remain intact in the original
+dataset; no assertion, shape or constraint is removed. A future graph-sensitive shape requires an
+explicit compatibility decision and new equivalence tests before using this projection.
+
+## Bounded public measurements
+
+Environment: Linux cloud workspace, Node 24.19.0, N3 2.7.2 and rdf-validate-shacl 0.6.5.
+Each case ran once in a fresh child process with a 45-second hard deadline. The prototype copied
+the same original dataset into a single graph only for validation; production-patch verification
+is recorded separately in [VALIDATION.md](VALIDATION.md).
+
+Fixtures contain public synthetic Workflow-to-Component assertions, cited metadata and one named
+relationship graph per fact. Shared topology repeats 50 nodes; unique topology uses two distinct
+nodes per fact. Timings below are seconds, with union construction separated from SHACL. RSS is
+the process sample after validation, including Node, libraries and both datasets, not a measurement
+of projection memory alone.
+
+| Facts / topology | Original quads | Original SHACL | Union construction | Union SHACL | Union RSS MiB |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| 1,000 shared | 20,203 | 3.889 | 0.061 | 0.239 | 352.3 |
+| 2,000 shared | 40,203 | 19.550 | 0.102 | 0.410 | 440.0 |
+| 1,000 unique | 28,003 | 5.731 | 0.104 | 0.308 | 352.6 |
+| 2,000 unique | 56,003 | 31.502 | 0.153 | 0.467 | 457.9 |
+| 5,000 shared | 100,203 | No verdict before process deadline | 0.251 | 1.131 | 825.5 |
+| 10,000 unique | 280,003 | Not measured | 0.779 | 2.119 | 1,287.3 |
+
+The 5,000-fact original-mode process was terminated after 45.051 seconds total elapsed time,
+including startup and construction. It produced no SHACL verdict; this is not an inferred
+completed validation duration. For the 10,000-fact unique union case, initial canonical graph
+construction took 1.058 seconds in addition to the times shown. Its roughly 1.3 GiB RSS is a
+material memory cost and should be measured on larger or different workloads.
+
+These single runs are not statistical latency estimates or production guarantees. The experiment
+does not measure semantic extraction, SCIP, SPARQL, serialization, snapshot publication or disk
+storage. No private consumer data or live provider calls were used. The repository benchmark
+`pnpm benchmark:shacl` reproduces bounded public comparisons using the current build; retain its
+configuration, versions and raw output when comparing results.
+
+## Correctness and compatibility
+
+The prototype compared sorted conformance issues between the original and union paths across
+valid repeated relationships/attributes, conflicting node kinds, out-of-range confidence, invalid
+citations, empty citation paths, colliding assertion IDs and combined violations. All seven paired
+cases agreed, and before/after serialization hashes preserved the canonical named dataset.
+
+Production regressions must compare both paths for all supported constraints, including negative
+cases, and preserve named-graph SPARQL results, fact projection, serialization and repeated calls.
+Those checks establish parity for the current shapes, not arbitrary future SHACL rules. Failed
+validation must continue to prevent publication and preserve the previous valid snapshot.
+
+The optimization changes neither facts nor evidence authority and does not require a new semantic
+cache identity. It does not make old evidence eligible for current-diff gates. Normal snapshot
+commit, configuration, scope and provenance checks continue to apply.
+
+## Release boundary and remaining T05 work
+
+Ship the demonstrated cold-validation improvement after its independent review and required
+validation, ahead of unrelated 1.0 work. A first graph cannot benefit from reusing a successful
+validation result that does not yet exist. Private consumer acceptance remains pending after release.
+
+Early identity checks, detailed progress/elapsed reporting, validation-result caching and safe
+incremental validation remain open. The validator's main validation loop is synchronous; timers
+on the same JavaScript thread cannot show a live heartbeat while it is blocked. Keep phase
+boundaries honest and evaluate bounded worker execution before promising in-phase progress or
+cancellation. Any later cached or incremental result must account for graph content, shapes,
+validator/policy versions, deletions, affected relationships and global constraints, with full
+validation when safe scope cannot be established.
+
+Native engines and optional persistent database backends remain measurement-driven alternatives,
+not prerequisites of this release. See the [1.0 task list](RELEASE-1.0-PLAN.md).

package/docs/RELEASE-1.0-PLAN.md

@@ -1,10 +1,13 @@
 # Consumer hardening and the 1.0 release plan
 
-Status: maintainer-directed work, recorded on 2026-10-26 from a consumer integration report.
-The starting release is `@weftlet/toolkit@0.2.0`. **Ship a patch for T01 first.** The remaining
-items improve setup, reuse, diagnostics and document coverage; they do not block that patch
-or justify expanding the consumer's migration. An official 1.0 follows this work and genuine
-consumer acceptance, with its exact release scope reviewed when the evidence is available.
+Status: maintainer-directed work; source checkpoint on 2026-10-26 before the planned 0.3.0 release.
+The report concerned `@weftlet/toolkit@0.2.0`. **T01's upstream patch shipped in 0.2.1**;
+genuine consumer acceptance remains pending. T02 is merged and awaits release. T05's focused
+cold-validation optimization is implemented in development source, with validation/release pending;
+its broader follow-ups and T03, T04, T06 and T07 remain open. These tasks improve setup, reuse, diagnostics and document coverage
+without expanding the consumer's migration. An official 1.0
+follows this work and genuine consumer acceptance, with its exact release scope reviewed when
+the evidence is available.
 
 This is the near-term release task list. The [Weftlet 2.0 product plan](WEFTLET-2.0-PLAN.md)
 remains a separate future direction for generation, approvals, a companion website and hosted
@@ -31,17 +34,20 @@ integration. Consumer acceptance is reported separately from upstream determinis
 
 | ID | Task | Priority/status | Dependencies and release boundary |
 | --- | --- | --- | --- |
-| **T01** | Coherent repository-document identity and kind | **Adoption blocker; patch work in progress** | First isolated patch; preserve SHACL and all assertions |
-| **T02** | No-model hydration/setup preview | Planned usability work | Shares selection/cache identity with hydration; extend it with T07 traversal coverage |
+| **T01** | Coherent repository-document identity and kind | **Upstream patch shipped in 0.2.1; consumer acceptance pending** | SHACL and assertions preserved; genuine consumer re-run is still required |
+| **T02** | No-model hydration/setup preview | Merged in PR #18; not published in 0.2.1 | Shared selection/cache inspection, human/JSON plans; extend with T07 traversal coverage |
 | **T03** | Artifact reuse across operational configuration changes | Planned efficiency work | Preserve T01's semantic identity separation; keep snapshot/config identity distinct from reusable artifact identity |
 | **T04** | Actionable, privacy-safe timeout diagnostics | Planned diagnostics work | Use stable source identifiers and run progress; document recovery with T02/T03 |
-| **T05** | Early identity checks and validation progress/profiling | Planned correctness/observability work | Build on T01; profile latency independently before selecting optimizations |
-| **T06** | Packaged release-documentation consistency | Planned release tooling work | Check the actual release archives; does not require a runtime architecture change |
+| **T05** | Early identity checks and validation progress/profiling | **Adoption blocker; cold-validation fix in source** | Validate and release union-projection optimization now; early checks, detailed progress and validation reuse remain open |
+| **T06** | Packaged release-documentation consistency | Open; README version pins removed in 0.2.1 | Automated archive/version consistency check remains to be implemented |
 | **T07** | Coverage-aware traversal from SOT/roadmap entry points | Planned bounded feature work | Use canonical identity from T01, preview from T02 and incremental reuse from T03; split selection/classification/extraction changes |
 
-T02–T07 can ship in independently useful increments after the blocker patch. Do not combine
-all seven into one diff or delay T01 for the optional platform work. Normal required release
-checks still apply to every shipped increment.
+T02 passed the local publication checks and hosted Ubuntu/Windows checks before
+[PR #18](https://github.com/jchewitt/weftlet/pull/18) merged. Prioritize the focused T05 performance
+release following the maintainer's reported timeout; do not wait for the remaining 1.0 features.
+T03, T04, T06 and T07 remain independently useful increments. Keep them
+separate from the optional platform work. Publication follows the normal release process; no new
+npm release is claimed for T02. Normal release checks apply to every shipped increment.
 
 ## T01 — Resolve repository-document identity conflicts
 
@@ -66,16 +72,40 @@ Acceptance:
   the patch, then request a genuine consumer re-run against the recorded inputs; do not infer
   consumer acceptance from the upstream fixture.
 
-Completion evidence will link the implementation PR, tested source commit, validation record,
-published patch and subsequent consumer result. No patch publication or consumer pass is claimed
-by this task list yet.
+Upstream completion evidence:
+
+- [Fix PR #15](https://github.com/jchewitt/weftlet/pull/15) implements the policy in
+  [ADR 015](ADR-015-repository-document-identity.md), with public cross-document regressions and
+  fresh/cache-reuse coverage. [Release PR #16](https://github.com/jchewitt/weftlet/pull/16)
+  advances the fixed core/toolkit group to 0.2.1.
+- [Release run 40382864083](https://github.com/jchewitt/weftlet/actions/runs/40382864083)
+  completed preparation, publication and release records on its first attempt at commit
+  `5d1db91bf19ed493a20849b2d0f92d3aeac35dfe`. Required checks included 227 deterministic tests,
+  12 advisory architecture findings, fresh packed-consumer checks and hosted Ubuntu/Windows CI.
+- Registry archives, provenance and tags match the reviewed release. A fresh public-registry
+  consumer passes the two-document regression, real SCIP, RDF/SHACL, queries and unchanged
+  snapshot identity/content on artifact reuse, without live model calls. See the detailed
+  [validation record](VALIDATION.md) for commits, checks and result boundaries.
+
+The private consumer has not yet supplied a successful full hydration/repeat against 0.2.1.
+Upstream publication and regression success do not close that acceptance gate or diagnose the
+separate full-graph validation latency observation.
 
 ## T02 — Preview scope and work without calling a model
 
-Add a no-model preview command, tentatively `hydrate --plan`; the CLI spelling is not implemented
-by this plan. Use the same source-selection and identity logic as hydration to avoid preview drift.
-Report selected sources and sizes, inclusion reasons, role/approval classification, expected
-artifact hits/misses and extraction count, plus document/byte limits and any violations.
+Implemented in development source: `hydrate --plan` renders a human summary and `--json` returns
+the structured plan. This feature is not in published 0.2.1. Shared source selection and identity
+logic report selected documents and sizes, inclusion reasons, configured role/source approval,
+unreviewed assertion status, artifact reuse/extraction estimates and document/per-source byte limits.
+See [ADR 016](ADR-016-hydration-preview.md) and [validation](VALIDATION.md).
+
+Missing or unsupported-only selectors and bounded agent omissions remain visible as warnings;
+incomplete plans need review and exit 1. They do not introduce new hydration gates that would
+break deletion of the last document. Budget, duplicate-ID and corrupt/unavailable cache errors
+block planning checks without truncating selected sources. An explicit empty source list selects
+none. Requested refs now govern agent scope; corrected snapshots use graph-v6 while valid 0.2.1
+semantic artifacts remain reusable. T03's operational cache changes and T07's traversal/default
+validation-document separation remain separate work.
 
 Acceptance:
 
@@ -126,19 +156,56 @@ Acceptance:
 
 ## T05 — Detect inexpensive conflicts early and explain validation work
 
-Add inexpensive graph-assembly checks for incompatible identities before expensive validation.
-Keep the full conformance requirement. Separately instrument and profile large RDF/SHACL
-validation to determine where time is spent before choosing optimizations.
+The implementation uses JavaScript on Node:
+`rdf-validate-shacl` validates an in-memory N3 dataset. Hydration constructs and validates the
+entire assembled graph, including when semantic artifacts were reused. There is no persistent
+GraphDB backend or incremental validation. Querying a saved snapshot does not rerun SHACL.
+
+The maintainer reports that validation pushes the consumer's overall generation beyond its
+30-minute timeout, preventing snapshot publication and making adoption unusable. This is now
+a release blocker, not an optional observability improvement. Upstream has measured a specific
+named-graph lookup cost in public synthetic fixtures, but has not measured that consumer graph or
+diagnosed its complete timeout. Ship the focused performance release once its correctness and
+improvement are demonstrated, ahead of the remaining 1.0 tasks.
+
+[ADR 017](ADR-017-shacl-validation-union.md) records the implemented cold-validation optimization:
+build a temporary union of every SPO triple for the same validator/shapes, while keeping original
+named assertion graphs canonical for queries and serialization. It removes repeated wildcard
+lookups across one graph per fact. In a bounded single-run public 2,000-fact unique-node case,
+the original SHACL path took 31.502 seconds; union construction took 0.153 seconds and SHACL
+took 0.467 seconds. A 10,000-fact unique-node union case used about 1.3 GiB process RSS. These
+are synthetic phase measurements, not a private consumer reproduction or end-to-end guarantee.
+
+The focused change still validates the whole graph each time. It does not add Rust, GraphDB,
+incremental validation or a validation-result cache. Early checks and broader observability/reuse
+remain follow-ups rather than reasons to delay the cold-path fix. Native code or a database does
+not by itself establish faster or correct incremental SHACL.
 
 Acceptance:
 
 - Early errors identify the conflicting identity/kinds and relevant evidence using the configured
   privacy policy. Preserve all underlying assertions for diagnosis; early checks are not a
   substitute for RDF/SHACL validation.
-- Report elapsed time and honest phase/heartbeat progress for long validation. Do not display a
-  fabricated completion percentage when the validator exposes no meaningful denominator.
+- Report elapsed time and honest progress for long validation. Current phase boundaries remain
+  the supported progress signal: the synchronous validator blocks same-thread timers. Evaluate
+  bounded worker execution before promising an in-phase heartbeat or cancellation; never display
+  a fabricated completion percentage when the validator exposes no meaningful denominator.
 - Record reproducible public-fixture sizes, fact/node counts, runtime versions, wall time and
-  resource observations. Separate measured bottlenecks from hypotheses.
+  resource observations for construction and validation separately. Separate measured bottlenecks
+  from hypotheses; include repeated identical input and small changed-input cases.
+- Demonstrate improved cold full-graph validation, because reusing a previous validation result
+  cannot unblock a first graph that has never completed. Keep private consumer acceptance and
+  the overall generation timeout separate from synthetic validator-only benchmarks.
+- Evaluate reuse of a successful conformance result only when validated graph content, shapes,
+  validator version and applicable policy/semantics match. This reuse must not make historical
+  evidence satisfy current-diff freshness, scope or provenance requirements.
+- Evaluate dependency-aware incremental validation with insertions, updates, deletions, renames,
+  affected relationships and global constraints. Revalidate the complete affected scope or fall
+  back to full validation when it cannot be bounded safely; changed nodes alone are insufficient.
+- Benchmark a suitable native/Rust-backed validator or optional persistent graph database if
+  measurements justify a candidate. Compare supported SHACL behavior, positive/negative parity,
+  installation, licensing, portability and maintenance cost. T02 adds no backend migration, and
+  this evaluation does not commit the standalone toolkit to a mandatory service.
 - Any optimized validation/projection path has equivalence tests, including negative cases;
   failure must preserve the prior snapshot. Keep model-independent profiling separate from live
   extraction performance and from the reported two-fact correctness defect.
@@ -146,9 +213,9 @@ Acceptance:
 ## T06 — Keep packaged documentation consistent with the release
 
 The consumer installed 0.2.0 while its packaged README still named 0.1.2 and some archive examples
-used 0.1.1. Source documentation has since been updated, but immutable published archives retain
-their original text. Add a repeatable release-documentation check rather than relying on a
-post-publication source edit.
+used 0.1.1. Release 0.2.1 removes the README version pins; immutable 0.2.0 archives retain their
+original text. The automated release-documentation consistency check remains open. Add that
+repeatable check rather than relying on manual edits alone.
 
 Acceptance:
 
@@ -194,7 +261,7 @@ Acceptance:
 
 ## Official 1.0 acceptance
 
-After T01's patch is available, work through T02–T07 in bounded increments and record implementation
+With T01's upstream patch published, work through T02–T07 in bounded increments and record implementation
 PRs and validation evidence here or in the linked validation record. Review the exact 1.0 scope
 against completed evidence; any deferral needs an explicit maintainer decision and must remain
 visible rather than being treated as done. The companion website, hosted AI and broader 2.0