@weftlet/toolkit 0.2.0 → 0.3.0

package/dist/extraction/semantic.js

@@ -8,7 +8,24 @@ import { resolveCodexInvocation } from './codex-command.js';
 import { runSemanticProcess, SemanticProviderError } from "./semantic-process.js";
 export { SemanticProviderError } from "./semantic-process.js";
 import { sha256, stableJson } from "../infrastructure/util.js";
+import { canonicalizeRepositoryDocumentNode } from './repository-nodes.js';
+import { semanticArtifactKey, semanticArtifactOutputDigest, semanticProviderIdentity } from './semantic-identity.js';
+import { inspectSemanticCache } from './semantic-cache.js';
 import { assertSemanticCitation, numberedSemanticSource, semanticCitationSchema, } from "./semantic-citations.js";
+function canonicalizeRepositoryDocuments(output) {
+    const ids = new Map();
+    const assertions = output.assertions.map((assertion) => {
+        const subject = canonicalizeRepositoryDocumentNode(assertion.subject);
+        const object = canonicalizeRepositoryDocumentNode(assertion.object);
+        ids.set(assertion.subject.id, subject.id);
+        ids.set(assertion.object.id, object.id);
+        return { ...assertion, subject, object };
+    });
+    return { ...output, assertions, diagnostics: output.diagnostics?.map((diagnostic) => ({
+            ...diagnostic,
+            relatedNodeIds: diagnostic.relatedNodeIds.map((id) => ids.get(id) ?? id),
+        })) };
+}
 function omlEscape(value) {
     return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\r?\n/g, '\\n');
 }
@@ -213,7 +230,7 @@ export class CodexCliSemanticExtractor {
         this.config = config;
         this.runner = runner;
         this.loginProbe = loginProbe;
-        this.identity = `codex-cli:${config.model}:${config.reasoningEffort}:${config.promptVersion}:contract-v2`;
+        this.identity = semanticProviderIdentity(config);
     }
     async extract(source, context, options = {}) {
         options.signal?.throwIfAborted();
@@ -235,7 +252,7 @@ export class CodexCliSemanticExtractor {
             'Use neutral graph kinds and the allowed precise predicates. Never guess a missing mapping.',
             'Use product-wide canonical IDs derived from durable concepts, not IDs scoped to this file.',
             'Reuse capability, policy, workflow, contract, and component IDs that another source would use for the same concept.',
-            'When the source names a repository path, use component:file:<path> so semantic and static facts join deterministically.',
+            'For a repository document path ending in .md, .mdx or .txt, use document:<path> with kind Document. For other repository files use component:file:<path>; test/spec files have kind Evidence, other files Component. Use exact repository-relative paths; keep business concepts in their own IDs instead of assigning their kind to a file.',
             'Capture why a rule exists, its decision authority, and cross-domain dependencies when the source explicitly states them.',
             'Every assertion must cite inclusive 1-based source line numbers and carry calibrated confidence.',
             `Source lines are explicitly prefixed N| below. Cite those original-source numbers, not prompt lines or visual wraps; valid lines are 1-${lineCount}.`,
@@ -318,29 +335,18 @@ export class SemanticHydrator {
     }
     async hydrate(source, context, forceRevision, options = {}) {
         options.signal?.throwIfAborted();
-        const keyInput = {
-            stableSemanticId: source.stableSemanticId,
-            sourceDigest: source.digest,
-            promptVersion: this.config.promptVersion,
-            extractorVersion: this.config.extractorVersion,
-            modelIdentity: this.extractor.identity,
-            adapterVersion: this.config.adapterVersion,
-            ontologyVersion: this.config.ontologyVersion,
-            minimumConfidence: this.config.minimumConfidence,
-            failOnUnresolved: this.config.failOnUnresolved,
-            forceRevision,
-            diagnosticContractVersion: 3,
-        };
-        const key = sha256(stableJson(keyInput));
-        const existing = await this.store.read(key);
-        if (existing)
-            return { artifact: existing, reused: true };
+        const key = semanticArtifactKey(this.config, source, this.extractor.identity, forceRevision);
+        const cached = await inspectSemanticCache(this.store, this.config, source, this.extractor.identity, forceRevision);
+        if (cached.status === 'hit')
+            return { artifact: cached.artifact, reused: true };
+        if (cached.status !== 'miss')
+            throw new Error(cached.reason);
         const extraction = await this.extractor.extract(source, context, options);
         assertSemanticOutput(extraction.parsed, source);
         if (typeof extraction.rawModelOutput !== 'string' || Buffer.byteLength(extraction.rawModelOutput) > 2 * 1024 * 1024) {
             throw new SemanticProviderError('invalid-output', 'Invalid or oversized raw semantic output');
         }
-        const output = extraction.parsed;
+        const output = canonicalizeRepositoryDocuments(extraction.parsed);
         const lineCount = source.content.split('\n').length;
         const facts = output.assertions.map((assertion, index) => {
             assertSemanticCitation(assertion.citation, source.path, lineCount, `assertion[${index}]`);
@@ -395,7 +401,7 @@ export class SemanticHydrator {
             throw new Error(`Unresolved semantic mappings for ${source.path}: ${[...unresolvedMappings, ...diagnostics.map((item) => item.reason)].join('; ')}`);
         }
         const compiledOml = renderOml(key, output.assertions);
-        const outputDigest = sha256(stableJson({ facts, diagnostics, compiledOml, rawModelOutput: extraction.rawModelOutput }));
+        const outputDigest = semanticArtifactOutputDigest({ facts, diagnostics, oml: compiledOml, rawModelOutput: extraction.rawModelOutput });
         const artifact = {
             key,
             stableSemanticId: source.stableSemanticId,

package/dist/graph/rdf.d.ts

@@ -1,5 +1,5 @@
 import type { GraphFact, GraphSnapshot } from "@weftlet/core";
-export declare const CG = "https://context-graph.dev/record#";
+export { CG } from './shacl-shapes.js';
 export declare const nodeIri: (id: string) => string;
 export declare const factIri: (id: string) => string;
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */

package/dist/graph/rdf.js

@@ -1,39 +1,16 @@
 import { QueryEngine } from '@comunica/query-sparql';
-import { DataFactory, Parser, Store, Writer } from 'n3';
+import { DataFactory, Store, Writer } from 'n3';
 import SHACLValidator from 'rdf-validate-shacl';
 import { contextPackEvidenceAuthority } from "./authority.js";
 import { stableJson } from "../infrastructure/util.js";
 import { createRequire } from 'node:module';
-const { namedNode: iri, literal, quad } = DataFactory;
-export const CG = 'https://context-graph.dev/record#';
-const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
-const XSD = 'http://www.w3.org/2001/XMLSchema#';
+import { CG, RDF, XSD, evidenceShapes } from './shacl-shapes.js';
+const { namedNode: iri, literal } = DataFactory;
+export { CG } from './shacl-shapes.js';
 export const nodeIri = (id) => `urn:context-graph:node:${encodeURIComponent(id)}`;
 export const factIri = (id) => `urn:context-graph:fact:${encodeURIComponent(id)}`;
 const engine = new QueryEngine();
 const SparqlParser = createRequire(import.meta.url)('sparqljs').Parser;
-const shapes = new Parser().parse(`
-@prefix cg: <${CG}>.
-@prefix rdf: <${RDF}>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <${XSD}>.
-cg:FactShape a sh:NodeShape; sh:targetClass rdf:Statement;
- sh:property [ sh:path cg:record; sh:minCount 1; sh:maxCount 1; sh:datatype xsd:string ];
- sh:property [ sh:path rdf:subject; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path rdf:predicate; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path rdf:object; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
- sh:property [ sh:path cg:authority; sh:minCount 1; sh:maxCount 1;
-   sh:in ("declared" "implemented" "validated" "observed" "inferred" "draft") ];
- sh:property [ sh:path cg:path; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:commit; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:sourceDigest; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
- sh:property [ sh:path cg:startLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
- sh:property [ sh:path cg:endLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
- sh:property [ sh:path cg:confidence; sh:minCount 1; sh:datatype xsd:decimal; sh:minInclusive 0; sh:maxInclusive 1 ];
- sh:property [ sh:path cg:extractor; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ].
-cg:NodeShape a sh:NodeShape; sh:targetClass cg:Node;
- sh:property [ sh:path cg:kind; sh:minCount 1; sh:maxCount 1 ].
-`);
 /** Lossless RDF dataset: one named graph per cited assertion, with provenance in the default graph. */
 export class RdfEvidenceGraph {
     snapshot;
@@ -85,7 +62,15 @@ export class RdfEvidenceGraph {
         }
     }
     async validate() {
-        const report = await new SHACLValidator(new Store(shapes)).validate(this.dataset);
+        // These graph-independent shapes already read the dataset as a union. Index
+        // that union once so every property lookup avoids scanning one graph per fact.
+        // Keep every triple; the original named dataset retains assertion provenance
+        // and remains the source for queries and serialization.
+        const validationGraph = new Store();
+        for (const { subject, predicate, object } of this.dataset) {
+            validationGraph.addQuad(subject, predicate, object);
+        }
+        const report = await new SHACLValidator(new Store(evidenceShapes)).validate(validationGraph);
         const issues = report.results.map((result) => ({
             category: 'extraction-gap',
             focus: result.focusNode?.value,

package/dist/graph/shacl-shapes.d.ts

@@ -0,0 +1,5 @@
+export declare const CG = "https://context-graph.dev/record#";
+export declare const RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
+export declare const XSD = "http://www.w3.org/2001/XMLSchema#";
+/** Graph-independent evidence constraints, evaluated over the union of all assertion graphs. */
+export declare const evidenceShapes: import("n3").Quad[];

package/dist/graph/shacl-shapes.js

@@ -0,0 +1,27 @@
+import { Parser } from 'n3';
+export const CG = 'https://context-graph.dev/record#';
+export const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
+export const XSD = 'http://www.w3.org/2001/XMLSchema#';
+/** Graph-independent evidence constraints, evaluated over the union of all assertion graphs. */
+export const evidenceShapes = new Parser().parse(`
+@prefix cg: <${CG}>.
+@prefix rdf: <${RDF}>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <${XSD}>.
+cg:FactShape a sh:NodeShape; sh:targetClass rdf:Statement;
+ sh:property [ sh:path cg:record; sh:minCount 1; sh:maxCount 1; sh:datatype xsd:string ];
+ sh:property [ sh:path rdf:subject; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path rdf:predicate; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path rdf:object; sh:minCount 1; sh:maxCount 1; sh:nodeKind sh:IRI ];
+ sh:property [ sh:path cg:authority; sh:minCount 1; sh:maxCount 1;
+   sh:in ("declared" "implemented" "validated" "observed" "inferred" "draft") ];
+ sh:property [ sh:path cg:path; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:commit; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:sourceDigest; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ];
+ sh:property [ sh:path cg:startLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
+ sh:property [ sh:path cg:endLine; sh:minCount 1; sh:datatype xsd:integer; sh:minInclusive 1 ];
+ sh:property [ sh:path cg:confidence; sh:minCount 1; sh:datatype xsd:decimal; sh:minInclusive 0; sh:maxInclusive 1 ];
+ sh:property [ sh:path cg:extractor; sh:minCount 1; sh:datatype xsd:string; sh:minLength 1 ].
+cg:NodeShape a sh:NodeShape; sh:targetClass cg:Node;
+ sh:property [ sh:path cg:kind; sh:minCount 1; sh:maxCount 1 ].
+`);

package/dist/index.d.ts

@@ -9,3 +9,4 @@ export * from "./visualization/visualization.js";
 export * from "./application/overlay.js";
 export * from "./graph/rdf.js";
 export * from "./extraction/providers.js";
+export * from './application/hydration-plan.js';

package/dist/index.js

@@ -10,3 +10,4 @@ export * from "./visualization/visualization.js";
 export * from "./application/overlay.js";
 export * from "./graph/rdf.js";
 export * from "./extraction/providers.js";
+export * from './application/hydration-plan.js';

package/dist/interfaces/cli.js

@@ -15,6 +15,7 @@ import { FileSnapshotStore } from "../storage/storage.js";
 import { diagnoseSetup } from "./doctor.js";
 import { runInit } from "./init.js";
 import { loadConfig } from "../configuration/config.js";
+import { planHydration } from '../application/hydration-plan.js';
 function valueAfter(args, name) {
     const index = args.indexOf(name);
     return index >= 0 ? args[index + 1] : undefined;
@@ -45,10 +46,41 @@ export function impactRequestFromArgs(args) {
         continuation: valueAfter(args, '--continuation'),
     };
 }
+function formatHydrationPlan(plan) {
+    const readiness = plan.issues.some((issue) => issue.severity === 'error')
+        ? 'blocked' : plan.ready ? 'ready' : 'needs review';
+    const lines = [
+        `Hydration plan: ${plan.repositoryId}`,
+        `Commit: ${plan.commitSha} (ref ${plan.ref})`,
+        `Configuration: ${plan.configurationDigest}`,
+        `Scope: ${plan.scope.mode}, ${plan.scope.complete ? 'complete' : 'incomplete'}; head ${plan.scope.headName}`,
+        `Documents: ${plan.budgets.selectedDocuments}/${plan.budgets.maxDocuments}; ${plan.budgets.totalSourceBytes} bytes total; ${plan.budgets.maxSourceBytes} bytes maximum per document`,
+        `Expected cache reuse: ${plan.estimates.reusableArtifacts}; extractions: ${plan.estimates.extractions}; unknown cache entries: ${plan.estimates.unknownArtifacts}`,
+        `Source/cache/budget checks: ${readiness}`,
+    ];
+    for (const document of plan.documents) {
+        lines.push(`- ${JSON.stringify(document.path)}: ${document.bytes} bytes, ${document.lines} lines; ${document.role}; source ${document.sourceAuthority}, extraction ${document.extractionReview}; cache ${document.cache.status}; selected by ${document.selectedBy.map((selector) => JSON.stringify(selector)).join(', ')}`);
+    }
+    for (const selection of plan.selections.filter((item) => item.status !== 'matched')) {
+        lines.push(`Selection ${JSON.stringify(selection.selector)}: ${selection.status}`);
+    }
+    for (const issue of plan.issues)
+        lines.push(`${issue.severity.toUpperCase()} [${issue.code}]: ${issue.message}`);
+    if (plan.scope.omittedDocuments.length) {
+        lines.push(`Documents omitted by scope limits: ${plan.scope.omittedDocuments.map((path) => JSON.stringify(path)).join(', ')}`);
+    }
+    if (plan.scope.ignoredSearchTerms)
+        lines.push(`Search terms omitted by scope limits: ${plan.scope.ignoredSearchTerms}`);
+    if (plan.excludedWorkingChanges.length) {
+        lines.push(`Working changes excluded: ${plan.excludedWorkingChanges.map((path) => JSON.stringify(path)).join(', ')}`);
+    }
+    lines.push(...plan.assumptions.map((assumption) => `Note: ${assumption}`));
+    return `${lines.join('\n')}\n`;
+}
 async function main() {
     const args = process.argv.slice(2);
     if (args.includes('--help') || !args.length) {
-        process.stdout.write('weftlet init|config show|config validate|doctor|hydrate|overlay|impact scan|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
+        process.stdout.write('weftlet init|config show|config validate|doctor|hydrate|overlay|impact scan|sparql|validate|govern|visualize|diagnostics show|invariant check|clean-compare|ontology compile\nUse --config <file>; queries accept --snapshot <id>. impact scan accepts --changed-file, --capability, --max-facts, --max-depth, --continuation.\nhydrate --plan [--json] [--ref <commit>] [--force-semantic] previews scope, cache and budgets without model calls or graph writes. Preview readiness is advisory and does not add a hydration gate.\ninit [--yes] [--directory <project>] [--config <file>] [--repository-id <id>] [--code-root <path>] [--semantic-source <path>] [--evidence-root <path>] [--model <name>] [--no-gitignore]\nSource flags may be repeated. init never overwrites an existing config. See README.md for examples.\n');
         return;
     }
     if (args[0] === 'init') {
@@ -85,6 +117,17 @@ async function main() {
         process.stdout.write(`${JSON.stringify(await diagnoseSetup(configPath), null, 2)}\n`);
         return;
     }
+    if (args[0] === 'hydrate' && args.includes('--plan')) {
+        const plan = await planHydration(configPath, {
+            ref: valueAfter(args, '--ref') ?? 'HEAD',
+            forceSemantic: args.includes('--force-semantic'),
+            request: impactRequestFromArgs(args),
+        });
+        process.stdout.write(args.includes('--json') ? `${JSON.stringify(plan, null, 2)}\n` : formatHydrationPlan(plan));
+        if (!plan.ready)
+            process.exitCode = 1;
+        return;
+    }
     const application = await createContextGraphApplication(configPath);
     const [command, subcommand] = args;
     if (command === 'sparql' || command === 'validate') {
@@ -120,9 +163,10 @@ async function main() {
         const controller = new AbortController();
         const cancel = () => controller.abort();
         process.once('SIGINT', cancel);
-        const scope = await application.scopeResolver?.resolve(impactRequestFromArgs(args));
+        const ref = valueAfter(args, '--ref') ?? 'HEAD';
+        const scope = await application.scopeResolver?.resolve(impactRequestFromArgs(args), ref);
         const result = await application.hydrator.hydrate({
-            ref: valueAfter(args, '--ref') ?? 'HEAD',
+            ref: scope?.commitSha ?? ref,
             headName: scope?.headName ?? application.headName,
             semanticSources: scope?.semanticSources,
             forceSemantic: args.includes('--force-semantic'),

package/dist/repository/git.d.ts

@@ -22,5 +22,5 @@ export declare class GitRepository {
     readFileAt(commitSha: string, path: string): Promise<string>;
     findTextAt(commitSha: string, needles: readonly string[], roots: readonly string[]): Promise<readonly GitTextMatch[]>;
     delta(baseSha: string | undefined, headSha: string, roots: readonly string[]): Promise<GitDelta>;
-    semanticSources(context: ExtractionContext, roots: readonly string[]): Promise<readonly SemanticSource[]>;
+    semanticSources(context: Pick<ExtractionContext, 'commitSha'>, roots: readonly string[]): Promise<readonly SemanticSource[]>;
 }

package/dist/repository/git.js

@@ -26,7 +26,7 @@ export class GitRepository {
     }
     async workingChanges() {
         const prefix = await this.run(['rev-parse', '--show-prefix']);
-        const output = await this.execute(['status', '--porcelain=v1', '-z', '--untracked-files=all'], false);
+        const output = await this.execute(['--no-optional-locks', 'status', '--porcelain=v1', '-z', '--untracked-files=all'], false);
         const records = output.split('\0');
         const paths = new Set();
         for (let index = 0; index < records.length; index++) {
@@ -65,6 +65,8 @@ export class GitRepository {
         }
     }
     async listFilesAt(commitSha, roots) {
+        if (roots.length === 0)
+            return [];
         const output = await this.run(['ls-tree', '-r', '--name-only', commitSha, '--', ...roots]);
         return output.split('\n').filter(Boolean).sort();
     }
@@ -107,6 +109,8 @@ export class GitRepository {
         return [...matches.values()].sort((left, right) => left.path.localeCompare(right.path));
     }
     async delta(baseSha, headSha, roots) {
+        if (roots.length === 0)
+            return { changedPaths: [], deletedPaths: [], renamedPaths: {} };
         if (!baseSha) {
             return {
                 changedPaths: await this.listFilesAt(headSha, roots),

package/docs/ADR-015-repository-document-identity.md

@@ -0,0 +1,57 @@
+# ADR-015: Canonical repository document identities
+
+Status: accepted for the document identity patch.
+
+## Problem
+
+The 0.2.0 semantic prompt requested `component:file:<path>` for every repository path,
+while deterministic Markdown linking used `document:<path>` for document files.
+Independently valid model assertions could assign Component and Document to the same
+file ID. Combining those facts correctly failed the single-kind SHACL constraint.
+This is an extraction identity defect, not evidence of contradictory product intent.
+
+## Decision
+
+Physical repository documents ending in `.md`, `.mdx` or `.txt` use
+`document:<repository-relative-path>` with kind Document. The semantic hydration
+boundary canonicalizes both reserved prefixes (`component:file:` and `document:`)
+for those extensions, using the same file-node policy as deterministic document links.
+This applies to Codex and module providers, before fact IDs, OML and artifact hashes
+are computed. Diagnostic references are remapped with the assertions.
+
+Paths retain their exact spelling and case; no basename matching, existence guess,
+ownership inference or concept merging is performed. Distinct business concepts such
+as `contract:retention` remain separate from the document that describes them.
+Every assertion, predicate, citation, confidence and label is retained. Raw provider
+output remains unchanged for inspection. Normalizing physical identity never approves
+an assertion or establishes that a referenced file exists. Snapshot assembly continues
+to mark semantic assertions inferred/unreviewed, independently of source approval.
+
+The RDF identity and SHACL shape stay unchanged. Other conflicting kinds still fail
+validation; this policy does not hide genuine conflicts or remove assertions.
+
+## Artifact and snapshot migration
+
+An implementation-owned `repositoryNodeIdentityVersion: 1` participates in every
+semantic artifact key, including custom providers with unchanged user configuration.
+Codex's prompt contract advances to v3. Assembly advances to `graph-v5`, records the
+policy version in the manifest and uses a different immutable snapshot ID even for
+the same commit/configuration. Historical snapshots and artifacts remain untouched
+and queryable; they are not relabeled as newly validated evidence.
+
+The first semantic refresh after upgrading requires extraction under the new contract.
+Unchanged subsequent refreshes reuse the new artifacts. Do not delete the old cache or
+force a revision as an upgrade workaround. Provider cost for that first refresh remains
+subject to the consumer's configured budgets and authorization.
+
+## Validation and limits
+
+The public regression uses two committed documents referencing the same implementation
+document, a deterministic module provider returning the conflicting legacy identities,
+real TypeScript indexing, deterministic links and RDF/SHACL. It checks preserved facts,
+citations and review authority, old-cache isolation and repeat reuse. An unnormalized
+negative control retains the SHACL failure. This is provider-independent deterministic
+coverage, not a claim that a private consumer's full extraction has passed.
+
+Operational-config cache reuse, broad identity diagnostics and full-graph SHACL latency
+remain separate work in the [1.0 plan](RELEASE-1.0-PLAN.md).

package/docs/ADR-016-hydration-preview.md

@@ -0,0 +1,91 @@
+# ADR-016: Read-only hydration planning
+
+Status: accepted and implemented for the 0.3.0 release plan; absent from 0.2.1.
+See [validation](VALIDATION.md) for recorded publication evidence.
+
+## Problem
+
+Consumers need to inspect document selection, extraction work and configured limits before
+invoking a model. Constructing the execution application is unsuitable for that preview because
+it loads configured repository adapters. Independently calculating source selection or cache keys
+would also allow the preview to disagree with hydration.
+
+## Decision
+
+Provide `hydrate --plan` with a human summary and `hydrate --plan --json` with a structured
+`hydration-plan` record. The public `planHydration(configPath, options)` function uses the same
+planner. `--ref` resolves once to a commit SHA, including for agent scope. Configuration and
+artifact storage are read from the current local environment, and the plan reports excluded
+working changes rather than using their bytes as committed evidence.
+
+The planner reads configuration, committed Git objects and existing semantic artifacts. It never
+constructs an execution application or loads a configured provider/adapter. It does not probe
+credentials, index TypeScript, collect runtime evidence, compile ontology artifacts, validate RDF,
+publish snapshots or create cache directories. Git status disables optional locks so the preview
+does not refresh the index. Source bodies and raw model output are not included in the result.
+
+Committed source selection, duplicate semantic-ID checks and budget assessment are shared with
+hydration. Provider identity and semantic artifact keys are pure functions shared with execution.
+Per-source entries identify exact path/content digest, stable semantic identity, inclusion selectors,
+line and UTF-8 byte counts, configured evidence role and matched source-approval patterns.
+
+Classification follows configuration, not an inference about prose meaning. `sourceAuthority`
+can be declared by an approved path while `extractionReview` remains unreviewed. A source matching
+`evidenceRoots` is labeled validation evidence, but selecting it still schedules semantic extraction.
+Recursive SOT traversal and default separation of validation documents belong to T07.
+
+## Cache and readiness semantics
+
+Cache inspection distinguishes these states without modifying evidence:
+
+| State | Planned meaning |
+| --- | --- |
+| `hit` | Readable artifact matches source identity/content, extraction contract, structure and output digest |
+| `miss` | No artifact exists for the current key; extraction is expected |
+| `forced` | Explicit force requests fresh extraction; no forced key or artifact is created by planning |
+| `corrupt` | Stored JSON, identity, structure or content digest is invalid; reuse is not claimed |
+| `unavailable` | The artifact could not be read; its reuse state is unknown |
+
+Corrupt/unavailable entries are not ordinary misses and are never silently repaired or overwritten.
+An artifact can remain reusable after a source rename or a different commit with unchanged content;
+its original path/commit need not equal the current path/commit. Hydration rebinds current provenance.
+The preview's estimate is conditional on unchanged inputs and storage state, not a guarantee against
+concurrent changes or a successful provider run.
+
+Limits apply to every selected document, including cache hits: `maxDocuments` bounds the document
+count and `maxSourceBytes` bounds each document's UTF-8 bytes. Total source bytes are reported but
+are not a token or monetary budget. Selection is never trimmed to make these limits pass.
+
+Budget, duplicate-ID and cache errors produce a blocked plan. Missing/no-document selectors and
+omissions from existing bounded agent selection produce warnings and an incomplete scope that
+needs review. Both yield `ready: false` and CLI exit 1, while retaining the full plan. `ready: true`
+and exit 0 mean only that these planning checks passed; execution prerequisites and product-intent
+completeness have not been established.
+
+Warnings do not create new hard hydration gates. Existing hydration behavior can still publish a
+new graph after deleting the last selected document, removing its old relationships. The preview
+exposes that empty match for review. An explicitly empty source list is complete and selects none.
+
+## Compatibility
+
+The shared key/output-digest implementation preserves the released 0.2.1 semantic artifact contract,
+including document identity-policy invalidation. Valid existing artifacts remain reusable; no
+semantic re-extraction is required merely by this refactor. Operational configuration changes can
+still affect cache identity/storage profiles; that is separate T03 work.
+
+The empty-source fix changes graph assembly behavior: empty roots previously could enumerate all
+repository documents. Assembly advances to `graph-v6` so corrected snapshots cannot collide with
+immutable older snapshots at the same commit/configuration. Old snapshots remain readable and
+unchanged. The requested ref also controls agent source selection instead of always using HEAD.
+
+## Validation and next work
+
+Public regressions cover committed and dirty inputs, unchanged index/files, absent or side-effecting
+provider/adapter modules, empty scope, warm/cold/forced cache, malformed and unreadable artifacts,
+released-key compatibility, budgets, duplicate IDs, missing selectors and historical agent scope
+with explicit omissions. Existing rename/delete lifecycle checks remain applicable. Full validation
+and release evidence are recorded separately in [VALIDATION.md](VALIDATION.md).
+
+This increment makes no SHACL performance change. T05 will measure graph construction and full
+validation before selecting caching, incremental validation or backend changes. See the
+[1.0 plan](RELEASE-1.0-PLAN.md) for bounded follow-ups and the separate 2.0 platform direction.

package/docs/ADR-017-shacl-validation-union.md

@@ -0,0 +1,102 @@
+# ADR-017: Validate a temporary RDF union without changing canonical evidence
+
+Status: accepted and implemented for the focused 0.3.0 release plan; absent from 0.2.1.
+See [validation](VALIDATION.md) for checks and recorded publication evidence.
+
+## Problem and measured mechanism
+
+The canonical RDF dataset has one named relationship graph per assertion, with assertion records,
+node metadata and provenance in the default graph. The JavaScript `rdf-validate-shacl` validator
+uses graph-wildcard dataset lookups through clownface. N3 indexes graphs first, so a lookup with
+no graph restriction visits each graph before consulting its subject/predicate/object indexes.
+Repeating those lookups for each focus node and shape property becomes expensive as assertion
+graphs grow.
+
+Bounded public experiments measured this cost in the current layout and compared an indexed union
+using the same validator and shapes. They establish a synthetic scaling problem and a useful
+optimization; they do not reproduce the private consumer's complete generation or diagnose its
+reported 30-minute timeout. The consumer's total graph size and other phase costs remain unknown.
+
+## Decision
+
+For each `RdfEvidenceGraph.validate()` call, build a temporary single-default-graph N3 store from
+every subject/predicate/object triple in the canonical dataset. Run the unchanged SHACL shapes
+and validator on that store, and retain the existing explicit citation-order check. The canonical
+named dataset remains unchanged and continues to serve SPARQL, fact projection and serialization.
+
+This is still full-graph validation on every refresh. It introduces no GraphDB service, native/Rust
+validator, incremental validation or validation-result cache. The short-lived union is not retained
+as another persistent graph or used to replace canonical evidence.
+
+RDF graphs are sets: identical SPO triples from multiple named graphs collapse in the validation
+projection. The present shapes already use union lookups and node/value sets, and do not inspect
+graph identity. Separate assertion records and their cited provenance remain intact in the original
+dataset; no assertion, shape or constraint is removed. A future graph-sensitive shape requires an
+explicit compatibility decision and new equivalence tests before using this projection.
+
+## Bounded public measurements
+
+Environment: Linux cloud workspace, Node 24.19.0, N3 2.7.2 and rdf-validate-shacl 0.6.5.
+Each case ran once in a fresh child process with a 45-second hard deadline. The prototype copied
+the same original dataset into a single graph only for validation; production-patch verification
+is recorded separately in [VALIDATION.md](VALIDATION.md).
+
+Fixtures contain public synthetic Workflow-to-Component assertions, cited metadata and one named
+relationship graph per fact. Shared topology repeats 50 nodes; unique topology uses two distinct
+nodes per fact. Timings below are seconds, with union construction separated from SHACL. RSS is
+the process sample after validation, including Node, libraries and both datasets, not a measurement
+of projection memory alone.
+
+| Facts / topology | Original quads | Original SHACL | Union construction | Union SHACL | Union RSS MiB |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| 1,000 shared | 20,203 | 3.889 | 0.061 | 0.239 | 352.3 |
+| 2,000 shared | 40,203 | 19.550 | 0.102 | 0.410 | 440.0 |
+| 1,000 unique | 28,003 | 5.731 | 0.104 | 0.308 | 352.6 |
+| 2,000 unique | 56,003 | 31.502 | 0.153 | 0.467 | 457.9 |
+| 5,000 shared | 100,203 | No verdict before process deadline | 0.251 | 1.131 | 825.5 |
+| 10,000 unique | 280,003 | Not measured | 0.779 | 2.119 | 1,287.3 |
+
+The 5,000-fact original-mode process was terminated after 45.051 seconds total elapsed time,
+including startup and construction. It produced no SHACL verdict; this is not an inferred
+completed validation duration. For the 10,000-fact unique union case, initial canonical graph
+construction took 1.058 seconds in addition to the times shown. Its roughly 1.3 GiB RSS is a
+material memory cost and should be measured on larger or different workloads.
+
+These single runs are not statistical latency estimates or production guarantees. The experiment
+does not measure semantic extraction, SCIP, SPARQL, serialization, snapshot publication or disk
+storage. No private consumer data or live provider calls were used. The repository benchmark
+`pnpm benchmark:shacl` reproduces bounded public comparisons using the current build; retain its
+configuration, versions and raw output when comparing results.
+
+## Correctness and compatibility
+
+The prototype compared sorted conformance issues between the original and union paths across
+valid repeated relationships/attributes, conflicting node kinds, out-of-range confidence, invalid
+citations, empty citation paths, colliding assertion IDs and combined violations. All seven paired
+cases agreed, and before/after serialization hashes preserved the canonical named dataset.
+
+Production regressions must compare both paths for all supported constraints, including negative
+cases, and preserve named-graph SPARQL results, fact projection, serialization and repeated calls.
+Those checks establish parity for the current shapes, not arbitrary future SHACL rules. Failed
+validation must continue to prevent publication and preserve the previous valid snapshot.
+
+The optimization changes neither facts nor evidence authority and does not require a new semantic
+cache identity. It does not make old evidence eligible for current-diff gates. Normal snapshot
+commit, configuration, scope and provenance checks continue to apply.
+
+## Release boundary and remaining T05 work
+
+Ship the demonstrated cold-validation improvement after its independent review and required
+validation, ahead of unrelated 1.0 work. A first graph cannot benefit from reusing a successful
+validation result that does not yet exist. Private consumer acceptance remains pending after release.
+
+Early identity checks, detailed progress/elapsed reporting, validation-result caching and safe
+incremental validation remain open. The validator's main validation loop is synchronous; timers
+on the same JavaScript thread cannot show a live heartbeat while it is blocked. Keep phase
+boundaries honest and evaluate bounded worker execution before promising in-phase progress or
+cancellation. Any later cached or incremental result must account for graph content, shapes,
+validator/policy versions, deletions, affected relationships and global constraints, with full
+validation when safe scope cannot be established.
+
+Native engines and optional persistent database backends remain measurement-driven alternatives,
+not prerequisites of this release. See the [1.0 task list](RELEASE-1.0-PLAN.md).

package/docs/CONSUMER-TRIAL.md

@@ -6,11 +6,12 @@ development workspace. Saved evidence queries work without a model login. Hydrat
 prose uses the configured provider; an OAuth failure may block new extraction but does not
 remove an already published snapshot.
 
-Use a pinned, reviewed toolkit version and its matching documentation. The current `main`
-contains features beyond npm `0.1.2`, including explicit routing declarations. To evaluate
-those features before their release, use both locally prepared package archives from the
-same reviewed commit; see [release preparation](RELEASING.md). Record that commit separately
-from the package version, which does not change with every development commit.
+Use a pinned, reviewed toolkit version and its matching documentation. npm `0.2.0` includes
+explicit routing declarations, committed TypeScript source projects, document linkage and
+the improved impact ranking. To evaluate a newer, unreleased commit, use both locally prepared
+package archives from that same reviewed commit; see [release preparation](RELEASING.md).
+Record the commit separately from the package version, which does not change with every
+development commit.
 
 ## Choose tasks before querying