@weftlet/toolkit 0.4.0 → 0.6.0

package/dist/application/hydrator.js

@@ -4,14 +4,16 @@ import { repositoryFileExtractorIdentity, repositoryContractExtractorIdentity, s
 import { sha256, stableJson } from "../infrastructure/util.js";
 import { documentLinkerIdentity, linkSemanticDocuments } from "../extraction/document.js";
 import { repositoryNodeIdentityVersion } from '../extraction/repository-nodes.js';
-import { selectSemanticInputs } from '../extraction/semantic-sources.js';
+import { selectHydrationInputs, classifyHydratedFacts, documentSelectionIdentity } from './hydration-inputs.js';
 import { runtimeEvidenceFacts } from "../extraction/runtime-evidence.js";
 import { matchesRepositoryGlob } from "../repository/glob.js";
 import { RdfEvidenceGraph } from "../graph/rdf.js";
 import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
 import { SemanticProviderError } from '../extraction/semantic-process.js';
 import { hydrateSemanticSources } from './semantic-run.js';
-import { graphAssemblyVersion, graphAssemblyIdentity } from './extraction-identity.js';
+import { graphAssemblyVersion, graphAssemblyIdentity, documentExtractionVersions } from './extraction-identity.js';
+import { assertGraphIdentities } from '../graph/identity-preflight.js';
+import { measureValidationPhase } from '../graph/validation-progress.js';
 export class ContextGraphHydrator {
     dependencies;
     constructor(dependencies) {
@@ -49,10 +51,16 @@ export class ContextGraphHydrator {
         const commitSha = await repository.resolveRef(options.ref);
         if (options.requireClean === true)
             await repository.requireExactCleanCommit(commitSha);
-        const selectedSemanticSources = options.semanticSources ?? config.semanticSources;
-        const selection = await selectSemanticInputs(repository, commitSha, selectedSemanticSources, config.semantic);
-        if (selection.issues.some((issue) => issue.severity === 'error')) {
-            throw new Error(selection.issues.map((issue) => issue.message).join('\n'));
+        if (config.documentTraversal && options.semanticSources !== undefined) {
+            throw new Error('Traversal hydration does not accept a direct semanticSources override');
+        }
+        const selection = await selectHydrationInputs(repository, commitSha, config, options.semanticSources);
+        const selectedSemanticSources = selection.documentSelection?.semanticPaths ?? options.semanticSources ?? config.semanticSources;
+        const scopePaths = selection.documentSelection?.documentPaths ?? selectedSemanticSources;
+        if (selection.documentSelection?.complete === false || selection.issues.some((issue) => issue.severity === 'error')) {
+            throw new Error(config.documentTraversal
+                ? 'Traversal hydration blocked: run hydrate --plan --json to review coverage, roles and budgets before retrying.'
+                : selection.issues.map((issue) => issue.message).join('\n'));
         }
         const { sources } = selection;
         await this.dependencies.ontologyCompiler.checkReady?.();
@@ -62,7 +70,7 @@ export class ContextGraphHydrator {
             : (prior?.manifest.commitSha ?? (await repository.firstParent(commitSha)));
         const allRoots = [
             ...new Set([
-                ...selectedSemanticSources,
+                ...scopePaths,
                 ...config.codeRoots,
                 ...config.evidenceRoots,
                 ...contractPatternRoots(config.contractPatterns),
@@ -84,11 +92,12 @@ export class ContextGraphHydrator {
             ? `runtime-${runtimeRevision.digest.slice(0, 12)}`
             : undefined;
         const configurationDigest = sha256(stableJson(config));
-        const scopeDigest = sha256(stableJson(selectedSemanticSources)).slice(0, 12);
+        const scopeDigest = sha256(stableJson(scopePaths)).slice(0, 12);
         const effectiveSuffix = [`graph-v${graphAssemblyVersion}`, `links-${documentLinkerIdentity.version}`, `files-${repositoryFileExtractorIdentity.version}`,
             `contracts-${repositoryContractExtractorIdentity.version}`,
             `symbols-${scipSymbolExtractorIdentity.version}`,
             ...(config.routingSources?.length ? [`routes-${routingExtractorIdentity.version}`] : []),
+            ...(selection.documentSelection ? [documentSelectionIdentity, selection.documentSelection.digest.slice(0, 16)] : []),
             configurationDigest.slice(0, 16), scopeDigest, baseSuffix, runtimeSuffix].filter(Boolean).join('-');
         const snapshotId = effectiveSuffix ? `${commitSha}-${effectiveSuffix}` : commitSha;
         const context = {
@@ -121,7 +130,7 @@ export class ContextGraphHydrator {
                 commitSha,
             })),
         })));
-        const semanticFacts = hydrated.flatMap(({ source, artifact }) => artifact.facts.map((fact) => ({
+        const semanticFacts = classifyHydratedFacts(hydrated.flatMap(({ source, artifact }) => artifact.facts.map((fact) => ({
             ...fact,
             id: sha256(stableJson({ artifact: artifact.key, fact: fact.id, path: source.path })),
             provenance: {
@@ -134,7 +143,7 @@ export class ContextGraphHydrator {
             authority: 'inferred',
             extractionReview: 'unreviewed',
             sourceAuthority: (config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern)) ? 'declared' : 'inferred'),
-        })));
+        }))), selection.documentEvidence);
         checkpoint();
         options.onProgress?.({ phase: 'index-typescript' });
         const scip = await this.dependencies.staticIndexer.index(context);
@@ -153,15 +162,25 @@ export class ContextGraphHydrator {
                     });
             }
         }
-        const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path));
+        const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path), { canonicalReferences: !config.documentTraversal });
         diagnostics.push(...documents.diagnostics);
-        diagnostics.push(...routing.diagnostics);
+        diagnostics.push(...routing.diagnostics, ...(selection.documentEvidence?.diagnostics ?? []));
         checkpoint();
-        options.onProgress?.({ phase: 'compile-semantic-rdf' });
-        const ontology = await this.dependencies.ontologyCompiler.compile(snapshotId, artifacts, context);
         const runtimeEvidence = runtimeRevision.references;
         const runtimeFacts = runtimeEvidenceFacts(context, runtimeEvidence);
-        const facts = [...semanticFacts, ...documents.facts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
+        const structuralFacts = selection.documentEvidence?.facts.map(fact => ({ ...fact, snapshotId })) ?? [];
+        const facts = [...semanticFacts, ...classifyHydratedFacts(documents.facts, selection.documentEvidence),
+            ...structuralFacts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
+        const validationCounts = { factCount: facts.length };
+        await measureValidationPhase('validate-identities', validationCounts, options.onProgress, () => {
+            const report = assertGraphIdentities(facts, {
+                repositoryId: config.repositoryId, includeDetails: options.validationDetails,
+            });
+            validationCounts.nodeCount = report.nodeCount;
+        });
+        checkpoint();
+        options.onProgress?.({ phase: 'compile-semantic-rdf' });
+        const ontology = await this.dependencies.ontologyCompiler.compile(snapshotId, artifacts, context);
         const invariants = (await Promise.all(this.dependencies.invariantEvaluators.map((evaluator) => evaluator.evaluate(snapshotId, facts)))).flat();
         const failures = invariants.filter((invariant) => invariant.status === 'fail');
         if (failures.length > 0) {
@@ -188,6 +207,7 @@ export class ContextGraphHydrator {
             unresolved,
             diagnostics,
             semanticSources: selectedSemanticSources,
+            ...(selection.documentSelection ? { documentSelection: selection.documentSelection } : {}),
         };
         const contentDigest = sha256(stableJson(logical));
         const snapshot = {
@@ -199,10 +219,12 @@ export class ContextGraphHydrator {
                 baseSha,
                 snapshotId,
                 configurationDigest,
-                scope: selectedSemanticSources,
+                scope: scopePaths,
+                ...(selection.documentSelection ? { documentSelection: selection.documentSelection } : {}),
                 ontologyVersion: config.semantic.ontologyVersion,
                 extractorVersions: {
                     assembly: graphAssemblyIdentity,
+                    ...documentExtractionVersions(config),
                     [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
                     [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
                     'repository-node-identity': String(repositoryNodeIdentityVersion),
@@ -231,8 +253,8 @@ export class ContextGraphHydrator {
             diagnostics,
         };
         checkpoint();
-        options.onProgress?.({ phase: 'validate-shacl' });
-        const validation = await new RdfEvidenceGraph(snapshot).validate();
+        const graph = await measureValidationPhase('build-rdf-evidence', validationCounts, options.onProgress, () => new RdfEvidenceGraph(snapshot));
+        const validation = await graph.validate({ onProgress: (event) => options.onProgress?.({ ...event, nodeCount: validationCounts.nodeCount }) });
         if (!validation.conforms) {
             throw new Error(`Graph extraction failed SHACL validation: ${JSON.stringify(validation.issues)}`);
         }

package/dist/application/overlay.d.ts

@@ -3,4 +3,6 @@ import type { GraphSnapshot } from "@weftlet/core";
 import { GitRepository } from "../repository/git.js";
 import { FileSnapshotStore } from "../storage/storage.js";
 /** Conservative code overlay: reindex implementation; invalidate changed prose, never synthesize intent. */
-export declare function buildWorkingOverlay(repository: GitRepository, config: ContextGraphConfig, store: FileSnapshotStore, storageRoot: string, baseSnapshotId: string, agent?: string): Promise<GraphSnapshot>;
+export declare function buildWorkingOverlay(repository: GitRepository, config: ContextGraphConfig, store: FileSnapshotStore, storageRoot: string, baseSnapshotId: string, agent?: string, options?: {
+    readonly validationDetails?: boolean;
+}): Promise<GraphSnapshot>;

package/dist/application/overlay.js

@@ -11,14 +11,18 @@ import { contextPackEvidenceAuthority } from "../graph/query.js";
 import { inWorkingScope, readWorkingContent } from "../repository/working-content.js";
 import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
 import { RdfEvidenceGraph } from '../graph/rdf.js';
-import { graphAssemblyIdentity } from './extraction-identity.js';
+import { graphAssemblyIdentity, documentExtractionVersions } from './extraction-identity.js';
+import { assertGraphIdentities } from '../graph/identity-preflight.js';
+import { selectHydrationInputs, documentSelectionIdentity } from './hydration-inputs.js';
+import { retainOverlayDocumentFact, overlayDocumentRoots, overlayDocumentManifest, overlayDocumentNotes } from './document-overlay.js';
+import { documentEvidenceExtractorIdentity } from '../extraction/document-evidence.js';
 const execute = promisify(execFile);
 const implementationExtractors = new Set([
     'repository-file-extractor', 'repository-contract-extractor', 'scip-symbol-extractor',
     routingExtractorIdentity.id,
 ]);
 /** Conservative code overlay: reindex implementation; invalidate changed prose, never synthesize intent. */
-export async function buildWorkingOverlay(repository, config, store, storageRoot, baseSnapshotId, agent = 'default') {
+export async function buildWorkingOverlay(repository, config, store, storageRoot, baseSnapshotId, agent = 'default', options = {}) {
     const base = await store.read(baseSnapshotId);
     if (!base || base.manifest.overlayDigest)
         throw new Error('Overlay requires a committed base snapshot');
@@ -38,6 +42,7 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
         commitSha: head, snapshotId: baseSnapshotId, changedPaths: [], deletedPaths: [], renamedPaths: {},
     }, routingSources, { allowMissingSources: true });
     const roots = [...config.codeRoots, ...config.semanticSources, ...config.evidenceRoots,
+        ...overlayDocumentRoots(config),
         ...config.contractPatterns, ...scipSourcePatterns(config.scip.sourceProject),
         ...routingSources, ...baseRouting.referencedPaths];
     const excluded = [];
@@ -63,6 +68,8 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
         repositoryContractExtractor: repositoryContractExtractorIdentity,
         scipSymbolExtractor: scipSymbolExtractorIdentity,
         ...(routingSources.length ? { routingExtractor: routingExtractorIdentity } : {}),
+        ...(config.documentTraversal ? { documentSelection: documentSelectionIdentity,
+            documentEvidence: documentEvidenceExtractorIdentity } : {}),
     })).slice(0, 32)}`;
     const affected = new Set(captured.map((file) => file.path));
     const extracted = await withCommittedTree(repository.root, head, async (root) => {
@@ -89,6 +96,13 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
             deletedPaths: captured.filter((f) => f.content === null).map((f) => f.path),
             renamedPaths: {},
         };
+        const documents = config.documentTraversal
+            ? await selectHydrationInputs(draftRepository, context.commitSha, config) : undefined;
+        const documentSelection = documents?.documentSelection ? {
+            ...documents.documentSelection,
+            // Temporary Git commit timestamps are not evidence identity. Bind drafts to captured bytes.
+            digest: sha256(stableJson({ ...documents.documentSelection, digest: undefined, baseSha: head, overlayDigest })),
+        } : undefined;
         const scip = await new ScipTypeScriptIndexer(config.scip, storageRoot).index(context);
         const routing = await extractRoutingDeclarations(draftRepository, context, routingSources, { allowMissingSources: true });
         const uncapturedEndpoints = routing.referencedPaths.filter((path) => !inWorkingScope(path, roots));
@@ -101,36 +115,41 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
             new ScipSymbolExtractor(draftRepository, scip),
         ];
         const facts = [...(await Promise.all(extractors.map((e) => e.extract(context)))).flat(),
-            ...routing.facts].map(({ id: _id, ...fact }) => {
+            ...routing.facts, ...(documents?.documentEvidence?.facts ?? [])].map(({ id: _id, ...fact }) => {
             const draft = { ...fact, snapshotId, authority: 'draft',
-                ...(fact.extractor.id === routingExtractorIdentity.id ? { sourceAuthority: 'draft' } : {}),
+                ...(fact.extractor.id === routingExtractorIdentity.id || fact.sourceRole !== undefined ? { sourceAuthority: 'draft' } : {}),
                 provenance: { ...fact.provenance, commitSha: head } };
             return { ...draft, id: sha256(stableJson(draft)) };
         });
-        const diagnostics = routing.diagnostics.map(({ id: _id, ...item }) => {
+        const diagnostics = [...routing.diagnostics, ...(documents?.documentEvidence?.diagnostics ?? [])].map(({ id: _id, ...item }) => {
             const draft = { ...item, provenance: item.provenance.map((span) => ({ ...span, commitSha: head })) };
             return { ...draft, id: sha256(stableJson(draft)) };
         }).sort((a, b) => a.id.localeCompare(b.id));
-        return { facts, diagnostics };
+        return { facts, diagnostics, documents, documentSelection };
     });
-    const retained = base.facts.filter((fact) => !implementationExtractors.has(fact.extractor.id) &&
+    const retained = base.facts.filter((fact) => retainOverlayDocumentFact(fact, extracted.documents, affected) &&
+        !implementationExtractors.has(fact.extractor.id) &&
         contextPackEvidenceAuthority(fact) !== 'implemented' &&
         fact.extractor.id !== 'otlp-runtime-evidence' &&
         !affected.has(fact.provenance.path) &&
         ![fact.subject, fact.object].some((node) => [...affected].some((path) => node.id === `component:file:${path}` || node.id === `document:${path}` || node.label === path))).map((fact) => ({ ...fact, snapshotId }));
     const allFacts = [...retained, ...extracted.facts].sort((a, b) => a.id.localeCompare(b.id));
+    assertGraphIdentities(allFacts, { repositoryId: config.repositoryId, includeDetails: options.validationDetails });
     const overlay = {
         ...base,
         manifest: {
             ...base.manifest, snapshotId, baseSha: head, overlayDigest, overlayCapture,
+            ...overlayDocumentManifest(extracted.documentSelection),
             extractorVersions: { ...base.manifest.extractorVersions,
                 assembly: graphAssemblyIdentity,
+                ...documentExtractionVersions(config),
                 [repositoryFileExtractorIdentity.id]: repositoryFileExtractorIdentity.version,
                 [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
                 [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
                 ...(routingSources.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}) },
-            generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length
-                ? { facts: allFacts, diagnostics: extracted.diagnostics } : allFacts)),
+            generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length || config.documentTraversal
+                ? { facts: allFacts, diagnostics: extracted.diagnostics,
+                    ...(extracted.documentSelection ? { documentSelection: extracted.documentSelection } : {}) } : allFacts)),
         },
         facts: allFacts,
         invariants: base.invariants.map((rule) => ({
@@ -139,6 +158,7 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
         runtimeEvidence: [],
         unresolved: [
             'Draft implementation overlay: changed semantic documents require explicit re-extraction.',
+            ...overlayDocumentNotes(config),
             `Changes outside configured source scope are excluded: ${excluded.join(', ') || '(none)'}`,
         ],
         diagnostics: extracted.diagnostics,

package/dist/application/saved-evidence.d.ts

@@ -15,13 +15,13 @@ export declare class SavedEvidenceQueryService implements ImpactQueryService {
             incompatibleExtractors?: readonly string[] | undefined;
             stale: boolean;
             configurationMatches: boolean;
-            overlayFreshness?: "current" | "stale" | "unknown" | undefined;
+            overlayFreshness?: "unknown" | "current" | "stale" | undefined;
             capturedWorkingChanges?: string[] | undefined;
             changedSinceCapture?: string[] | undefined;
             currentCommit: string | undefined;
             excludedWorkingChanges: string[];
         };
-        snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture"> & Partial<Pick<import("@weftlet/core").SnapshotManifest, "extractorVersions">>;
+        snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture"> & Partial<Pick<import("@weftlet/core").SnapshotManifest, "extractorVersions" | "documentSelection">>;
         capabilities: readonly import("@weftlet/core").GraphNode[];
         facts: readonly import("@weftlet/core").GraphFact[];
         invariants: readonly import("@weftlet/core").InvariantResult[];

package/dist/application/semantic-run.d.ts

@@ -7,6 +7,9 @@ export interface HydrationProgressEvent {
     readonly total?: number;
     readonly sourceId?: string;
     readonly elapsedMs?: number;
+    readonly status?: 'started' | 'completed' | 'failed';
+    readonly factCount?: number;
+    readonly nodeCount?: number;
 }
 /** Settle active workers before reporting failure; never publish snapshots here. */
 export declare function hydrateSemanticSources(options: {

package/dist/application/semantic-scope.d.ts

@@ -2,7 +2,7 @@ import type { ImpactScanRequest } from '@weftlet/core';
 import type { ContextGraphConfig } from '../configuration/config.js';
 import type { GitRepository } from '../repository/git.js';
 export interface SemanticScope {
-    readonly mode: 'configured' | 'agent';
+    readonly mode: 'configured' | 'agent' | 'traversal';
     readonly selectors: readonly string[];
     readonly digest: string;
     readonly headName: string;

package/dist/application/semantic-scope.js

@@ -5,8 +5,8 @@ export async function resolveSemanticScope(config, repository, commitSha, reques
     const agent = config.agentContext
         ? await new AgentSemanticScopeResolver(config, repository).resolve(request, commitSha)
         : undefined;
-    const selectors = agent?.semanticSources ?? config.semanticSources;
-    return { mode: agent ? 'agent' : 'configured', selectors, digest: sha256(stableJson(selectors)),
+    const selectors = config.documentTraversal?.entryPoints ?? agent?.semanticSources ?? config.semanticSources;
+    return { mode: config.documentTraversal ? 'traversal' : agent ? 'agent' : 'configured', selectors, digest: sha256(stableJson(selectors)),
         headName: agent?.headName ?? 'default', omittedDocuments: agent?.omittedDocuments ?? [],
         ignoredSearchTerms: agent?.ignoredSearchTerms ?? 0 };
 }

package/dist/configuration/cache.js

@@ -2,6 +2,7 @@
 import { homedir } from 'node:os';
 import { resolve } from 'node:path';
 import { sha256, stableJson } from "../infrastructure/util.js";
+import { assertDocumentTraversalConfig } from './document-traversal.js';
 export function resolveStorageRoot(repositoryRoot, config, userCacheRoot = resolve(homedir(), '.cache', 'context-graph')) {
     if (config.storageScope !== 'user')
         return resolve(repositoryRoot, config.storagePath);
@@ -14,14 +15,25 @@ const codexOperationalFields = new Set([
     'timeoutMs', 'retries', 'concurrency', 'maxDocuments', 'maxSourceBytes',
 ]);
 export function semanticCachePartition(repositoryRoot, config) {
+    if (config.documentTraversal)
+        assertDocumentTraversalConfig(config.documentTraversal);
     const semantic = config.semantic.provider === 'codex-cli'
         ? Object.fromEntries(Object.entries(config.semantic).filter(([field]) => !codexOperationalFields.has(field)))
         : config.semantic;
-    return sha256(stableJson({ config: { ...config, semantic }, checkout: resolve(repositoryRoot) }));
+    // Traversal/role metadata is assembled after the built-in Codex artifact is validated.
+    // Keep the old partition exactly when enabling it; do not ignore unknown configuration.
+    const { documentTraversal: _traversal, ...directConfig } = config;
+    const inputs = config.semantic.provider === 'codex-cli' ? directConfig : config;
+    return sha256(stableJson({ config: { ...inputs, semantic }, checkout: resolve(repositoryRoot) }));
 }
 /** Snapshot profiles retain the full configuration. Only Codex user artifacts share operational changes. */
 export function resolveSemanticStorageRoots(repositoryRoot, config, userCacheRoot = resolve(homedir(), '.cache', 'context-graph')) {
     const legacy = resolveStorageRoot(repositoryRoot, config, userCacheRoot);
+    // Module providers can inspect arbitrary extraction context. Opt-in traversal isolates their
+    // artifacts even in workspace storage, while unchanged committed sources still reuse them.
+    if (config.documentTraversal && config.semantic.provider === 'module') {
+        return { primary: resolve(legacy, 'document-semantic-v1', semanticCachePartition(repositoryRoot, config)) };
+    }
     if (config.storageScope !== 'user' || config.semantic.provider !== 'codex-cli')
         return { primary: legacy };
     return { primary: resolve(userCacheRoot, 'semantic-artifacts-v1', semanticCachePartition(repositoryRoot, config)), legacy };

package/dist/configuration/config.d.ts

@@ -1,4 +1,5 @@
 import { type ScipSourceProject } from './scip-project.js';
+import { type DocumentTraversalConfig } from './document-traversal.js';
 export interface ContextGraphConfig {
     readonly schemaVersion: 1;
     readonly repositoryRoot?: string;
@@ -6,6 +7,7 @@ export interface ContextGraphConfig {
     readonly namespace: string;
     readonly adapter?: string;
     readonly semanticSources: readonly string[];
+    readonly documentTraversal?: DocumentTraversalConfig;
     /** Explicit owner-approved source paths/globs; other model-derived prose remains inferred. */
     readonly approvedIntentSources?: readonly string[];
     readonly codeRoots: readonly string[];

package/dist/configuration/config.js

@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { assertScipSourceProject, assertSourceProjectInvocation } from './scip-project.js';
+import { assertDocumentTraversalConfig } from './document-traversal.js';
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
@@ -34,6 +35,10 @@ function validateConfig(value, absolutePath) {
     if (value.approvedIntentSources !== undefined) {
         requireConfig(isStringArray(value.approvedIntentSources), 'approvedIntentSources must be an array of strings', absolutePath);
     }
+    if (value.documentTraversal !== undefined) {
+        assertDocumentTraversalConfig(value.documentTraversal);
+        requireConfig(value.agentContext === undefined, 'documentTraversal and bounded agentContext selection cannot be combined', absolutePath);
+    }
     if (value.routingSources !== undefined) {
         requireConfig(isStringArray(value.routingSources) && value.routingSources.length <= 16 &&
             new Set(value.routingSources).size === value.routingSources.length &&
@@ -113,7 +118,7 @@ function validateConfig(value, absolutePath) {
                 requireConfig(auth.apiKeyEnv === undefined || (typeof auth.apiKeyEnv === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(auth.apiKeyEnv)), 'semantic.auth.apiKeyEnv must name an environment variable', absolutePath);
             }
         }
-        for (const [field, max] of [['timeoutMs', 300_000], ['retries', 2], ['maxDocuments', 1000], ['maxSourceBytes', 1_048_576]]) {
+        for (const [field, max] of [['timeoutMs', 600_000], ['retries', 2], ['maxDocuments', 1000], ['maxSourceBytes', 1_048_576]]) {
             const item = value.semantic[field];
             if (item !== undefined)
                 requireConfig(Number.isInteger(item) && Number(item) >= (field === 'retries' ? 0 : 1) && Number(item) <= max, `semantic.${field} exceeds supported bounds`, absolutePath);

package/dist/configuration/document-traversal.d.ts

@@ -0,0 +1,7 @@
+import { type DocumentTraversalOptions } from '../extraction/document-traversal.js';
+import { type DocumentClassificationPolicy } from '../extraction/document-classification.js';
+/** Explicit execution mode; the direct semanticSources list is not added to its closure. */
+export interface DocumentTraversalConfig extends Omit<DocumentTraversalOptions, 'maxDocuments' | 'maxSourceBytes'> {
+    readonly policy?: DocumentClassificationPolicy;
+}
+export declare function assertDocumentTraversalConfig(value: unknown): asserts value is DocumentTraversalConfig;

package/dist/configuration/document-traversal.js

@@ -0,0 +1,13 @@
+import { validateDocumentTraversalOptions } from '../extraction/document-traversal.js';
+import { parseDocumentClassificationPolicy } from '../extraction/document-classification.js';
+export function assertDocumentTraversalConfig(value) {
+    const fields = ['entryPoints', 'roots', 'maxReadDocuments', 'maxReadBytes', 'maxDocumentBytes', 'policy'];
+    if (!value || typeof value !== 'object' || Array.isArray(value) ||
+        Object.keys(value).some(key => !fields.includes(key))) {
+        throw new Error('documentTraversal must contain only supported traversal options and policy');
+    }
+    const options = value;
+    validateDocumentTraversalOptions(options);
+    if (options.policy !== undefined)
+        parseDocumentClassificationPolicy(options.policy);
+}

package/dist/extraction/canonical-document-links.d.ts

@@ -0,0 +1,28 @@
+export interface CanonicalDocumentDefinition {
+    readonly target: string;
+    readonly startLine: number;
+    readonly endLine: number;
+}
+export interface CanonicalDocumentLink {
+    readonly kind: 'link' | 'ambiguous-reference';
+    readonly target?: string;
+    readonly startLine: number;
+    readonly endLine: number;
+    readonly identifier?: string;
+    readonly definitions?: readonly CanonicalDocumentDefinition[];
+    /** Reference uses and their destination declarations can occur on different lines. */
+    readonly referenceStartLine?: number;
+    readonly referenceEndLine?: number;
+}
+/** Explicit Markdown links only: this deliberately does not change the graph linker's policy. */
+export declare function canonicalDocumentLinks(content: string): readonly CanonicalDocumentLink[];
+export type CanonicalDocumentLinkResolution = {
+    readonly kind: 'path';
+    readonly path: string;
+    readonly directoryHint?: true;
+} | {
+    readonly kind: 'external' | 'fragment' | 'malformed' | 'outside-repository';
+    readonly reason: string;
+};
+/** Resolve one explicit destination without consulting the checkout, guessing files or fetching URLs. */
+export declare function resolveCanonicalDocumentLink(sourcePath: string, target: string): CanonicalDocumentLinkResolution;

package/dist/extraction/canonical-document-links.js

@@ -0,0 +1,95 @@
+import { posix } from 'node:path';
+import { fromMarkdown } from 'mdast-util-from-markdown';
+/** Explicit Markdown links only: this deliberately does not change the graph linker's policy. */
+export function canonicalDocumentLinks(content) {
+    const tree = fromMarkdown(content);
+    const definitions = new Map();
+    const collect = (node) => {
+        if (node.type === 'definition') {
+            const existing = definitions.get(node.identifier) ?? [];
+            existing.push(node);
+            definitions.set(node.identifier, existing);
+        }
+        if ('children' in node)
+            for (const child of node.children)
+                collect(child);
+    };
+    collect(tree);
+    const links = [];
+    const visit = (node) => {
+        if (node.type === 'link' && node.position) {
+            links.push({ kind: 'link', target: node.url,
+                startLine: node.position.start.line, endLine: node.position.end.line });
+            return;
+        }
+        if (node.type === 'linkReference' && node.position) {
+            const declared = (definitions.get(node.identifier) ?? []).flatMap(definition => definition.position
+                ? [{ target: definition.url, startLine: definition.position.start.line, endLine: definition.position.end.line }]
+                : []);
+            const first = declared[0];
+            if (!first)
+                return;
+            const reference = { identifier: node.identifier,
+                referenceStartLine: node.position.start.line, referenceEndLine: node.position.end.line };
+            if (new Set(declared.map(definition => definition.target)).size > 1) {
+                // Never silently choose the first conflicting definition, as normal Markdown rendering does.
+                links.push({ kind: 'ambiguous-reference', ...reference,
+                    startLine: node.position.start.line, endLine: node.position.end.line, definitions: declared });
+            }
+            else {
+                links.push({ kind: 'link', ...first, ...reference });
+            }
+            return;
+        }
+        if (node.type === 'image' || node.type === 'imageReference' || node.type === 'code' ||
+            node.type === 'inlineCode' || node.type === 'html' || node.type === 'definition')
+            return;
+        if ('children' in node)
+            for (const child of node.children)
+                visit(child);
+    };
+    visit(tree);
+    return links;
+}
+const unsafeCharacters = /[\\\x00-\x1f\x7f-\x9f]/;
+const scheme = /^[a-z][a-z0-9+.-]*:/i;
+function isCanonicalDocumentSourcePath(path) {
+    return path.length > 0 && !unsafeCharacters.test(path) && !posix.isAbsolute(path) &&
+        path !== '.' && path !== '..' && !path.startsWith('../') && !path.endsWith('/') &&
+        posix.normalize(path) === path;
+}
+/** Resolve one explicit destination without consulting the checkout, guessing files or fetching URLs. */
+export function resolveCanonicalDocumentLink(sourcePath, target) {
+    if (!isCanonicalDocumentSourcePath(sourcePath)) {
+        return { kind: 'malformed', reason: 'Source must be a canonical repository-relative file path.' };
+    }
+    if (unsafeCharacters.test(target)) {
+        return { kind: 'malformed', reason: 'Backslashes and control characters are unsupported in link destinations.' };
+    }
+    if (target.startsWith('//') || scheme.test(target)) {
+        return { kind: 'external', reason: 'External destinations are reported without fetching them.' };
+    }
+    const rawPath = target.split(/[?#]/, 1)[0];
+    if (!rawPath)
+        return { kind: 'fragment', reason: 'Same-document links do not select another file.' };
+    let decoded;
+    try {
+        decoded = decodeURIComponent(rawPath);
+        // Plain relative ../ is supported. Encoded traversal segments are intentionally not followed.
+        if (rawPath.split('/').some(segment => decodeURIComponent(segment).split('/')
+            .some(part => (part === '.' || part === '..') && part !== segment))) {
+            return { kind: 'malformed', reason: 'Percent-encoded traversal segments are unsupported.' };
+        }
+    }
+    catch {
+        return { kind: 'malformed', reason: 'Link destination contains malformed percent encoding.' };
+    }
+    if (unsafeCharacters.test(decoded) || decoded.startsWith('//') || scheme.test(decoded)) {
+        return { kind: 'malformed', reason: 'Decoded destination is not a supported repository path.' };
+    }
+    const path = posix.normalize(decoded.startsWith('/') ? decoded.slice(1) : posix.join(posix.dirname(sourcePath), decoded));
+    if (path === '..' || path.startsWith('../') || posix.isAbsolute(path)) {
+        return { kind: 'outside-repository', reason: 'Link destination escapes the repository root.' };
+    }
+    return { kind: 'path', path: path.replace(/\/$/, ''), ...(decoded.endsWith('/') ? { directoryHint: true } : {}) };
+}

package/dist/extraction/document-classification.d.ts

@@ -0,0 +1,44 @@
+import { type SourceDocumentRole, type SourceOwnership } from '@weftlet/core';
+export interface DocumentClassificationPolicy {
+    readonly schemaVersion: 1;
+    readonly roles?: readonly {
+        readonly id: string;
+        readonly pattern: string;
+        readonly role: Exclude<SourceDocumentRole, 'unclassified'>;
+    }[];
+    readonly owners?: readonly {
+        readonly id: string;
+        readonly pattern: string;
+        readonly ownerDocument: string;
+    }[];
+}
+export interface DocumentClassificationContext {
+    readonly approvedIntentSources: readonly string[];
+    readonly evidenceRoots: readonly string[];
+    readonly availableDocuments: readonly string[];
+}
+export interface DocumentClassification {
+    readonly path: string;
+    readonly sourceRole: SourceDocumentRole;
+    readonly roleStatus: 'default' | 'explicit' | 'ambiguous';
+    readonly matchedRoleMappings: readonly string[];
+    readonly approvedBy: readonly string[];
+    readonly sourceAuthority: 'declared' | 'inferred';
+    readonly ownership: SourceOwnership;
+}
+export interface DocumentClassificationIssue {
+    readonly code: string;
+    readonly severity: 'warning' | 'error';
+    readonly message: string;
+    readonly paths: readonly string[];
+}
+export interface DocumentClassificationReport {
+    readonly policyDigest: string;
+    readonly documents: readonly DocumentClassification[];
+    readonly issues: readonly DocumentClassificationIssue[];
+    readonly complete: boolean;
+}
+/** Parse bounded explicit policy; normalize ordering, never infer approval or ownership. */
+export declare function parseDocumentClassificationPolicy(value: unknown): DocumentClassificationPolicy;
+/** Classify source-level evidence only; links, owner mappings and roles never approve a child. */
+export declare function classifyDocumentSources(paths: readonly string[], context: DocumentClassificationContext, policy?: DocumentClassificationPolicy): DocumentClassificationReport;