@weftlet/toolkit 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +139 -14
  2. package/dist/application/cache-adoption.d.ts +2 -1
  3. package/dist/application/cache-adoption.js +12 -5
  4. package/dist/application/document-coverage.d.ts +28 -0
  5. package/dist/application/document-coverage.js +58 -0
  6. package/dist/application/document-overlay.d.ts +14 -0
  7. package/dist/application/document-overlay.js +31 -0
  8. package/dist/application/extraction-identity.d.ts +2 -1
  9. package/dist/application/extraction-identity.js +7 -0
  10. package/dist/application/hydration-inputs.d.ts +15 -0
  11. package/dist/application/hydration-inputs.js +55 -0
  12. package/dist/application/hydration-plan.d.ts +7 -0
  13. package/dist/application/hydration-plan.js +18 -5
  14. package/dist/application/hydrator.js +26 -14
  15. package/dist/application/overlay.js +26 -8
  16. package/dist/application/saved-evidence.d.ts +2 -2
  17. package/dist/application/semantic-scope.d.ts +1 -1
  18. package/dist/application/semantic-scope.js +2 -2
  19. package/dist/configuration/cache.js +13 -1
  20. package/dist/configuration/config.d.ts +2 -0
  21. package/dist/configuration/config.js +5 -0
  22. package/dist/configuration/document-traversal.d.ts +7 -0
  23. package/dist/configuration/document-traversal.js +13 -0
  24. package/dist/extraction/canonical-document-links.d.ts +28 -0
  25. package/dist/extraction/canonical-document-links.js +95 -0
  26. package/dist/extraction/document-classification.d.ts +44 -0
  27. package/dist/extraction/document-classification.js +161 -0
  28. package/dist/extraction/document-evidence.d.ts +21 -0
  29. package/dist/extraction/document-evidence.js +83 -0
  30. package/dist/extraction/document-traversal.d.ts +76 -0
  31. package/dist/extraction/document-traversal.js +210 -0
  32. package/dist/extraction/document.d.ts +3 -1
  33. package/dist/extraction/document.js +3 -1
  34. package/dist/graph/authority.js +3 -0
  35. package/dist/graph/coverage.d.ts +8 -0
  36. package/dist/graph/coverage.js +96 -0
  37. package/dist/graph/document-role-description.d.ts +5 -0
  38. package/dist/graph/document-role-description.js +18 -0
  39. package/dist/graph/query.d.ts +2 -2
  40. package/dist/graph/query.js +22 -62
  41. package/dist/graph/rdf.js +11 -0
  42. package/dist/graph/shacl-shapes.js +26 -0
  43. package/dist/index.d.ts +5 -0
  44. package/dist/index.js +4 -0
  45. package/dist/interfaces/cli.js +18 -1
  46. package/dist/interfaces/document-coverage.d.ts +9 -0
  47. package/dist/interfaces/document-coverage.js +117 -0
  48. package/dist/repository/git.d.ts +11 -0
  49. package/dist/repository/git.js +38 -0
  50. package/dist/visualization/visualization.js +21 -6
  51. package/docs/ADR-017-shacl-validation-union.md +4 -0
  52. package/docs/ADR-021-identity-preflight-and-validation-timing.md +5 -2
  53. package/docs/ADR-022-release-documentation.md +44 -0
  54. package/docs/ADR-023-document-coverage-preview.md +64 -0
  55. package/docs/ADR-024-implementation-coverage.md +51 -0
  56. package/docs/ADR-025-document-classification.md +78 -0
  57. package/docs/ADR-026-traversal-hydration.md +94 -0
  58. package/docs/ADR-027-validation-closeout.md +155 -0
  59. package/docs/CONSUMER-TRIAL.md +1 -1
  60. package/docs/MIGRATING-TO-WEFTLET.md +1 -1
  61. package/docs/PROVIDERS.md +10 -1
  62. package/docs/RELEASE-1.0-PLAN.md +135 -23
  63. package/docs/RELEASING.md +46 -8
  64. package/docs/ROADMAP.md +41 -16
  65. package/docs/VALIDATION.md +312 -0
  66. package/docs/benchmarks/validation-2026-10-28/README.md +25 -0
  67. package/docs/benchmarks/validation-2026-10-28/large.json +1 -0
  68. package/docs/benchmarks/validation-2026-10-28/lifecycle.json +1 -0
  69. package/docs/benchmarks/validation-2026-10-28/native.json +1 -0
  70. package/docs/benchmarks/validation-2026-10-28/rich.json +1 -0
  71. package/docs/extraction-manifest.json +67 -17
  72. package/examples/generic-typescript/document-policy.json +9 -0
  73. package/package.json +7 -4
@@ -4,14 +4,14 @@ import { repositoryFileExtractorIdentity, repositoryContractExtractorIdentity, s
4
4
  import { sha256, stableJson } from "../infrastructure/util.js";
5
5
  import { documentLinkerIdentity, linkSemanticDocuments } from "../extraction/document.js";
6
6
  import { repositoryNodeIdentityVersion } from '../extraction/repository-nodes.js';
7
- import { selectSemanticInputs } from '../extraction/semantic-sources.js';
7
+ import { selectHydrationInputs, classifyHydratedFacts, documentSelectionIdentity } from './hydration-inputs.js';
8
8
  import { runtimeEvidenceFacts } from "../extraction/runtime-evidence.js";
9
9
  import { matchesRepositoryGlob } from "../repository/glob.js";
10
10
  import { RdfEvidenceGraph } from "../graph/rdf.js";
11
11
  import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
12
12
  import { SemanticProviderError } from '../extraction/semantic-process.js';
13
13
  import { hydrateSemanticSources } from './semantic-run.js';
14
- import { graphAssemblyVersion, graphAssemblyIdentity } from './extraction-identity.js';
14
+ import { graphAssemblyVersion, graphAssemblyIdentity, documentExtractionVersions } from './extraction-identity.js';
15
15
  import { assertGraphIdentities } from '../graph/identity-preflight.js';
16
16
  import { measureValidationPhase } from '../graph/validation-progress.js';
17
17
  export class ContextGraphHydrator {
@@ -51,10 +51,16 @@ export class ContextGraphHydrator {
51
51
  const commitSha = await repository.resolveRef(options.ref);
52
52
  if (options.requireClean === true)
53
53
  await repository.requireExactCleanCommit(commitSha);
54
- const selectedSemanticSources = options.semanticSources ?? config.semanticSources;
55
- const selection = await selectSemanticInputs(repository, commitSha, selectedSemanticSources, config.semantic);
56
- if (selection.issues.some((issue) => issue.severity === 'error')) {
57
- throw new Error(selection.issues.map((issue) => issue.message).join('\n'));
54
+ if (config.documentTraversal && options.semanticSources !== undefined) {
55
+ throw new Error('Traversal hydration does not accept a direct semanticSources override');
56
+ }
57
+ const selection = await selectHydrationInputs(repository, commitSha, config, options.semanticSources);
58
+ const selectedSemanticSources = selection.documentSelection?.semanticPaths ?? options.semanticSources ?? config.semanticSources;
59
+ const scopePaths = selection.documentSelection?.documentPaths ?? selectedSemanticSources;
60
+ if (selection.documentSelection?.complete === false || selection.issues.some((issue) => issue.severity === 'error')) {
61
+ throw new Error(config.documentTraversal
62
+ ? 'Traversal hydration blocked: run hydrate --plan --json to review coverage, roles and budgets before retrying.'
63
+ : selection.issues.map((issue) => issue.message).join('\n'));
58
64
  }
59
65
  const { sources } = selection;
60
66
  await this.dependencies.ontologyCompiler.checkReady?.();
@@ -64,7 +70,7 @@ export class ContextGraphHydrator {
64
70
  : (prior?.manifest.commitSha ?? (await repository.firstParent(commitSha)));
65
71
  const allRoots = [
66
72
  ...new Set([
67
- ...selectedSemanticSources,
73
+ ...scopePaths,
68
74
  ...config.codeRoots,
69
75
  ...config.evidenceRoots,
70
76
  ...contractPatternRoots(config.contractPatterns),
@@ -86,11 +92,12 @@ export class ContextGraphHydrator {
86
92
  ? `runtime-${runtimeRevision.digest.slice(0, 12)}`
87
93
  : undefined;
88
94
  const configurationDigest = sha256(stableJson(config));
89
- const scopeDigest = sha256(stableJson(selectedSemanticSources)).slice(0, 12);
95
+ const scopeDigest = sha256(stableJson(scopePaths)).slice(0, 12);
90
96
  const effectiveSuffix = [`graph-v${graphAssemblyVersion}`, `links-${documentLinkerIdentity.version}`, `files-${repositoryFileExtractorIdentity.version}`,
91
97
  `contracts-${repositoryContractExtractorIdentity.version}`,
92
98
  `symbols-${scipSymbolExtractorIdentity.version}`,
93
99
  ...(config.routingSources?.length ? [`routes-${routingExtractorIdentity.version}`] : []),
100
+ ...(selection.documentSelection ? [documentSelectionIdentity, selection.documentSelection.digest.slice(0, 16)] : []),
94
101
  configurationDigest.slice(0, 16), scopeDigest, baseSuffix, runtimeSuffix].filter(Boolean).join('-');
95
102
  const snapshotId = effectiveSuffix ? `${commitSha}-${effectiveSuffix}` : commitSha;
96
103
  const context = {
@@ -123,7 +130,7 @@ export class ContextGraphHydrator {
123
130
  commitSha,
124
131
  })),
125
132
  })));
126
- const semanticFacts = hydrated.flatMap(({ source, artifact }) => artifact.facts.map((fact) => ({
133
+ const semanticFacts = classifyHydratedFacts(hydrated.flatMap(({ source, artifact }) => artifact.facts.map((fact) => ({
127
134
  ...fact,
128
135
  id: sha256(stableJson({ artifact: artifact.key, fact: fact.id, path: source.path })),
129
136
  provenance: {
@@ -136,7 +143,7 @@ export class ContextGraphHydrator {
136
143
  authority: 'inferred',
137
144
  extractionReview: 'unreviewed',
138
145
  sourceAuthority: (config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern)) ? 'declared' : 'inferred'),
139
- })));
146
+ }))), selection.documentEvidence);
140
147
  checkpoint();
141
148
  options.onProgress?.({ phase: 'index-typescript' });
142
149
  const scip = await this.dependencies.staticIndexer.index(context);
@@ -155,13 +162,15 @@ export class ContextGraphHydrator {
155
162
  });
156
163
  }
157
164
  }
158
- const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path));
165
+ const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path), { canonicalReferences: !config.documentTraversal });
159
166
  diagnostics.push(...documents.diagnostics);
160
- diagnostics.push(...routing.diagnostics);
167
+ diagnostics.push(...routing.diagnostics, ...(selection.documentEvidence?.diagnostics ?? []));
161
168
  checkpoint();
162
169
  const runtimeEvidence = runtimeRevision.references;
163
170
  const runtimeFacts = runtimeEvidenceFacts(context, runtimeEvidence);
164
- const facts = [...semanticFacts, ...documents.facts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
171
+ const structuralFacts = selection.documentEvidence?.facts.map(fact => ({ ...fact, snapshotId })) ?? [];
172
+ const facts = [...semanticFacts, ...classifyHydratedFacts(documents.facts, selection.documentEvidence),
173
+ ...structuralFacts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
165
174
  const validationCounts = { factCount: facts.length };
166
175
  await measureValidationPhase('validate-identities', validationCounts, options.onProgress, () => {
167
176
  const report = assertGraphIdentities(facts, {
@@ -198,6 +207,7 @@ export class ContextGraphHydrator {
198
207
  unresolved,
199
208
  diagnostics,
200
209
  semanticSources: selectedSemanticSources,
210
+ ...(selection.documentSelection ? { documentSelection: selection.documentSelection } : {}),
201
211
  };
202
212
  const contentDigest = sha256(stableJson(logical));
203
213
  const snapshot = {
@@ -209,10 +219,12 @@ export class ContextGraphHydrator {
209
219
  baseSha,
210
220
  snapshotId,
211
221
  configurationDigest,
212
- scope: selectedSemanticSources,
222
+ scope: scopePaths,
223
+ ...(selection.documentSelection ? { documentSelection: selection.documentSelection } : {}),
213
224
  ontologyVersion: config.semantic.ontologyVersion,
214
225
  extractorVersions: {
215
226
  assembly: graphAssemblyIdentity,
227
+ ...documentExtractionVersions(config),
216
228
  [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
217
229
  [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
218
230
  'repository-node-identity': String(repositoryNodeIdentityVersion),
@@ -11,8 +11,11 @@ import { contextPackEvidenceAuthority } from "../graph/query.js";
11
11
  import { inWorkingScope, readWorkingContent } from "../repository/working-content.js";
12
12
  import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
13
13
  import { RdfEvidenceGraph } from '../graph/rdf.js';
14
- import { graphAssemblyIdentity } from './extraction-identity.js';
14
+ import { graphAssemblyIdentity, documentExtractionVersions } from './extraction-identity.js';
15
15
  import { assertGraphIdentities } from '../graph/identity-preflight.js';
16
+ import { selectHydrationInputs, documentSelectionIdentity } from './hydration-inputs.js';
17
+ import { retainOverlayDocumentFact, overlayDocumentRoots, overlayDocumentManifest, overlayDocumentNotes } from './document-overlay.js';
18
+ import { documentEvidenceExtractorIdentity } from '../extraction/document-evidence.js';
16
19
  const execute = promisify(execFile);
17
20
  const implementationExtractors = new Set([
18
21
  'repository-file-extractor', 'repository-contract-extractor', 'scip-symbol-extractor',
@@ -39,6 +42,7 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
39
42
  commitSha: head, snapshotId: baseSnapshotId, changedPaths: [], deletedPaths: [], renamedPaths: {},
40
43
  }, routingSources, { allowMissingSources: true });
41
44
  const roots = [...config.codeRoots, ...config.semanticSources, ...config.evidenceRoots,
45
+ ...overlayDocumentRoots(config),
42
46
  ...config.contractPatterns, ...scipSourcePatterns(config.scip.sourceProject),
43
47
  ...routingSources, ...baseRouting.referencedPaths];
44
48
  const excluded = [];
@@ -64,6 +68,8 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
64
68
  repositoryContractExtractor: repositoryContractExtractorIdentity,
65
69
  scipSymbolExtractor: scipSymbolExtractorIdentity,
66
70
  ...(routingSources.length ? { routingExtractor: routingExtractorIdentity } : {}),
71
+ ...(config.documentTraversal ? { documentSelection: documentSelectionIdentity,
72
+ documentEvidence: documentEvidenceExtractorIdentity } : {}),
67
73
  })).slice(0, 32)}`;
68
74
  const affected = new Set(captured.map((file) => file.path));
69
75
  const extracted = await withCommittedTree(repository.root, head, async (root) => {
@@ -90,6 +96,13 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
90
96
  deletedPaths: captured.filter((f) => f.content === null).map((f) => f.path),
91
97
  renamedPaths: {},
92
98
  };
99
+ const documents = config.documentTraversal
100
+ ? await selectHydrationInputs(draftRepository, context.commitSha, config) : undefined;
101
+ const documentSelection = documents?.documentSelection ? {
102
+ ...documents.documentSelection,
103
+ // Temporary Git commit timestamps are not evidence identity. Bind drafts to captured bytes.
104
+ digest: sha256(stableJson({ ...documents.documentSelection, digest: undefined, baseSha: head, overlayDigest })),
105
+ } : undefined;
93
106
  const scip = await new ScipTypeScriptIndexer(config.scip, storageRoot).index(context);
94
107
  const routing = await extractRoutingDeclarations(draftRepository, context, routingSources, { allowMissingSources: true });
95
108
  const uncapturedEndpoints = routing.referencedPaths.filter((path) => !inWorkingScope(path, roots));
@@ -102,19 +115,20 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
102
115
  new ScipSymbolExtractor(draftRepository, scip),
103
116
  ];
104
117
  const facts = [...(await Promise.all(extractors.map((e) => e.extract(context)))).flat(),
105
- ...routing.facts].map(({ id: _id, ...fact }) => {
118
+ ...routing.facts, ...(documents?.documentEvidence?.facts ?? [])].map(({ id: _id, ...fact }) => {
106
119
  const draft = { ...fact, snapshotId, authority: 'draft',
107
- ...(fact.extractor.id === routingExtractorIdentity.id ? { sourceAuthority: 'draft' } : {}),
120
+ ...(fact.extractor.id === routingExtractorIdentity.id || fact.sourceRole !== undefined ? { sourceAuthority: 'draft' } : {}),
108
121
  provenance: { ...fact.provenance, commitSha: head } };
109
122
  return { ...draft, id: sha256(stableJson(draft)) };
110
123
  });
111
- const diagnostics = routing.diagnostics.map(({ id: _id, ...item }) => {
124
+ const diagnostics = [...routing.diagnostics, ...(documents?.documentEvidence?.diagnostics ?? [])].map(({ id: _id, ...item }) => {
112
125
  const draft = { ...item, provenance: item.provenance.map((span) => ({ ...span, commitSha: head })) };
113
126
  return { ...draft, id: sha256(stableJson(draft)) };
114
127
  }).sort((a, b) => a.id.localeCompare(b.id));
115
- return { facts, diagnostics };
128
+ return { facts, diagnostics, documents, documentSelection };
116
129
  });
117
- const retained = base.facts.filter((fact) => !implementationExtractors.has(fact.extractor.id) &&
130
+ const retained = base.facts.filter((fact) => retainOverlayDocumentFact(fact, extracted.documents, affected) &&
131
+ !implementationExtractors.has(fact.extractor.id) &&
118
132
  contextPackEvidenceAuthority(fact) !== 'implemented' &&
119
133
  fact.extractor.id !== 'otlp-runtime-evidence' &&
120
134
  !affected.has(fact.provenance.path) &&
@@ -125,14 +139,17 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
125
139
  ...base,
126
140
  manifest: {
127
141
  ...base.manifest, snapshotId, baseSha: head, overlayDigest, overlayCapture,
142
+ ...overlayDocumentManifest(extracted.documentSelection),
128
143
  extractorVersions: { ...base.manifest.extractorVersions,
129
144
  assembly: graphAssemblyIdentity,
145
+ ...documentExtractionVersions(config),
130
146
  [repositoryFileExtractorIdentity.id]: repositoryFileExtractorIdentity.version,
131
147
  [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
132
148
  [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
133
149
  ...(routingSources.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}) },
134
- generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length
135
- ? { facts: allFacts, diagnostics: extracted.diagnostics } : allFacts)),
150
+ generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length || config.documentTraversal
151
+ ? { facts: allFacts, diagnostics: extracted.diagnostics,
152
+ ...(extracted.documentSelection ? { documentSelection: extracted.documentSelection } : {}) } : allFacts)),
136
153
  },
137
154
  facts: allFacts,
138
155
  invariants: base.invariants.map((rule) => ({
@@ -141,6 +158,7 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
141
158
  runtimeEvidence: [],
142
159
  unresolved: [
143
160
  'Draft implementation overlay: changed semantic documents require explicit re-extraction.',
161
+ ...overlayDocumentNotes(config),
144
162
  `Changes outside configured source scope are excluded: ${excluded.join(', ') || '(none)'}`,
145
163
  ],
146
164
  diagnostics: extracted.diagnostics,
@@ -15,13 +15,13 @@ export declare class SavedEvidenceQueryService implements ImpactQueryService {
15
15
  incompatibleExtractors?: readonly string[] | undefined;
16
16
  stale: boolean;
17
17
  configurationMatches: boolean;
18
- overlayFreshness?: "current" | "stale" | "unknown" | undefined;
18
+ overlayFreshness?: "unknown" | "current" | "stale" | undefined;
19
19
  capturedWorkingChanges?: string[] | undefined;
20
20
  changedSinceCapture?: string[] | undefined;
21
21
  currentCommit: string | undefined;
22
22
  excludedWorkingChanges: string[];
23
23
  };
24
- snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture"> & Partial<Pick<import("@weftlet/core").SnapshotManifest, "extractorVersions">>;
24
+ snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture"> & Partial<Pick<import("@weftlet/core").SnapshotManifest, "extractorVersions" | "documentSelection">>;
25
25
  capabilities: readonly import("@weftlet/core").GraphNode[];
26
26
  facts: readonly import("@weftlet/core").GraphFact[];
27
27
  invariants: readonly import("@weftlet/core").InvariantResult[];
@@ -2,7 +2,7 @@ import type { ImpactScanRequest } from '@weftlet/core';
2
2
  import type { ContextGraphConfig } from '../configuration/config.js';
3
3
  import type { GitRepository } from '../repository/git.js';
4
4
  export interface SemanticScope {
5
- readonly mode: 'configured' | 'agent';
5
+ readonly mode: 'configured' | 'agent' | 'traversal';
6
6
  readonly selectors: readonly string[];
7
7
  readonly digest: string;
8
8
  readonly headName: string;
@@ -5,8 +5,8 @@ export async function resolveSemanticScope(config, repository, commitSha, reques
5
5
  const agent = config.agentContext
6
6
  ? await new AgentSemanticScopeResolver(config, repository).resolve(request, commitSha)
7
7
  : undefined;
8
- const selectors = agent?.semanticSources ?? config.semanticSources;
9
- return { mode: agent ? 'agent' : 'configured', selectors, digest: sha256(stableJson(selectors)),
8
+ const selectors = config.documentTraversal?.entryPoints ?? agent?.semanticSources ?? config.semanticSources;
9
+ return { mode: config.documentTraversal ? 'traversal' : agent ? 'agent' : 'configured', selectors, digest: sha256(stableJson(selectors)),
10
10
  headName: agent?.headName ?? 'default', omittedDocuments: agent?.omittedDocuments ?? [],
11
11
  ignoredSearchTerms: agent?.ignoredSearchTerms ?? 0 };
12
12
  }
@@ -2,6 +2,7 @@
2
2
  import { homedir } from 'node:os';
3
3
  import { resolve } from 'node:path';
4
4
  import { sha256, stableJson } from "../infrastructure/util.js";
5
+ import { assertDocumentTraversalConfig } from './document-traversal.js';
5
6
  export function resolveStorageRoot(repositoryRoot, config, userCacheRoot = resolve(homedir(), '.cache', 'context-graph')) {
6
7
  if (config.storageScope !== 'user')
7
8
  return resolve(repositoryRoot, config.storagePath);
@@ -14,14 +15,25 @@ const codexOperationalFields = new Set([
14
15
  'timeoutMs', 'retries', 'concurrency', 'maxDocuments', 'maxSourceBytes',
15
16
  ]);
16
17
  export function semanticCachePartition(repositoryRoot, config) {
18
+ if (config.documentTraversal)
19
+ assertDocumentTraversalConfig(config.documentTraversal);
17
20
  const semantic = config.semantic.provider === 'codex-cli'
18
21
  ? Object.fromEntries(Object.entries(config.semantic).filter(([field]) => !codexOperationalFields.has(field)))
19
22
  : config.semantic;
20
- return sha256(stableJson({ config: { ...config, semantic }, checkout: resolve(repositoryRoot) }));
23
+ // Traversal/role metadata is assembled after the built-in Codex artifact is validated.
24
+ // Keep the old partition exactly when enabling it; do not ignore unknown configuration.
25
+ const { documentTraversal: _traversal, ...directConfig } = config;
26
+ const inputs = config.semantic.provider === 'codex-cli' ? directConfig : config;
27
+ return sha256(stableJson({ config: { ...inputs, semantic }, checkout: resolve(repositoryRoot) }));
21
28
  }
22
29
  /** Snapshot profiles retain the full configuration. Only Codex user artifacts share operational changes. */
23
30
  export function resolveSemanticStorageRoots(repositoryRoot, config, userCacheRoot = resolve(homedir(), '.cache', 'context-graph')) {
24
31
  const legacy = resolveStorageRoot(repositoryRoot, config, userCacheRoot);
32
+ // Module providers can inspect arbitrary extraction context. Opt-in traversal isolates their
33
+ // artifacts even in workspace storage, while unchanged committed sources still reuse them.
34
+ if (config.documentTraversal && config.semantic.provider === 'module') {
35
+ return { primary: resolve(legacy, 'document-semantic-v1', semanticCachePartition(repositoryRoot, config)) };
36
+ }
25
37
  if (config.storageScope !== 'user' || config.semantic.provider !== 'codex-cli')
26
38
  return { primary: legacy };
27
39
  return { primary: resolve(userCacheRoot, 'semantic-artifacts-v1', semanticCachePartition(repositoryRoot, config)), legacy };
@@ -1,4 +1,5 @@
1
1
  import { type ScipSourceProject } from './scip-project.js';
2
+ import { type DocumentTraversalConfig } from './document-traversal.js';
2
3
  export interface ContextGraphConfig {
3
4
  readonly schemaVersion: 1;
4
5
  readonly repositoryRoot?: string;
@@ -6,6 +7,7 @@ export interface ContextGraphConfig {
6
7
  readonly namespace: string;
7
8
  readonly adapter?: string;
8
9
  readonly semanticSources: readonly string[];
10
+ readonly documentTraversal?: DocumentTraversalConfig;
9
11
  /** Explicit owner-approved source paths/globs; other model-derived prose remains inferred. */
10
12
  readonly approvedIntentSources?: readonly string[];
11
13
  readonly codeRoots: readonly string[];
@@ -3,6 +3,7 @@ import { existsSync } from 'node:fs';
3
3
  import { readFile } from 'node:fs/promises';
4
4
  import { dirname, resolve } from 'node:path';
5
5
  import { assertScipSourceProject, assertSourceProjectInvocation } from './scip-project.js';
6
+ import { assertDocumentTraversalConfig } from './document-traversal.js';
6
7
  function isRecord(value) {
7
8
  return typeof value === 'object' && value !== null && !Array.isArray(value);
8
9
  }
@@ -34,6 +35,10 @@ function validateConfig(value, absolutePath) {
34
35
  if (value.approvedIntentSources !== undefined) {
35
36
  requireConfig(isStringArray(value.approvedIntentSources), 'approvedIntentSources must be an array of strings', absolutePath);
36
37
  }
38
+ if (value.documentTraversal !== undefined) {
39
+ assertDocumentTraversalConfig(value.documentTraversal);
40
+ requireConfig(value.agentContext === undefined, 'documentTraversal and bounded agentContext selection cannot be combined', absolutePath);
41
+ }
37
42
  if (value.routingSources !== undefined) {
38
43
  requireConfig(isStringArray(value.routingSources) && value.routingSources.length <= 16 &&
39
44
  new Set(value.routingSources).size === value.routingSources.length &&
@@ -0,0 +1,7 @@
1
+ import { type DocumentTraversalOptions } from '../extraction/document-traversal.js';
2
+ import { type DocumentClassificationPolicy } from '../extraction/document-classification.js';
3
+ /** Explicit execution mode; the direct semanticSources list is not added to its closure. */
4
+ export interface DocumentTraversalConfig extends Omit<DocumentTraversalOptions, 'maxDocuments' | 'maxSourceBytes'> {
5
+ readonly policy?: DocumentClassificationPolicy;
6
+ }
7
+ export declare function assertDocumentTraversalConfig(value: unknown): asserts value is DocumentTraversalConfig;
@@ -0,0 +1,13 @@
1
+ import { validateDocumentTraversalOptions } from '../extraction/document-traversal.js';
2
+ import { parseDocumentClassificationPolicy } from '../extraction/document-classification.js';
3
+ export function assertDocumentTraversalConfig(value) {
4
+ const fields = ['entryPoints', 'roots', 'maxReadDocuments', 'maxReadBytes', 'maxDocumentBytes', 'policy'];
5
+ if (!value || typeof value !== 'object' || Array.isArray(value) ||
6
+ Object.keys(value).some(key => !fields.includes(key))) {
7
+ throw new Error('documentTraversal must contain only supported traversal options and policy');
8
+ }
9
+ const options = value;
10
+ validateDocumentTraversalOptions(options);
11
+ if (options.policy !== undefined)
12
+ parseDocumentClassificationPolicy(options.policy);
13
+ }
@@ -0,0 +1,28 @@
1
+ export interface CanonicalDocumentDefinition {
2
+ readonly target: string;
3
+ readonly startLine: number;
4
+ readonly endLine: number;
5
+ }
6
+ export interface CanonicalDocumentLink {
7
+ readonly kind: 'link' | 'ambiguous-reference';
8
+ readonly target?: string;
9
+ readonly startLine: number;
10
+ readonly endLine: number;
11
+ readonly identifier?: string;
12
+ readonly definitions?: readonly CanonicalDocumentDefinition[];
13
+ /** Reference uses and their destination declarations can occur on different lines. */
14
+ readonly referenceStartLine?: number;
15
+ readonly referenceEndLine?: number;
16
+ }
17
+ /** Explicit Markdown links only: this deliberately does not change the graph linker's policy. */
18
+ export declare function canonicalDocumentLinks(content: string): readonly CanonicalDocumentLink[];
19
+ export type CanonicalDocumentLinkResolution = {
20
+ readonly kind: 'path';
21
+ readonly path: string;
22
+ readonly directoryHint?: true;
23
+ } | {
24
+ readonly kind: 'external' | 'fragment' | 'malformed' | 'outside-repository';
25
+ readonly reason: string;
26
+ };
27
+ /** Resolve one explicit destination without consulting the checkout, guessing files or fetching URLs. */
28
+ export declare function resolveCanonicalDocumentLink(sourcePath: string, target: string): CanonicalDocumentLinkResolution;
@@ -0,0 +1,95 @@
1
+ import { posix } from 'node:path';
2
+ import { fromMarkdown } from 'mdast-util-from-markdown';
3
+ /** Explicit Markdown links only: this deliberately does not change the graph linker's policy. */
4
+ export function canonicalDocumentLinks(content) {
5
+ const tree = fromMarkdown(content);
6
+ const definitions = new Map();
7
+ const collect = (node) => {
8
+ if (node.type === 'definition') {
9
+ const existing = definitions.get(node.identifier) ?? [];
10
+ existing.push(node);
11
+ definitions.set(node.identifier, existing);
12
+ }
13
+ if ('children' in node)
14
+ for (const child of node.children)
15
+ collect(child);
16
+ };
17
+ collect(tree);
18
+ const links = [];
19
+ const visit = (node) => {
20
+ if (node.type === 'link' && node.position) {
21
+ links.push({ kind: 'link', target: node.url,
22
+ startLine: node.position.start.line, endLine: node.position.end.line });
23
+ return;
24
+ }
25
+ if (node.type === 'linkReference' && node.position) {
26
+ const declared = (definitions.get(node.identifier) ?? []).flatMap(definition => definition.position
27
+ ? [{ target: definition.url, startLine: definition.position.start.line, endLine: definition.position.end.line }]
28
+ : []);
29
+ const first = declared[0];
30
+ if (!first)
31
+ return;
32
+ const reference = { identifier: node.identifier,
33
+ referenceStartLine: node.position.start.line, referenceEndLine: node.position.end.line };
34
+ if (new Set(declared.map(definition => definition.target)).size > 1) {
35
+ // Never silently choose the first conflicting definition, as normal Markdown rendering does.
36
+ links.push({ kind: 'ambiguous-reference', ...reference,
37
+ startLine: node.position.start.line, endLine: node.position.end.line, definitions: declared });
38
+ }
39
+ else {
40
+ links.push({ kind: 'link', ...first, ...reference });
41
+ }
42
+ return;
43
+ }
44
+ if (node.type === 'image' || node.type === 'imageReference' || node.type === 'code' ||
45
+ node.type === 'inlineCode' || node.type === 'html' || node.type === 'definition')
46
+ return;
47
+ if ('children' in node)
48
+ for (const child of node.children)
49
+ visit(child);
50
+ };
51
+ visit(tree);
52
+ return links;
53
+ }
54
+ const unsafeCharacters = /[\\\x00-\x1f\x7f-\x9f]/;
55
+ const scheme = /^[a-z][a-z0-9+.-]*:/i;
56
+ function isCanonicalDocumentSourcePath(path) {
57
+ return path.length > 0 && !unsafeCharacters.test(path) && !posix.isAbsolute(path) &&
58
+ path !== '.' && path !== '..' && !path.startsWith('../') && !path.endsWith('/') &&
59
+ posix.normalize(path) === path;
60
+ }
61
+ /** Resolve one explicit destination without consulting the checkout, guessing files or fetching URLs. */
62
+ export function resolveCanonicalDocumentLink(sourcePath, target) {
63
+ if (!isCanonicalDocumentSourcePath(sourcePath)) {
64
+ return { kind: 'malformed', reason: 'Source must be a canonical repository-relative file path.' };
65
+ }
66
+ if (unsafeCharacters.test(target)) {
67
+ return { kind: 'malformed', reason: 'Backslashes and control characters are unsupported in link destinations.' };
68
+ }
69
+ if (target.startsWith('//') || scheme.test(target)) {
70
+ return { kind: 'external', reason: 'External destinations are reported without fetching them.' };
71
+ }
72
+ const rawPath = target.split(/[?#]/, 1)[0];
73
+ if (!rawPath)
74
+ return { kind: 'fragment', reason: 'Same-document links do not select another file.' };
75
+ let decoded;
76
+ try {
77
+ decoded = decodeURIComponent(rawPath);
78
+ // Plain relative ../ is supported. Encoded traversal segments are intentionally not followed.
79
+ if (rawPath.split('/').some(segment => decodeURIComponent(segment).split('/')
80
+ .some(part => (part === '.' || part === '..') && part !== segment))) {
81
+ return { kind: 'malformed', reason: 'Percent-encoded traversal segments are unsupported.' };
82
+ }
83
+ }
84
+ catch {
85
+ return { kind: 'malformed', reason: 'Link destination contains malformed percent encoding.' };
86
+ }
87
+ if (unsafeCharacters.test(decoded) || decoded.startsWith('//') || scheme.test(decoded)) {
88
+ return { kind: 'malformed', reason: 'Decoded destination is not a supported repository path.' };
89
+ }
90
+ const path = posix.normalize(decoded.startsWith('/') ? decoded.slice(1) : posix.join(posix.dirname(sourcePath), decoded));
91
+ if (path === '..' || path.startsWith('../') || posix.isAbsolute(path)) {
92
+ return { kind: 'outside-repository', reason: 'Link destination escapes the repository root.' };
93
+ }
94
+ return { kind: 'path', path: path.replace(/\/$/, ''), ...(decoded.endsWith('/') ? { directoryHint: true } : {}) };
95
+ }
@@ -0,0 +1,44 @@
1
+ import { type SourceDocumentRole, type SourceOwnership } from '@weftlet/core';
2
+ export interface DocumentClassificationPolicy {
3
+ readonly schemaVersion: 1;
4
+ readonly roles?: readonly {
5
+ readonly id: string;
6
+ readonly pattern: string;
7
+ readonly role: Exclude<SourceDocumentRole, 'unclassified'>;
8
+ }[];
9
+ readonly owners?: readonly {
10
+ readonly id: string;
11
+ readonly pattern: string;
12
+ readonly ownerDocument: string;
13
+ }[];
14
+ }
15
+ export interface DocumentClassificationContext {
16
+ readonly approvedIntentSources: readonly string[];
17
+ readonly evidenceRoots: readonly string[];
18
+ readonly availableDocuments: readonly string[];
19
+ }
20
+ export interface DocumentClassification {
21
+ readonly path: string;
22
+ readonly sourceRole: SourceDocumentRole;
23
+ readonly roleStatus: 'default' | 'explicit' | 'ambiguous';
24
+ readonly matchedRoleMappings: readonly string[];
25
+ readonly approvedBy: readonly string[];
26
+ readonly sourceAuthority: 'declared' | 'inferred';
27
+ readonly ownership: SourceOwnership;
28
+ }
29
+ export interface DocumentClassificationIssue {
30
+ readonly code: string;
31
+ readonly severity: 'warning' | 'error';
32
+ readonly message: string;
33
+ readonly paths: readonly string[];
34
+ }
35
+ export interface DocumentClassificationReport {
36
+ readonly policyDigest: string;
37
+ readonly documents: readonly DocumentClassification[];
38
+ readonly issues: readonly DocumentClassificationIssue[];
39
+ readonly complete: boolean;
40
+ }
41
+ /** Parse bounded explicit policy; normalize ordering, never infer approval or ownership. */
42
+ export declare function parseDocumentClassificationPolicy(value: unknown): DocumentClassificationPolicy;
43
+ /** Classify source-level evidence only; links, owner mappings and roles never approve a child. */
44
+ export declare function classifyDocumentSources(paths: readonly string[], context: DocumentClassificationContext, policy?: DocumentClassificationPolicy): DocumentClassificationReport;