@weftlet/toolkit 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +26 -1
  2. package/dist/application/application.js +7 -4
  3. package/dist/application/cache-adoption.d.ts +43 -0
  4. package/dist/application/cache-adoption.js +129 -0
  5. package/dist/application/extraction-identity.d.ts +6 -0
  6. package/dist/application/extraction-identity.js +15 -0
  7. package/dist/application/hydration-failure.d.ts +41 -0
  8. package/dist/application/hydration-failure.js +43 -0
  9. package/dist/application/hydration-plan.d.ts +4 -7
  10. package/dist/application/hydration-plan.js +7 -15
  11. package/dist/application/hydrator.d.ts +5 -5
  12. package/dist/application/hydrator.js +37 -52
  13. package/dist/application/overlay.d.ts +3 -1
  14. package/dist/application/overlay.js +13 -4
  15. package/dist/application/saved-evidence.d.ts +6 -3
  16. package/dist/application/saved-evidence.js +7 -1
  17. package/dist/application/semantic-artifact-store.d.ts +4 -0
  18. package/dist/application/semantic-artifact-store.js +7 -0
  19. package/dist/application/semantic-run.d.ts +32 -0
  20. package/dist/application/semantic-run.js +111 -0
  21. package/dist/application/semantic-scope.d.ts +13 -0
  22. package/dist/application/semantic-scope.js +12 -0
  23. package/dist/application/snapshot-checks.d.ts +2 -1
  24. package/dist/application/snapshot-checks.js +8 -1
  25. package/dist/configuration/cache.d.ts +6 -0
  26. package/dist/configuration/cache.js +18 -0
  27. package/dist/configuration/config.js +1 -1
  28. package/dist/extraction/document.js +1 -1
  29. package/dist/extraction/providers.js +10 -3
  30. package/dist/extraction/repository-nodes.d.ts +2 -3
  31. package/dist/extraction/repository-nodes.js +5 -11
  32. package/dist/extraction/routing.d.ts +1 -1
  33. package/dist/extraction/routing.js +3 -6
  34. package/dist/extraction/semantic-cache.d.ts +1 -0
  35. package/dist/extraction/semantic-cache.js +9 -2
  36. package/dist/extraction/semantic-diagnostics.d.ts +9 -0
  37. package/dist/extraction/semantic-diagnostics.js +15 -0
  38. package/dist/extraction/semantic.d.ts +5 -1
  39. package/dist/extraction/semantic.js +39 -6
  40. package/dist/extraction/static.d.ts +13 -5
  41. package/dist/extraction/static.js +8 -15
  42. package/dist/governance/governance.js +11 -7
  43. package/dist/graph/identity-preflight.d.ts +57 -0
  44. package/dist/graph/identity-preflight.js +71 -0
  45. package/dist/graph/query.js +2 -0
  46. package/dist/graph/rdf.d.ts +4 -1
  47. package/dist/graph/rdf.js +10 -6
  48. package/dist/graph/validation-progress.d.ts +14 -0
  49. package/dist/graph/validation-progress.js +18 -0
  50. package/dist/index.d.ts +5 -0
  51. package/dist/index.js +5 -0
  52. package/dist/interfaces/cli.js +52 -22
  53. package/dist/storage/storage.d.ts +7 -1
  54. package/dist/storage/storage.js +15 -3
  55. package/dist/visualization/visualization.js +1 -0
  56. package/docs/ADR-016-hydration-preview.md +1 -1
  57. package/docs/ADR-017-shacl-validation-union.md +1 -1
  58. package/docs/ADR-018-operational-semantic-cache.md +100 -0
  59. package/docs/ADR-019-semantic-failure-diagnostics.md +82 -0
  60. package/docs/ADR-020-canonical-physical-file-producers.md +97 -0
  61. package/docs/ADR-021-identity-preflight-and-validation-timing.md +81 -0
  62. package/docs/PROVIDERS.md +58 -1
  63. package/docs/RELEASE-1.0-PLAN.md +100 -34
  64. package/docs/RELEASING.md +4 -2
  65. package/docs/ROADMAP.md +40 -15
  66. package/docs/VALIDATION.md +351 -4
  67. package/docs/extraction-manifest.json +50 -23
  68. package/package.json +2 -2
@@ -1,6 +1,6 @@
1
1
  import { scipSourcePatterns } from '../configuration/scip-project.js';
2
2
  import { contractPatternRoots } from "../repository/glob.js";
3
- import { repositoryFileExtractorIdentity } from '../extraction/static.js';
3
+ import { repositoryFileExtractorIdentity, repositoryContractExtractorIdentity, scipSymbolExtractorIdentity } from '../extraction/static.js';
4
4
  import { sha256, stableJson } from "../infrastructure/util.js";
5
5
  import { documentLinkerIdentity, linkSemanticDocuments } from "../extraction/document.js";
6
6
  import { repositoryNodeIdentityVersion } from '../extraction/repository-nodes.js';
@@ -9,27 +9,11 @@ import { runtimeEvidenceFacts } from "../extraction/runtime-evidence.js";
9
9
  import { matchesRepositoryGlob } from "../repository/glob.js";
10
10
  import { RdfEvidenceGraph } from "../graph/rdf.js";
11
11
  import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
12
- async function concurrentMap(values, concurrency, worker) {
13
- const results = new Array(values.length);
14
- let cursor = 0;
15
- let failure;
16
- let failed = false;
17
- await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, values.length)) }, async () => {
18
- while (!failed && cursor < values.length) {
19
- const index = cursor++;
20
- try {
21
- results[index] = await worker(values[index], index);
22
- }
23
- catch (error) {
24
- failed = true;
25
- failure ??= error;
26
- }
27
- }
28
- }));
29
- if (failed)
30
- throw failure;
31
- return results;
32
- }
12
+ import { SemanticProviderError } from '../extraction/semantic-process.js';
13
+ import { hydrateSemanticSources } from './semantic-run.js';
14
+ import { graphAssemblyVersion, graphAssemblyIdentity } from './extraction-identity.js';
15
+ import { assertGraphIdentities } from '../graph/identity-preflight.js';
16
+ import { measureValidationPhase } from '../graph/validation-progress.js';
33
17
  export class ContextGraphHydrator {
34
18
  dependencies;
35
19
  constructor(dependencies) {
@@ -57,7 +41,11 @@ export class ContextGraphHydrator {
57
41
  };
58
42
  }
59
43
  async hydrate(options) {
60
- options.signal?.throwIfAborted();
44
+ const checkpoint = () => {
45
+ if (options.signal?.aborted)
46
+ throw new SemanticProviderError('cancelled', 'Hydration cancelled');
47
+ };
48
+ checkpoint();
61
49
  options.onProgress?.({ phase: 'resolve-committed-sources' });
62
50
  const { config, repository, snapshotStore } = this.dependencies;
63
51
  const commitSha = await repository.resolveRef(options.ref);
@@ -99,7 +87,9 @@ export class ContextGraphHydrator {
99
87
  : undefined;
100
88
  const configurationDigest = sha256(stableJson(config));
101
89
  const scopeDigest = sha256(stableJson(selectedSemanticSources)).slice(0, 12);
102
- const effectiveSuffix = ['graph-v6', `links-${documentLinkerIdentity.version}`, `files-${repositoryFileExtractorIdentity.version}`,
90
+ const effectiveSuffix = [`graph-v${graphAssemblyVersion}`, `links-${documentLinkerIdentity.version}`, `files-${repositoryFileExtractorIdentity.version}`,
91
+ `contracts-${repositoryContractExtractorIdentity.version}`,
92
+ `symbols-${scipSymbolExtractorIdentity.version}`,
103
93
  ...(config.routingSources?.length ? [`routes-${routingExtractorIdentity.version}`] : []),
104
94
  configurationDigest.slice(0, 16), scopeDigest, baseSuffix, runtimeSuffix].filter(Boolean).join('-');
105
95
  const snapshotId = effectiveSuffix ? `${commitSha}-${effectiveSuffix}` : commitSha;
@@ -117,26 +107,10 @@ export class ContextGraphHydrator {
117
107
  };
118
108
  // Validate committed routing inputs before invoking any provider or indexer.
119
109
  const routing = await extractRoutingDeclarations(repository, context, config.routingSources ?? []);
120
- let semanticExtracted = 0;
121
- let semanticReused = 0;
122
- const controller = new AbortController();
123
- const signal = AbortSignal.any([controller.signal, ...(options.signal ? [options.signal] : [])]);
124
- const hydrated = await concurrentMap(sources, config.semantic.concurrency, async (source) => {
125
- let result;
126
- try {
127
- result = await this.dependencies.semanticHydrator.hydrate(source, context, forceRevision, { signal });
128
- }
129
- catch (error) {
130
- controller.abort();
131
- throw error;
132
- }
133
- if (result.reused)
134
- semanticReused += 1;
135
- else
136
- semanticExtracted += 1;
137
- options.onProgress?.({ phase: result.reused ? 'semantic-cache-hit' : 'semantic-extracted',
138
- completed: semanticReused + semanticExtracted, total: sources.length });
139
- return { source, ...result };
110
+ const { hydrated, semanticExtracted, semanticReused } = await hydrateSemanticSources({
111
+ sources, context, config: config.semantic, configurationDigest,
112
+ hydrator: this.dependencies.semanticHydrator, forceRevision,
113
+ signal: options.signal, onProgress: options.onProgress,
140
114
  });
141
115
  const artifacts = hydrated.map((result) => result.artifact);
142
116
  const diagnostics = hydrated.flatMap(({ source, artifact }) => (artifact.diagnostics ?? []).map((item) => ({
@@ -163,7 +137,7 @@ export class ContextGraphHydrator {
163
137
  extractionReview: 'unreviewed',
164
138
  sourceAuthority: (config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern)) ? 'declared' : 'inferred'),
165
139
  })));
166
- options.signal?.throwIfAborted();
140
+ checkpoint();
167
141
  options.onProgress?.({ phase: 'index-typescript' });
168
142
  const scip = await this.dependencies.staticIndexer.index(context);
169
143
  const extractorFacts = (await Promise.all(this.dependencies.extractorFactory(scip).map((extractor) => extractor.extract(context)))).flat();
@@ -184,12 +158,20 @@ export class ContextGraphHydrator {
184
158
  const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path));
185
159
  diagnostics.push(...documents.diagnostics);
186
160
  diagnostics.push(...routing.diagnostics);
187
- options.signal?.throwIfAborted();
188
- options.onProgress?.({ phase: 'compile-semantic-rdf' });
189
- const ontology = await this.dependencies.ontologyCompiler.compile(snapshotId, artifacts, context);
161
+ checkpoint();
190
162
  const runtimeEvidence = runtimeRevision.references;
191
163
  const runtimeFacts = runtimeEvidenceFacts(context, runtimeEvidence);
192
164
  const facts = [...semanticFacts, ...documents.facts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
165
+ const validationCounts = { factCount: facts.length };
166
+ await measureValidationPhase('validate-identities', validationCounts, options.onProgress, () => {
167
+ const report = assertGraphIdentities(facts, {
168
+ repositoryId: config.repositoryId, includeDetails: options.validationDetails,
169
+ });
170
+ validationCounts.nodeCount = report.nodeCount;
171
+ });
172
+ checkpoint();
173
+ options.onProgress?.({ phase: 'compile-semantic-rdf' });
174
+ const ontology = await this.dependencies.ontologyCompiler.compile(snapshotId, artifacts, context);
193
175
  const invariants = (await Promise.all(this.dependencies.invariantEvaluators.map((evaluator) => evaluator.evaluate(snapshotId, facts)))).flat();
194
176
  const failures = invariants.filter((invariant) => invariant.status === 'fail');
195
177
  if (failures.length > 0) {
@@ -230,7 +212,9 @@ export class ContextGraphHydrator {
230
212
  scope: selectedSemanticSources,
231
213
  ontologyVersion: config.semantic.ontologyVersion,
232
214
  extractorVersions: {
233
- assembly: 'standalone-graph-v6',
215
+ assembly: graphAssemblyIdentity,
216
+ [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
217
+ [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
234
218
  'repository-node-identity': String(repositoryNodeIdentityVersion),
235
219
  [documentLinkerIdentity.id]: documentLinkerIdentity.version,
236
220
  ...(config.routingSources?.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}),
@@ -256,12 +240,13 @@ export class ContextGraphHydrator {
256
240
  unresolved,
257
241
  diagnostics,
258
242
  };
259
- options.signal?.throwIfAborted();
260
- options.onProgress?.({ phase: 'validate-shacl' });
261
- const validation = await new RdfEvidenceGraph(snapshot).validate();
243
+ checkpoint();
244
+ const graph = await measureValidationPhase('build-rdf-evidence', validationCounts, options.onProgress, () => new RdfEvidenceGraph(snapshot));
245
+ const validation = await graph.validate({ onProgress: (event) => options.onProgress?.({ ...event, nodeCount: validationCounts.nodeCount }) });
262
246
  if (!validation.conforms) {
263
247
  throw new Error(`Graph extraction failed SHACL validation: ${JSON.stringify(validation.issues)}`);
264
248
  }
249
+ checkpoint();
265
250
  options.onProgress?.({ phase: 'publish-snapshot' });
266
251
  await snapshotStore.write(snapshot);
267
252
  await snapshotStore.updateHead(options.headName, snapshotId);
@@ -3,4 +3,6 @@ import type { GraphSnapshot } from "@weftlet/core";
3
3
  import { GitRepository } from "../repository/git.js";
4
4
  import { FileSnapshotStore } from "../storage/storage.js";
5
5
  /** Conservative code overlay: reindex implementation; invalidate changed prose, never synthesize intent. */
6
- export declare function buildWorkingOverlay(repository: GitRepository, config: ContextGraphConfig, store: FileSnapshotStore, storageRoot: string, baseSnapshotId: string, agent?: string): Promise<GraphSnapshot>;
6
+ export declare function buildWorkingOverlay(repository: GitRepository, config: ContextGraphConfig, store: FileSnapshotStore, storageRoot: string, baseSnapshotId: string, agent?: string, options?: {
7
+ readonly validationDetails?: boolean;
8
+ }): Promise<GraphSnapshot>;
@@ -5,19 +5,21 @@ import { promisify } from 'node:util';
5
5
  import { scipSourcePatterns } from '../configuration/scip-project.js';
6
6
  import { GitRepository } from "../repository/git.js";
7
7
  import { withCommittedTree } from "../repository/committed-tree.js";
8
- import { RepositoryContractExtractor, RepositoryFileExtractor, ScipSymbolExtractor, ScipTypeScriptIndexer, repositoryFileExtractorIdentity } from "../extraction/static.js";
8
+ import { RepositoryContractExtractor, RepositoryFileExtractor, ScipSymbolExtractor, ScipTypeScriptIndexer, repositoryFileExtractorIdentity, repositoryContractExtractorIdentity, scipSymbolExtractorIdentity } from "../extraction/static.js";
9
9
  import { sha256, stableJson } from "../infrastructure/util.js";
10
10
  import { contextPackEvidenceAuthority } from "../graph/query.js";
11
11
  import { inWorkingScope, readWorkingContent } from "../repository/working-content.js";
12
12
  import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
13
13
  import { RdfEvidenceGraph } from '../graph/rdf.js';
14
+ import { graphAssemblyIdentity } from './extraction-identity.js';
15
+ import { assertGraphIdentities } from '../graph/identity-preflight.js';
14
16
  const execute = promisify(execFile);
15
17
  const implementationExtractors = new Set([
16
18
  'repository-file-extractor', 'repository-contract-extractor', 'scip-symbol-extractor',
17
19
  routingExtractorIdentity.id,
18
20
  ]);
19
21
  /** Conservative code overlay: reindex implementation; invalidate changed prose, never synthesize intent. */
20
- export async function buildWorkingOverlay(repository, config, store, storageRoot, baseSnapshotId, agent = 'default') {
22
+ export async function buildWorkingOverlay(repository, config, store, storageRoot, baseSnapshotId, agent = 'default', options = {}) {
21
23
  const base = await store.read(baseSnapshotId);
22
24
  if (!base || base.manifest.overlayDigest)
23
25
  throw new Error('Overlay requires a committed base snapshot');
@@ -57,7 +59,10 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
57
59
  // identity also binds the extractor so rebuilding an old base cannot replace an old overlay.
58
60
  const snapshotId = `${head}-overlay-${sha256(stableJson({
59
61
  baseContentDigest: base.manifest.contentDigest, overlayDigest,
62
+ assembly: graphAssemblyIdentity,
60
63
  repositoryFileExtractor: repositoryFileExtractorIdentity,
64
+ repositoryContractExtractor: repositoryContractExtractorIdentity,
65
+ scipSymbolExtractor: scipSymbolExtractorIdentity,
61
66
  ...(routingSources.length ? { routingExtractor: routingExtractorIdentity } : {}),
62
67
  })).slice(0, 32)}`;
63
68
  const affected = new Set(captured.map((file) => file.path));
@@ -115,12 +120,16 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
115
120
  !affected.has(fact.provenance.path) &&
116
121
  ![fact.subject, fact.object].some((node) => [...affected].some((path) => node.id === `component:file:${path}` || node.id === `document:${path}` || node.label === path))).map((fact) => ({ ...fact, snapshotId }));
117
122
  const allFacts = [...retained, ...extracted.facts].sort((a, b) => a.id.localeCompare(b.id));
123
+ assertGraphIdentities(allFacts, { repositoryId: config.repositoryId, includeDetails: options.validationDetails });
118
124
  const overlay = {
119
125
  ...base,
120
126
  manifest: {
121
127
  ...base.manifest, snapshotId, baseSha: head, overlayDigest, overlayCapture,
122
128
  extractorVersions: { ...base.manifest.extractorVersions,
129
+ assembly: graphAssemblyIdentity,
123
130
  [repositoryFileExtractorIdentity.id]: repositoryFileExtractorIdentity.version,
131
+ [repositoryContractExtractorIdentity.id]: repositoryContractExtractorIdentity.version,
132
+ [scipSymbolExtractorIdentity.id]: scipSymbolExtractorIdentity.version,
124
133
  ...(routingSources.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}) },
125
134
  generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length
126
135
  ? { facts: allFacts, diagnostics: extracted.diagnostics } : allFacts)),
@@ -136,8 +145,8 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
136
145
  ],
137
146
  diagnostics: extracted.diagnostics,
138
147
  };
139
- if (routingSources.length && !(await new RdfEvidenceGraph(overlay).validate()).conforms) {
140
- throw new Error('Draft routing overlay failed SHACL validation; saved evidence was preserved');
148
+ if (!(await new RdfEvidenceGraph(overlay).validate()).conforms) {
149
+ throw new Error('Draft overlay failed SHACL validation; saved evidence was preserved');
141
150
  }
142
151
  if (await repository.resolveRef('HEAD') !== head)
143
152
  throw new Error('HEAD changed during overlay capture; retry');
@@ -5,20 +5,23 @@ export declare class SavedEvidenceQueryService implements ImpactQueryService {
5
5
  private readonly queries;
6
6
  private readonly repository;
7
7
  private readonly configurationDigest;
8
- constructor(queries: ImpactQueryService, repository: GitRepository, configurationDigest: string);
8
+ private readonly expectedExtractors?;
9
+ constructor(queries: ImpactQueryService, repository: GitRepository, configurationDigest: string, expectedExtractors?: Readonly<Record<string, string>> | undefined);
9
10
  scan(request: ImpactScanRequest): Promise<{
10
11
  markdown: string;
11
12
  evidenceState: {
13
+ eligibleForCurrentDiffValidation: false;
14
+ extractionCompatible?: boolean | undefined;
15
+ incompatibleExtractors?: readonly string[] | undefined;
12
16
  stale: boolean;
13
17
  configurationMatches: boolean;
14
- eligibleForCurrentDiffValidation: false;
15
18
  overlayFreshness?: "current" | "stale" | "unknown" | undefined;
16
19
  capturedWorkingChanges?: string[] | undefined;
17
20
  changedSinceCapture?: string[] | undefined;
18
21
  currentCommit: string | undefined;
19
22
  excludedWorkingChanges: string[];
20
23
  };
21
- snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture">;
24
+ snapshot: Pick<import("@weftlet/core").SnapshotManifest, "snapshotId" | "commitSha" | "contentDigest" | "configurationDigest" | "scope" | "baseSha" | "overlayDigest" | "overlayCapture"> & Partial<Pick<import("@weftlet/core").SnapshotManifest, "extractorVersions">>;
22
25
  capabilities: readonly import("@weftlet/core").GraphNode[];
23
26
  facts: readonly import("@weftlet/core").GraphFact[];
24
27
  invariants: readonly import("@weftlet/core").InvariantResult[];
@@ -1,15 +1,18 @@
1
1
  import { MarkdownContextPackRenderer } from "../graph/query.js";
2
2
  import { inWorkingScope, readWorkingContent } from "../repository/working-content.js";
3
3
  import { sha256 } from "../infrastructure/util.js";
4
+ import { incompatibleSnapshotExtractors } from './extraction-identity.js';
4
5
  /** Read-side metadata; never calls an indexer, model, compiler, or refresh operation. */
5
6
  export class SavedEvidenceQueryService {
6
7
  queries;
7
8
  repository;
8
9
  configurationDigest;
9
- constructor(queries, repository, configurationDigest) {
10
+ expectedExtractors;
11
+ constructor(queries, repository, configurationDigest, expectedExtractors) {
10
12
  this.queries = queries;
11
13
  this.repository = repository;
12
14
  this.configurationDigest = configurationDigest;
15
+ this.expectedExtractors = expectedExtractors;
13
16
  }
14
17
  async scan(request) {
15
18
  const pack = await this.queries.scan(request);
@@ -45,12 +48,15 @@ export class SavedEvidenceQueryService {
45
48
  }
46
49
  }
47
50
  const excludedWorkingChanges = workingChanges.filter((path) => !capturedWorkingChanges.includes(path));
51
+ const incompatibleExtractors = this.expectedExtractors
52
+ ? incompatibleSnapshotExtractors(pack.snapshot.extractorVersions, this.expectedExtractors) : undefined;
48
53
  const evidenceState = {
49
54
  currentCommit,
50
55
  excludedWorkingChanges,
51
56
  ...(overlayFreshness ? { overlayFreshness, capturedWorkingChanges, changedSinceCapture: [...changedSinceCapture].sort() } : {}),
52
57
  stale: currentCommit !== pack.snapshot.commitSha || (overlayFreshness !== undefined && overlayFreshness !== 'current'),
53
58
  configurationMatches: pack.snapshot.configurationDigest === this.configurationDigest,
59
+ ...(incompatibleExtractors ? { extractionCompatible: incompatibleExtractors.length === 0, incompatibleExtractors } : {}),
54
60
  eligibleForCurrentDiffValidation: false,
55
61
  };
56
62
  const result = { ...pack, evidenceState };
@@ -0,0 +1,4 @@
1
+ import type { ContextGraphConfig } from '../configuration/config.js';
2
+ import { SemanticArtifactStore } from '../storage/storage.js';
3
+ /** Shared by planning and execution; construction never creates directories or reads credentials. */
4
+ export declare function createSemanticArtifactStore(repositoryRoot: string, config: ContextGraphConfig, userCacheRoot?: string): SemanticArtifactStore;
@@ -0,0 +1,7 @@
1
+ import { resolveSemanticStorageRoots } from '../configuration/cache.js';
2
+ import { SemanticArtifactStore } from '../storage/storage.js';
3
+ /** Shared by planning and execution; construction never creates directories or reads credentials. */
4
+ export function createSemanticArtifactStore(repositoryRoot, config, userCacheRoot) {
5
+ const { primary, legacy } = resolveSemanticStorageRoots(repositoryRoot, config, userCacheRoot);
6
+ return new SemanticArtifactStore(primary, legacy ? [legacy] : []);
7
+ }
@@ -0,0 +1,32 @@
1
+ import type { ExtractionContext, SemanticSource } from '@weftlet/core';
2
+ import type { ContextGraphConfig } from '../configuration/config.js';
3
+ import type { SemanticHydrator } from '../extraction/semantic.js';
4
+ export interface HydrationProgressEvent {
5
+ readonly phase: string;
6
+ readonly completed?: number;
7
+ readonly total?: number;
8
+ readonly sourceId?: string;
9
+ readonly elapsedMs?: number;
10
+ readonly status?: 'started' | 'completed' | 'failed';
11
+ readonly factCount?: number;
12
+ readonly nodeCount?: number;
13
+ }
14
+ /** Settle active workers before reporting failure; never publish snapshots here. */
15
+ export declare function hydrateSemanticSources(options: {
16
+ readonly sources: readonly SemanticSource[];
17
+ readonly context: ExtractionContext;
18
+ readonly config: ContextGraphConfig['semantic'];
19
+ readonly configurationDigest: string;
20
+ readonly hydrator: SemanticHydrator;
21
+ readonly forceRevision?: string;
22
+ readonly signal?: AbortSignal;
23
+ readonly onProgress?: (event: HydrationProgressEvent) => void;
24
+ }): Promise<{
25
+ hydrated: readonly {
26
+ artifact: import("@weftlet/core").SemanticArtifact;
27
+ reused: boolean;
28
+ source: SemanticSource;
29
+ }[];
30
+ semanticExtracted: number;
31
+ semanticReused: number;
32
+ }>;
@@ -0,0 +1,111 @@
1
+ import { semanticSourceId } from '../extraction/semantic-diagnostics.js';
2
+ import { SemanticHydrationError, semanticFailureCategory, semanticFailureCode } from './hydration-failure.js';
3
+ async function concurrentMap(values, concurrency, worker, shouldStop) {
4
+ const results = new Array(values.length);
5
+ let cursor = 0;
6
+ let failure;
7
+ let failed = false;
8
+ await Promise.all(Array.from({ length: Math.max(1, Math.min(concurrency, values.length)) }, async () => {
9
+ while (!failed && !shouldStop() && cursor < values.length) {
10
+ const index = cursor++;
11
+ try {
12
+ results[index] = await worker(values[index], index);
13
+ }
14
+ catch (error) {
15
+ failed = true;
16
+ failure ??= error;
17
+ }
18
+ }
19
+ }));
20
+ if (failed)
21
+ throw failure;
22
+ return results;
23
+ }
24
+ /** Settle active workers before reporting failure; never publish snapshots here. */
25
+ export async function hydrateSemanticSources(options) {
26
+ const { sources, context, config, configurationDigest, hydrator, forceRevision } = options;
27
+ const { commitSha } = context;
28
+ const runStarted = performance.now();
29
+ let semanticExtracted = 0;
30
+ let semanticReused = 0;
31
+ let started = 0;
32
+ let failed = 0;
33
+ let cancelled = 0;
34
+ let firstFailure;
35
+ const controller = new AbortController();
36
+ const signal = AbortSignal.any([controller.signal, ...(options.signal ? [options.signal] : [])]);
37
+ const hydrated = await concurrentMap(sources, config.concurrency, async (source) => {
38
+ started += 1;
39
+ const sourceId = semanticSourceId(context.repositoryId, source);
40
+ const sourceStarted = performance.now();
41
+ let phase = 'semantic-cache-read';
42
+ options.onProgress?.({ phase: 'semantic-source-started', sourceId,
43
+ completed: semanticExtracted + semanticReused, total: sources.length });
44
+ let result;
45
+ try {
46
+ result = await hydrator.hydrate(source, context, forceRevision, { signal, onPhase: value => { phase = value; } });
47
+ }
48
+ catch (error) {
49
+ // Record the initiating failure before abort synchronously notifies other workers.
50
+ const code = semanticFailureCode(error, phase, signal);
51
+ if (code === 'cancelled')
52
+ cancelled += 1;
53
+ else
54
+ failed += 1;
55
+ firstFailure ??= { code, sourceId, phase, elapsedScope: 'source', elapsedMs: Math.round(performance.now() - sourceStarted) };
56
+ controller.abort();
57
+ throw error;
58
+ }
59
+ if (result.reused)
60
+ semanticReused += 1;
61
+ else
62
+ semanticExtracted += 1;
63
+ options.onProgress?.({ phase: result.reused ? 'semantic-cache-hit' : 'semantic-extracted',
64
+ sourceId, elapsedMs: Math.round(performance.now() - sourceStarted),
65
+ completed: semanticReused + semanticExtracted, total: sources.length });
66
+ return { source, ...result };
67
+ }, () => signal.aborted).catch(() => {
68
+ if (!firstFailure)
69
+ throw new Error('Semantic hydration failed before source processing could be reported');
70
+ return undefined;
71
+ });
72
+ // Cancellation can arrive during a durable write or completion callback, without
73
+ // a failed source. Do not relabel successfully retained work as cancelled.
74
+ if (!firstFailure && signal.aborted)
75
+ firstFailure = { code: 'cancelled', phase: 'semantic-scheduling',
76
+ elapsedScope: 'semantic-run', elapsedMs: Math.round(performance.now() - runStarted) };
77
+ if (firstFailure) {
78
+ // concurrentMap settles every active worker before returning: these are final counts.
79
+ const completed = semanticExtracted + semanticReused;
80
+ const recovery = [
81
+ 'No snapshot was published by this run. Previously published evidence and completed artifacts were retained.',
82
+ 'Inspect hydrate --plan --json locally with the same --config, committed --ref and scope options to map sourceId to its path and check expected reuse. The preview contains local paths; review it before sharing.',
83
+ ];
84
+ if (firstFailure.code === 'timeout')
85
+ recovery.push('Review semantic.timeoutMs before retrying. This limit is per Codex attempt (authentication is separately bounded), or covers module import/factory/extraction; source elapsed time can be longer.');
86
+ else if (firstFailure.code === 'unavailable')
87
+ recovery.push('Run doctor with the same configuration to check provider installation and authentication.');
88
+ else if (firstFailure.code === 'cache-failure')
89
+ recovery.push('Inspect cache integrity and permissions. Corrupt or conflicting complete artifacts are retained and must be repaired explicitly; a timeout change cannot repair them.');
90
+ else if (firstFailure.code === 'invalid-output')
91
+ recovery.push('Review source coverage and extraction policy locally; invalid output is an extraction gap, not an approved rule or a passing validation.');
92
+ else if (firstFailure.code === 'unresolved' || firstFailure.code === 'low-confidence')
93
+ recovery.push('Review cited sources and configured confidence/unresolved policy locally. Required evidence needs review; a policy rejection is not approved intent or a passing validation.');
94
+ else if (firstFailure.code === 'exit' || firstFailure.code === 'provider-failure')
95
+ recovery.push('Check provider readiness locally. Private provider messages are omitted; this failure is not proof of a timeout.');
96
+ recovery.push(forceRevision
97
+ ? 'This run used --force-semantic. Its completed revision artifacts remain stored, but an ordinary rerun or a new forced run does not automatically reuse that revision.'
98
+ : 'Rerun hydrate without --force-semantic after resolving the failure; only valid compatible completed artifacts are reusable.');
99
+ recovery.push(config.provider === 'codex-cli'
100
+ ? 'Built-in Codex supports reuse across operational timeout changes. For profiles created before shared caching, use cache adopt --from-config with the exact previous configuration; preview with --dry-run first.'
101
+ : 'Custom provider identities include their full semantic configuration; changing a module timeout may invalidate reuse. Preview again before extraction.');
102
+ throw new SemanticHydrationError({ schemaVersion: 1, kind: 'semantic-hydration-failure', ...firstFailure,
103
+ category: semanticFailureCategory(firstFailure.code), commitSha, configurationDigest,
104
+ timeoutMs: config.timeoutMs ?? 120_000,
105
+ timeoutScope: config.provider === 'codex-cli' ? 'provider-attempt' : 'module-execution',
106
+ progress: { total: sources.length, completed, extracted: semanticExtracted, reused: semanticReused,
107
+ failed, cancelled, notStarted: sources.length - started, remaining: sources.length - completed },
108
+ forced: !!forceRevision, recovery });
109
+ }
110
+ return { hydrated: hydrated, semanticExtracted, semanticReused };
111
+ }
@@ -0,0 +1,13 @@
1
+ import type { ImpactScanRequest } from '@weftlet/core';
2
+ import type { ContextGraphConfig } from '../configuration/config.js';
3
+ import type { GitRepository } from '../repository/git.js';
4
+ export interface SemanticScope {
5
+ readonly mode: 'configured' | 'agent';
6
+ readonly selectors: readonly string[];
7
+ readonly digest: string;
8
+ readonly headName: string;
9
+ readonly omittedDocuments: readonly string[];
10
+ readonly ignoredSearchTerms: number;
11
+ }
12
+ /** Resolve the same committed selection for previews and explicit artifact recovery. */
13
+ export declare function resolveSemanticScope(config: ContextGraphConfig, repository: GitRepository, commitSha: string, request?: ImpactScanRequest): Promise<SemanticScope>;
@@ -0,0 +1,12 @@
1
+ import { sha256, stableJson } from '../infrastructure/util.js';
2
+ import { AgentSemanticScopeResolver } from './agent-context.js';
3
+ /** Resolve the same committed selection for previews and explicit artifact recovery. */
4
+ export async function resolveSemanticScope(config, repository, commitSha, request = {}) {
5
+ const agent = config.agentContext
6
+ ? await new AgentSemanticScopeResolver(config, repository).resolve(request, commitSha)
7
+ : undefined;
8
+ const selectors = agent?.semanticSources ?? config.semanticSources;
9
+ return { mode: agent ? 'agent' : 'configured', selectors, digest: sha256(stableJson(selectors)),
10
+ headName: agent?.headName ?? 'default', omittedDocuments: agent?.omittedDocuments ?? [],
11
+ ignoredSearchTerms: agent?.ignoredSearchTerms ?? 0 };
12
+ }
@@ -8,7 +8,8 @@ export declare class SnapshotChecks {
8
8
  private readonly hydrator;
9
9
  private readonly scopeResolver?;
10
10
  private readonly configurationDigest?;
11
- constructor(store: Pick<SnapshotStore, 'readHead'>, repository: Pick<GitRepository, 'resolveRef'> & Partial<Pick<GitRepository, 'workingChanges'>>, hydrator: Pick<ContextGraphHydrator, 'compareClean'>, scopeResolver?: Pick<AgentSemanticScopeResolver, "resolve"> | undefined, configurationDigest?: string | undefined);
11
+ private readonly expectedExtractors;
12
+ constructor(store: Pick<SnapshotStore, 'readHead'>, repository: Pick<GitRepository, 'resolveRef'> & Partial<Pick<GitRepository, 'workingChanges'>>, hydrator: Pick<ContextGraphHydrator, 'compareClean'>, scopeResolver?: Pick<AgentSemanticScopeResolver, "resolve"> | undefined, configurationDigest?: string | undefined, expectedExtractors?: Readonly<Record<string, string>>);
12
13
  private target;
13
14
  private readTarget;
14
15
  read(request: ImpactScanRequest, ref?: string): Promise<import("@weftlet/core").GraphSnapshot>;
@@ -1,3 +1,4 @@
1
+ import { incompatibleSnapshotExtractors, requiredSnapshotExtractorVersions } from './extraction-identity.js';
1
2
  // Final checks must use the same scope resolver as prepared impact scans, without silently
2
3
  // starting hydration or substituting another owner's (or an older commit's) evidence.
3
4
  export class SnapshotChecks {
@@ -6,12 +7,14 @@ export class SnapshotChecks {
6
7
  hydrator;
7
8
  scopeResolver;
8
9
  configurationDigest;
9
- constructor(store, repository, hydrator, scopeResolver, configurationDigest) {
10
+ expectedExtractors;
11
+ constructor(store, repository, hydrator, scopeResolver, configurationDigest, expectedExtractors = requiredSnapshotExtractorVersions()) {
10
12
  this.store = store;
11
13
  this.repository = repository;
12
14
  this.hydrator = hydrator;
13
15
  this.scopeResolver = scopeResolver;
14
16
  this.configurationDigest = configurationDigest;
17
+ this.expectedExtractors = expectedExtractors;
15
18
  }
16
19
  async target(request, ref) {
17
20
  if (request.snapshotId)
@@ -38,6 +41,10 @@ export class SnapshotChecks {
38
41
  if (this.configurationDigest && snapshot.manifest.configurationDigest !== this.configurationDigest) {
39
42
  throw new Error('Snapshot configuration differs from current configuration; refresh required');
40
43
  }
44
+ const incompatible = incompatibleSnapshotExtractors(snapshot.manifest.extractorVersions, this.expectedExtractors);
45
+ if (incompatible.length) {
46
+ throw new Error(`Snapshot extraction versions require refresh (${incompatible.join(', ')}). Historical evidence remains queryable; hydrate with the current toolkit before final validation.`);
47
+ }
41
48
  return snapshot;
42
49
  }
43
50
  async read(request, ref = 'HEAD') {
@@ -1,2 +1,8 @@
1
1
  import type { ContextGraphConfig } from "./config.js";
2
2
  export declare function resolveStorageRoot(repositoryRoot: string, config: ContextGraphConfig, userCacheRoot?: string): string;
3
+ export declare function semanticCachePartition(repositoryRoot: string, config: ContextGraphConfig): string;
4
+ /** Snapshot profiles retain the full configuration. Only Codex user artifacts share operational changes. */
5
+ export declare function resolveSemanticStorageRoots(repositoryRoot: string, config: ContextGraphConfig, userCacheRoot?: string): {
6
+ primary: string;
7
+ legacy?: string;
8
+ };
@@ -8,3 +8,21 @@ export function resolveStorageRoot(repositoryRoot, config, userCacheRoot = resol
8
8
  const profile = sha256(stableJson({ config, checkout: resolve(repositoryRoot) })).slice(0, 20);
9
9
  return resolve(userCacheRoot, profile);
10
10
  }
11
+ // These controls do not change a successful built-in Codex extraction. Unknown
12
+ // fields and module-provider configuration remain conservative identity inputs.
13
+ const codexOperationalFields = new Set([
14
+ 'timeoutMs', 'retries', 'concurrency', 'maxDocuments', 'maxSourceBytes',
15
+ ]);
16
+ export function semanticCachePartition(repositoryRoot, config) {
17
+ const semantic = config.semantic.provider === 'codex-cli'
18
+ ? Object.fromEntries(Object.entries(config.semantic).filter(([field]) => !codexOperationalFields.has(field)))
19
+ : config.semantic;
20
+ return sha256(stableJson({ config: { ...config, semantic }, checkout: resolve(repositoryRoot) }));
21
+ }
22
+ /** Snapshot profiles retain the full configuration. Only Codex user artifacts share operational changes. */
23
+ export function resolveSemanticStorageRoots(repositoryRoot, config, userCacheRoot = resolve(homedir(), '.cache', 'context-graph')) {
24
+ const legacy = resolveStorageRoot(repositoryRoot, config, userCacheRoot);
25
+ if (config.storageScope !== 'user' || config.semantic.provider !== 'codex-cli')
26
+ return { primary: legacy };
27
+ return { primary: resolve(userCacheRoot, 'semantic-artifacts-v1', semanticCachePartition(repositoryRoot, config)), legacy };
28
+ }
@@ -113,7 +113,7 @@ function validateConfig(value, absolutePath) {
113
113
  requireConfig(auth.apiKeyEnv === undefined || (typeof auth.apiKeyEnv === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(auth.apiKeyEnv)), 'semantic.auth.apiKeyEnv must name an environment variable', absolutePath);
114
114
  }
115
115
  }
116
- for (const [field, max] of [['timeoutMs', 300_000], ['retries', 2], ['maxDocuments', 1000], ['maxSourceBytes', 1_048_576]]) {
116
+ for (const [field, max] of [['timeoutMs', 600_000], ['retries', 2], ['maxDocuments', 1000], ['maxSourceBytes', 1_048_576]]) {
117
117
  const item = value.semantic[field];
118
118
  if (item !== undefined)
119
119
  requireConfig(Number.isInteger(item) && Number(item) >= (field === 'retries' ? 0 : 1) && Number(item) <= max, `semantic.${field} exceeds supported bounds`, absolutePath);
@@ -1,6 +1,6 @@
1
1
  import { sha256, stableJson } from '../infrastructure/util.js';
2
2
  import { documentReferences, resolveDocumentReference } from './document-references.js';
3
- import { repositoryFileNode as fileNode } from './repository-nodes.js';
3
+ import { repositoryFileNode as fileNode } from '@weftlet/core';
4
4
  export const documentLinkerIdentity = { id: 'document-linker', version: '2.0.0' };
5
5
  export function linkSemanticDocuments(context, hydrated, trackedPaths, approvedSourcePaths = []) {
6
6
  const facts = new Map();
@@ -31,14 +31,21 @@ export function createSemanticExtractor(config, repositoryRoot) {
31
31
  Promise.resolve().then(async () => {
32
32
  checkpoint();
33
33
  if (!extractor) {
34
- const module = await import(resolveConsumerModule(modulePath, repositoryRoot));
34
+ let module;
35
+ try {
36
+ module = await import(resolveConsumerModule(modulePath, repositoryRoot));
37
+ }
38
+ catch {
39
+ checkpoint();
40
+ throw new SemanticProviderError('unavailable', 'Semantic provider module could not be loaded');
41
+ }
35
42
  checkpoint();
36
43
  if (typeof module.createSemanticExtractor !== 'function')
37
- throw new Error('Provider module must export createSemanticExtractor(config)');
44
+ throw new SemanticProviderError('unavailable', 'Provider module must export createSemanticExtractor(config)');
38
45
  const candidate = await module.createSemanticExtractor(config, { ...options, signal });
39
46
  checkpoint();
40
47
  if (candidate?.identity !== config.providerIdentity || typeof candidate.extract !== 'function') {
41
- throw new Error('Provider identity/contract does not match configuration');
48
+ throw new SemanticProviderError('unavailable', 'Provider identity/contract does not match configuration');
42
49
  }
43
50
  extractor = candidate;
44
51
  }
@@ -1,7 +1,6 @@
1
- import type { GraphNode } from '@weftlet/core';
1
+ import { type GraphNode } from '@weftlet/core';
2
+ export { repositoryFileNode } from '@weftlet/core';
2
3
  /** Internal cache contract: changing this policy must also change snapshot assembly identity. */
3
4
  export declare const repositoryNodeIdentityVersion = 1;
4
- /** Physical file identity; this does not establish existence, intent, ownership or approval. */
5
- export declare function repositoryFileNode(path: string): GraphNode;
6
5
  /** Normalize only reserved document/file identities, never inferred business concept IDs. */
7
6
  export declare function canonicalizeRepositoryDocumentNode(node: GraphNode): GraphNode;