@weftlet/toolkit 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +36 -1
  2. package/THIRD_PARTY_NOTICES.md +33 -0
  3. package/dist/application/hydrator.js +18 -5
  4. package/dist/application/overlay.js +46 -9
  5. package/dist/application/saved-evidence.d.ts +1 -0
  6. package/dist/configuration/config.d.ts +4 -0
  7. package/dist/configuration/config.js +12 -0
  8. package/dist/configuration/scip-project.d.ts +9 -0
  9. package/dist/configuration/scip-project.js +40 -0
  10. package/dist/extraction/codex-auth.js +9 -2
  11. package/dist/extraction/codex-command.d.ts +8 -0
  12. package/dist/extraction/codex-command.js +81 -0
  13. package/dist/extraction/document-references.d.ts +18 -0
  14. package/dist/extraction/document-references.js +94 -0
  15. package/dist/extraction/document.d.ts +9 -2
  16. package/dist/extraction/document.js +44 -46
  17. package/dist/extraction/module-request.d.ts +3 -0
  18. package/dist/extraction/module-request.js +25 -0
  19. package/dist/extraction/routing.d.ts +14 -0
  20. package/dist/extraction/routing.js +151 -0
  21. package/dist/extraction/scip-project.d.ts +3 -0
  22. package/dist/extraction/scip-project.js +33 -0
  23. package/dist/extraction/semantic-process.js +17 -5
  24. package/dist/extraction/semantic.js +13 -2
  25. package/dist/extraction/static.d.ts +7 -3
  26. package/dist/extraction/static.js +15 -9
  27. package/dist/graph/query.js +21 -34
  28. package/dist/graph/ranking.d.ts +10 -0
  29. package/dist/graph/ranking.js +85 -0
  30. package/dist/interfaces/doctor.js +12 -3
  31. package/docs/ADR-002-saved-evidence.md +7 -1
  32. package/docs/ADR-003-draft-overlays.md +6 -0
  33. package/docs/ADR-004-rdf-query-path.md +6 -0
  34. package/docs/ADR-006-provider-boundaries.md +5 -0
  35. package/docs/ADR-010-committed-source-indexing.md +77 -0
  36. package/docs/ADR-011-explicit-document-links.md +48 -0
  37. package/docs/ADR-012-impact-ranking.md +50 -0
  38. package/docs/ADR-013-relative-module-requests.md +58 -0
  39. package/docs/ADR-014-routing-declarations.md +108 -0
  40. package/docs/CONSUMER-TRIAL.md +165 -0
  41. package/docs/ENVIRONMENT-VALIDATION.md +9 -0
  42. package/docs/IMPACT-EVALUATION.md +363 -0
  43. package/docs/PROVIDERS.md +30 -2
  44. package/docs/RELEASING.md +3 -1
  45. package/docs/ROADMAP.md +44 -4
  46. package/docs/SOURCE-INVENTORY.md +10 -0
  47. package/docs/VALIDATION.md +366 -0
  48. package/docs/VIEWER.md +5 -0
  49. package/docs/extraction-manifest.json +87 -16
  50. package/package.json +5 -2
package/README.md CHANGED
@@ -7,11 +7,13 @@ contracts, implementing code, downstream consumers, tests and observed runtime e
7
7
  Agents can query cited evidence and explore dependencies across module and asynchronous
8
8
  boundaries. See the [roadmap](docs/ROADMAP.md) and [validation](docs/VALIDATION.md) for
9
9
  supported capabilities and limitations.
10
+ For a first real-project evaluation, use the [private consumer trial](docs/CONSUMER-TRIAL.md)
11
+ to retain reproducible queries, logs and human-reviewed findings.
10
12
 
11
13
  ## Set up a project
12
14
 
13
15
  Weftlet uses `@weftlet/toolkit`; its neutral `@weftlet/core` dependency is installed
14
- automatically. Both packages are available on npm at `0.1.1`.
16
+ automatically. Both packages are available on npm at `0.1.2`.
15
17
  See [release status](docs/RELEASING.md). Existing Tently users should follow
16
18
  the [migration guide](docs/MIGRATING-TO-WEFTLET.md).
17
19
 
@@ -100,6 +102,25 @@ Copy `examples/generic-typescript` into a separate repository, or adapt its
100
102
  `context-graph.config.json` beside your own sources. Paths resolve relative to that configuration.
101
103
  Configure semantic source selection, TypeScript roots/indexer arguments, contract aliases,
102
104
  evidence roots, runtime evidence, storage and optional adapter explicitly.
105
+ For tests excluded by the build config or workspace packages whose outputs are absent from
106
+ committed archives, configure an [explicit TypeScript source project](docs/ADR-010-committed-source-indexing.md)
107
+ with selected files and local package aliases.
108
+
109
+ In a selected document such as `docs/intent.md`, link a responsibility to its sources:
110
+
111
+ ```md
112
+ Committed snapshots stay queryable during edits. See [snapshot storage](../src/storage.ts)
113
+ and [lifecycle checks](../tests/storage.test.ts).
114
+ ```
115
+
116
+ [Explicit document links](docs/ADR-011-explicit-document-links.md) resolve against the recorded
117
+ commit and retain source-line citations. Missing files produce diagnostics; directory mentions
118
+ are not expanded. A reference does not approve intent or establish that a linked test passed.
119
+
120
+ For a consumer that shares no typed symbol with its producer, opt into
121
+ [explicit routing declarations](docs/ADR-014-routing-declarations.md) using `routingSources`.
122
+ Each committed JSON record names a scoped binding and exact producer/consumer files. These
123
+ are cited candidate dependencies; they do not assert an approved policy or observed delivery.
103
124
 
104
125
  Install the locally packed core and toolkit tarballs together. The installed commands are
105
126
  `weftlet`, `weftlet-mcp`, and the compatible `context-graph` / `context-graph-mcp` aliases. For example, after preparing a release:
@@ -131,6 +152,16 @@ pending re-extraction. Queries compare captured file digests with current workin
131
152
  report subsequent edits separately from captured and excluded changes. Older overlays without
132
153
  capture metadata have unknown freshness. Historical or draft evidence cannot validate a current diff.
133
154
 
155
+ Impact queries [rank nearby consumers and varied evidence sources](docs/ADR-012-impact-ranking.md)
156
+ ahead of repeated detail and shared-import context. Ranking changes ordering, not saved facts or
157
+ the reachable scope. Responses identify the ranking version; after a ranking upgrade, restart
158
+ scans whose old continuation tokens are rejected. No snapshot refresh is needed for new ordering.
159
+
160
+ [Relative import evidence](docs/ADR-013-relative-module-requests.md) now scopes module requests
161
+ to the importing directory. An explicit refresh applies this extraction fix; saved snapshots
162
+ keep their original facts. These requests remain unresolved evidence, while SCIP supplies
163
+ typed-symbol links. Matching queue topic strings alone does not establish a downstream route.
164
+
134
165
  ## Evidence limits
135
166
 
136
167
  Approved owner documents govern intent. Code describes implementation; tests describe what
@@ -158,6 +189,10 @@ See [provider contract](docs/PROVIDERS.md) for bounded extraction and adapters,
158
189
 
159
190
  See the [bounded viewer guide](docs/VIEWER.md) and [cloud setup](docs/CLOUD-ENVIRONMENT.md).
160
191
 
192
+ For development, `pnpm evaluate:impact` measures bounded candidate-file retrieval on Weftlet's
193
+ own committed source without model calls. See the [evaluation protocol and baseline](docs/IMPACT-EVALUATION.md)
194
+ for observed gaps and the distinction between retrieval measurements and live semantic quality.
195
+
161
196
  ## License and provenance
162
197
 
163
198
  Apache-2.0. See [LICENSE](LICENSE), [NOTICE](NOTICE), and the
@@ -388,10 +388,13 @@ Do not edit this file manually.
388
388
  | `@traqula/parser-sparql-1-2` | 1.4.0 | MIT | [link](https://github.com/comunica/traqula#readme) |
389
389
  | `@traqula/rules-sparql-1-1` | 1.4.0 | MIT | [link](https://github.com/comunica/traqula#readme) |
390
390
  | `@traqula/rules-sparql-1-2` | 1.4.0 | MIT | [link](https://github.com/comunica/traqula#readme) |
391
+ | `@types/debug` | 4.1.13 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/debug) |
391
392
  | `@types/http-cache-semantics` | 4.2.0 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/http-cache-semantics) |
392
393
  | `@types/http-link-header` | 1.0.7 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/http-link-header) |
393
394
  | `@types/imurmurhash` | 0.1.4 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/imurmurhash) |
395
+ | `@types/mdast` | 4.0.4 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/mdast) |
394
396
  | `@types/minimist` | 1.2.5 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/minimist) |
397
+ | `@types/ms` | 2.1.0 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/ms) |
395
398
  | `@types/n3` | 1.26.1 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/n3) |
396
399
  | `@types/node` | 18.19.130, 20.19.11, 24.19.2 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/node) |
397
400
  | `@types/rdfjs__namespace` | 2.0.10 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/rdfjs__namespace) |
@@ -400,6 +403,7 @@ Do not edit this file manually.
400
403
  | `@types/spark-md5` | 3.0.5 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/spark-md5) |
401
404
  | `@types/sparqljs` | 3.1.12 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/sparqljs) |
402
405
  | `@types/triple-beam` | 1.3.5 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/triple-beam) |
406
+ | `@types/unist` | 3.0.3 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/unist) |
403
407
  | `@types/uritemplate` | 0.3.6 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/uritemplate) |
404
408
  | `@types/uuid` | 10.0.0 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/uuid) |
405
409
  | `@types/yargs` | 17.0.35 | MIT | [link](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/yargs) |
@@ -423,6 +427,7 @@ Do not edit this file manually.
423
427
  | `bytes` | 3.1.2 | MIT | [link](https://github.com/visionmedia/bytes.js#readme) |
424
428
  | `call-bind-apply-helpers` | 1.0.2 | MIT | [link](https://github.com/ljharb/call-bind-apply-helpers#readme) |
425
429
  | `call-bound` | 1.0.4 | MIT | [link](https://github.com/ljharb/call-bound#readme) |
430
+ | `character-entities` | 2.0.2 | MIT | [link](https://github.com/wooorm/character-entities#readme) |
426
431
  | `clownface` | 2.0.3 | MIT | [link](https://zazuko.github.io/clownface/#/) |
427
432
  | `color` | 5.0.3 | MIT | [link](https://github.com/Qix-/color#readme) |
428
433
  | `color-convert` | 2.0.1, 3.1.3 | MIT | [link](https://github.com/Qix-/color-convert#readme) |
@@ -438,7 +443,10 @@ Do not edit this file manually.
438
443
  | `cross-fetch` | 3.2.0 | MIT | [link](https://github.com/lquixada/cross-fetch) |
439
444
  | `cross-spawn` | 7.0.6 | MIT | [link](https://github.com/moxystudio/node-cross-spawn) |
440
445
  | `debug` | 4.4.3 | MIT | [link](https://github.com/debug-js/debug#readme) |
446
+ | `decode-named-character-reference` | 1.3.0 | MIT | [link](https://github.com/wooorm/decode-named-character-reference#readme) |
441
447
  | `depd` | 2.0.0 | MIT | [link](https://github.com/dougwilson/nodejs-depd#readme) |
448
+ | `dequal` | 2.0.3 | MIT | [link](https://github.com/lukeed/dequal#readme) |
449
+ | `devlop` | 1.1.0 | MIT | [link](https://github.com/wooorm/devlop#readme) |
442
450
  | `dom-serializer` | 2.0.0 | MIT | [link](https://github.com/cheeriojs/dom-serializer#readme) |
443
451
  | `dunder-proto` | 1.0.1 | MIT | [link](https://github.com/es-shims/dunder-proto#readme) |
444
452
  | `ee-first` | 1.1.1 | MIT | [link](https://github.com/jonathanong/ee-first#readme) |
@@ -495,9 +503,33 @@ Do not edit this file manually.
495
503
  | `kuler` | 2.0.0 | MIT | [link](https://github.com/3rd-Eden/kuler) |
496
504
  | `logform` | 2.7.0 | MIT | [link](https://github.com/winstonjs/logform#readme) |
497
505
  | `math-intrinsics` | 1.1.0 | MIT | [link](https://github.com/es-shims/math-intrinsics#readme) |
506
+ | `mdast-util-from-markdown` | 2.1.0 | MIT | [link](https://github.com/syntax-tree/mdast-util-from-markdown#readme) |
507
+ | `mdast-util-to-string` | 4.0.0 | MIT | [link](https://github.com/syntax-tree/mdast-util-to-string#readme) |
498
508
  | `media-typer` | 1.1.1 | MIT | [link](https://github.com/jshttp/media-typer#readme) |
499
509
  | `merge-descriptors` | 2.0.0 | MIT | [link](https://github.com/sindresorhus/merge-descriptors#readme) |
500
510
  | `microdata-rdf-streaming-parser` | 2.0.1, 3.0.0 | MIT | [link](https://github.com/rubensworks/microdata-rdf-streaming-parser.js#readme) |
511
+ | `micromark` | 4.0.3 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
512
+ | `micromark-core-commonmark` | 2.0.4 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
513
+ | `micromark-factory-destination` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
514
+ | `micromark-factory-label` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
515
+ | `micromark-factory-space` | 2.1.0 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
516
+ | `micromark-factory-title` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
517
+ | `micromark-factory-whitespace` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
518
+ | `micromark-util-character` | 2.1.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
519
+ | `micromark-util-chunked` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
520
+ | `micromark-util-classify-character` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
521
+ | `micromark-util-combine-extensions` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
522
+ | `micromark-util-decode-numeric-character-reference` | 2.0.2 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
523
+ | `micromark-util-decode-string` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
524
+ | `micromark-util-edit-map` | 1.0.0 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
525
+ | `micromark-util-encode` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
526
+ | `micromark-util-html-tag-name` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
527
+ | `micromark-util-normalize-identifier` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
528
+ | `micromark-util-resolve-all` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
529
+ | `micromark-util-sanitize-uri` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
530
+ | `micromark-util-subtokenize` | 2.1.0 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
531
+ | `micromark-util-symbol` | 2.0.1 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
532
+ | `micromark-util-types` | 2.0.3 | MIT | [link](https://github.com/micromark/micromark/tree/main#readme) |
501
533
  | `mime-db` | 1.54.0 | MIT | [link](https://github.com/jshttp/mime-db#readme) |
502
534
  | `mime-types` | 3.0.2 | MIT | [link](https://github.com/jshttp/mime-types#readme) |
503
535
  | `minimist` | 1.2.8 | MIT | [link](https://github.com/minimistjs/minimist) |
@@ -576,6 +608,7 @@ Do not edit this file manually.
576
608
  | `type-is` | 2.1.0 | MIT | [link](https://github.com/jshttp/type-is#readme) |
577
609
  | `undici` | 8.12.3 | MIT | [link](https://undici.nodejs.org) |
578
610
  | `undici-types` | 5.26.5, 6.21.0, 7.24.6 | MIT | [link](https://undici.nodejs.org) |
611
+ | `unist-util-stringify-position` | 4.0.0 | MIT | [link](https://github.com/syntax-tree/unist-util-stringify-position#readme) |
579
612
  | `unpipe` | 1.0.0 | MIT | [link](https://github.com/stream-utils/unpipe#readme) |
580
613
  | `uritemplate` | 0.3.4 | MIT | [link](https://www.github.com/fxa/uritemplate-js) |
581
614
  | `util-deprecate` | 1.0.2 | MIT | [link](https://github.com/TooTallNate/util-deprecate) |
@@ -1,9 +1,12 @@
1
+ import { scipSourcePatterns } from '../configuration/scip-project.js';
1
2
  import { contractPatternRoots } from "../repository/glob.js";
3
+ import { repositoryFileExtractorIdentity } from '../extraction/static.js';
2
4
  import { sha256, stableJson } from "../infrastructure/util.js";
3
- import { linkSemanticDocuments } from "../extraction/document.js";
5
+ import { documentLinkerIdentity, linkSemanticDocuments } from "../extraction/document.js";
4
6
  import { runtimeEvidenceFacts } from "../extraction/runtime-evidence.js";
5
7
  import { matchesRepositoryGlob } from "../repository/glob.js";
6
8
  import { RdfEvidenceGraph } from "../graph/rdf.js";
9
+ import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
7
10
  async function concurrentMap(values, concurrency, worker) {
8
11
  const results = new Array(values.length);
9
12
  let cursor = 0;
@@ -70,6 +73,8 @@ export class ContextGraphHydrator {
70
73
  ...config.codeRoots,
71
74
  ...config.evidenceRoots,
72
75
  ...contractPatternRoots(config.contractPatterns),
76
+ ...contractPatternRoots(scipSourcePatterns(config.scip.sourceProject)),
77
+ ...(config.routingSources ?? []),
73
78
  ]),
74
79
  ];
75
80
  const delta = await repository.delta(baseSha, commitSha, allRoots);
@@ -87,7 +92,9 @@ export class ContextGraphHydrator {
87
92
  : undefined;
88
93
  const configurationDigest = sha256(stableJson(config));
89
94
  const scopeDigest = sha256(stableJson(selectedSemanticSources)).slice(0, 12);
90
- const effectiveSuffix = ['graph-v2', configurationDigest.slice(0, 16), scopeDigest, baseSuffix, runtimeSuffix].filter(Boolean).join('-');
95
+ const effectiveSuffix = ['graph-v4', `links-${documentLinkerIdentity.version}`, `files-${repositoryFileExtractorIdentity.version}`,
96
+ ...(config.routingSources?.length ? [`routes-${routingExtractorIdentity.version}`] : []),
97
+ configurationDigest.slice(0, 16), scopeDigest, baseSuffix, runtimeSuffix].filter(Boolean).join('-');
91
98
  const snapshotId = effectiveSuffix ? `${commitSha}-${effectiveSuffix}` : commitSha;
92
99
  const context = {
93
100
  repositoryRoot: repository.root,
@@ -101,6 +108,8 @@ export class ContextGraphHydrator {
101
108
  renamedPaths: delta.renamedPaths,
102
109
  semanticPaths: selectedSemanticSources,
103
110
  };
111
+ // Validate committed routing inputs before invoking any provider or indexer.
112
+ const routing = await extractRoutingDeclarations(repository, context, config.routingSources ?? []);
104
113
  const sources = await repository.semanticSources(context, selectedSemanticSources);
105
114
  if (sources.length > (config.semantic.maxDocuments ?? 8) ||
106
115
  sources.some((source) => Buffer.byteLength(source.content) > (config.semantic.maxSourceBytes ?? 65_536))) {
@@ -177,13 +186,15 @@ export class ContextGraphHydrator {
177
186
  });
178
187
  }
179
188
  }
180
- const documentFacts = linkSemanticDocuments(context, hydrated, trackedPaths);
189
+ const documents = linkSemanticDocuments(context, hydrated, trackedPaths, sources.filter((source) => config.approvedIntentSources?.some((pattern) => matchesRepositoryGlob(source.path, pattern))).map((source) => source.path));
190
+ diagnostics.push(...documents.diagnostics);
191
+ diagnostics.push(...routing.diagnostics);
181
192
  options.signal?.throwIfAborted();
182
193
  options.onProgress?.({ phase: 'compile-semantic-rdf' });
183
194
  const ontology = await this.dependencies.ontologyCompiler.compile(snapshotId, artifacts, context);
184
195
  const runtimeEvidence = runtimeRevision.references;
185
196
  const runtimeFacts = runtimeEvidenceFacts(context, runtimeEvidence);
186
- const facts = [...semanticFacts, ...documentFacts, ...extractorFacts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
197
+ const facts = [...semanticFacts, ...documents.facts, ...extractorFacts, ...routing.facts, ...runtimeFacts].sort((left, right) => left.id.localeCompare(right.id));
187
198
  const invariants = (await Promise.all(this.dependencies.invariantEvaluators.map((evaluator) => evaluator.evaluate(snapshotId, facts)))).flat();
188
199
  const failures = invariants.filter((invariant) => invariant.status === 'fail');
189
200
  if (failures.length > 0) {
@@ -224,7 +235,9 @@ export class ContextGraphHydrator {
224
235
  scope: selectedSemanticSources,
225
236
  ontologyVersion: config.semantic.ontologyVersion,
226
237
  extractorVersions: {
227
- assembly: 'standalone-graph-v2',
238
+ assembly: 'standalone-graph-v4',
239
+ [documentLinkerIdentity.id]: documentLinkerIdentity.version,
240
+ ...(config.routingSources?.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}),
228
241
  semantic: config.semantic.extractorVersion,
229
242
  scip: scip.version,
230
243
  ontology: config.ontology.toolVersion,
@@ -2,15 +2,19 @@ import { execFile } from 'node:child_process';
2
2
  import { mkdir, rm, writeFile } from 'node:fs/promises';
3
3
  import { dirname, join, resolve } from 'node:path';
4
4
  import { promisify } from 'node:util';
5
+ import { scipSourcePatterns } from '../configuration/scip-project.js';
5
6
  import { GitRepository } from "../repository/git.js";
6
7
  import { withCommittedTree } from "../repository/committed-tree.js";
7
- import { RepositoryContractExtractor, RepositoryFileExtractor, ScipSymbolExtractor, ScipTypeScriptIndexer } from "../extraction/static.js";
8
+ import { RepositoryContractExtractor, RepositoryFileExtractor, ScipSymbolExtractor, ScipTypeScriptIndexer, repositoryFileExtractorIdentity } from "../extraction/static.js";
8
9
  import { sha256, stableJson } from "../infrastructure/util.js";
9
10
  import { contextPackEvidenceAuthority } from "../graph/query.js";
10
11
  import { inWorkingScope, readWorkingContent } from "../repository/working-content.js";
12
+ import { extractRoutingDeclarations, routingExtractorIdentity } from '../extraction/routing.js';
13
+ import { RdfEvidenceGraph } from '../graph/rdf.js';
11
14
  const execute = promisify(execFile);
12
15
  const implementationExtractors = new Set([
13
16
  'repository-file-extractor', 'repository-contract-extractor', 'scip-symbol-extractor',
17
+ routingExtractorIdentity.id,
14
18
  ]);
15
19
  /** Conservative code overlay: reindex implementation; invalidate changed prose, never synthesize intent. */
16
20
  export async function buildWorkingOverlay(repository, config, store, storageRoot, baseSnapshotId, agent = 'default') {
@@ -27,8 +31,14 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
27
31
  if (changed.length > 500)
28
32
  throw new Error('Overlay exceeds the 500 changed-path bound');
29
33
  const captured = [];
34
+ const routingSources = config.routingSources ?? [];
35
+ const baseRouting = await extractRoutingDeclarations(repository, {
36
+ repositoryRoot: repository.root, repositoryId: config.repositoryId, ref: head,
37
+ commitSha: head, snapshotId: baseSnapshotId, changedPaths: [], deletedPaths: [], renamedPaths: {},
38
+ }, routingSources, { allowMissingSources: true });
30
39
  const roots = [...config.codeRoots, ...config.semanticSources, ...config.evidenceRoots,
31
- ...config.contractPatterns];
40
+ ...config.contractPatterns, ...scipSourcePatterns(config.scip.sourceProject),
41
+ ...routingSources, ...baseRouting.referencedPaths];
32
42
  const excluded = [];
33
43
  for (const path of changed) {
34
44
  if (!inWorkingScope(path, roots)) {
@@ -43,9 +53,15 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
43
53
  sourcePatterns: roots, excludedPaths: excluded,
44
54
  };
45
55
  const overlayDigest = sha256(stableJson({ version: 2, ...overlayCapture }));
46
- const snapshotId = `${head}-overlay-${sha256(base.manifest.contentDigest + overlayDigest).slice(0, 32)}`;
56
+ // Capture identity describes the same working bytes across extractor upgrades. Publication
57
+ // identity also binds the extractor so rebuilding an old base cannot replace an old overlay.
58
+ const snapshotId = `${head}-overlay-${sha256(stableJson({
59
+ baseContentDigest: base.manifest.contentDigest, overlayDigest,
60
+ repositoryFileExtractor: repositoryFileExtractorIdentity,
61
+ ...(routingSources.length ? { routingExtractor: routingExtractorIdentity } : {}),
62
+ })).slice(0, 32)}`;
47
63
  const affected = new Set(captured.map((file) => file.path));
48
- const facts = await withCommittedTree(repository.root, head, async (root) => {
64
+ const extracted = await withCommittedTree(repository.root, head, async (root) => {
49
65
  for (const file of captured) {
50
66
  const target = join(root, file.path);
51
67
  if (file.content === null)
@@ -57,7 +73,9 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
57
73
  }
58
74
  const git = (...args) => execute('git', args, { cwd: root });
59
75
  await git('init', '--quiet');
60
- await git('add', '.');
76
+ // The archive already contains only committed files plus explicitly captured changes.
77
+ // Preserve tracked inputs even when their repository .gitignore also matches them.
78
+ await git('add', '--force', '.');
61
79
  await git('-c', 'user.name=Context Graph Draft', '-c', 'user.email=draft@example.invalid', '-c', 'commit.gpgsign=false', 'commit', '--quiet', '-m', 'Isolated overlay input');
62
80
  const draftRepository = new GitRepository(root);
63
81
  const context = {
@@ -68,28 +86,44 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
68
86
  renamedPaths: {},
69
87
  };
70
88
  const scip = await new ScipTypeScriptIndexer(config.scip, storageRoot).index(context);
89
+ const routing = await extractRoutingDeclarations(draftRepository, context, routingSources, { allowMissingSources: true });
90
+ const uncapturedEndpoints = routing.referencedPaths.filter((path) => !inWorkingScope(path, roots));
91
+ if (uncapturedEndpoints.length) {
92
+ throw new Error(`Draft routing endpoints are outside captured source scope; add them to configured source scope before rebuilding: ${uncapturedEndpoints.join(', ')}`);
93
+ }
71
94
  const extractors = [
72
95
  new RepositoryFileExtractor(draftRepository, config.codeRoots, scip.digest),
73
96
  new RepositoryContractExtractor(draftRepository, config.contractPatterns, config.contractConsumerRoots ?? config.codeRoots, config.contractAliases),
74
97
  new ScipSymbolExtractor(draftRepository, scip),
75
98
  ];
76
- return (await Promise.all(extractors.map((e) => e.extract(context)))).flat().map(({ id: _id, ...fact }) => {
99
+ const facts = [...(await Promise.all(extractors.map((e) => e.extract(context)))).flat(),
100
+ ...routing.facts].map(({ id: _id, ...fact }) => {
77
101
  const draft = { ...fact, snapshotId, authority: 'draft',
102
+ ...(fact.extractor.id === routingExtractorIdentity.id ? { sourceAuthority: 'draft' } : {}),
78
103
  provenance: { ...fact.provenance, commitSha: head } };
79
104
  return { ...draft, id: sha256(stableJson(draft)) };
80
105
  });
106
+ const diagnostics = routing.diagnostics.map(({ id: _id, ...item }) => {
107
+ const draft = { ...item, provenance: item.provenance.map((span) => ({ ...span, commitSha: head })) };
108
+ return { ...draft, id: sha256(stableJson(draft)) };
109
+ }).sort((a, b) => a.id.localeCompare(b.id));
110
+ return { facts, diagnostics };
81
111
  });
82
112
  const retained = base.facts.filter((fact) => !implementationExtractors.has(fact.extractor.id) &&
83
113
  contextPackEvidenceAuthority(fact) !== 'implemented' &&
84
114
  fact.extractor.id !== 'otlp-runtime-evidence' &&
85
115
  !affected.has(fact.provenance.path) &&
86
116
  ![fact.subject, fact.object].some((node) => [...affected].some((path) => node.id === `component:file:${path}` || node.id === `document:${path}` || node.label === path))).map((fact) => ({ ...fact, snapshotId }));
87
- const allFacts = [...retained, ...facts].sort((a, b) => a.id.localeCompare(b.id));
117
+ const allFacts = [...retained, ...extracted.facts].sort((a, b) => a.id.localeCompare(b.id));
88
118
  const overlay = {
89
119
  ...base,
90
120
  manifest: {
91
121
  ...base.manifest, snapshotId, baseSha: head, overlayDigest, overlayCapture,
92
- generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(allFacts)),
122
+ extractorVersions: { ...base.manifest.extractorVersions,
123
+ [repositoryFileExtractorIdentity.id]: repositoryFileExtractorIdentity.version,
124
+ ...(routingSources.length ? { [routingExtractorIdentity.id]: routingExtractorIdentity.version } : {}) },
125
+ generatedAt: new Date().toISOString(), contentDigest: sha256(stableJson(routingSources.length
126
+ ? { facts: allFacts, diagnostics: extracted.diagnostics } : allFacts)),
93
127
  },
94
128
  facts: allFacts,
95
129
  invariants: base.invariants.map((rule) => ({
@@ -100,8 +134,11 @@ export async function buildWorkingOverlay(repository, config, store, storageRoot
100
134
  'Draft implementation overlay: changed semantic documents require explicit re-extraction.',
101
135
  `Changes outside configured source scope are excluded: ${excluded.join(', ') || '(none)'}`,
102
136
  ],
103
- diagnostics: [],
137
+ diagnostics: extracted.diagnostics,
104
138
  };
139
+ if (routingSources.length && !(await new RdfEvidenceGraph(overlay).validate()).conforms) {
140
+ throw new Error('Draft routing overlay failed SHACL validation; saved evidence was preserved');
141
+ }
105
142
  if (await repository.resolveRef('HEAD') !== head)
106
143
  throw new Error('HEAD changed during overlay capture; retry');
107
144
  await store.write(overlay);
@@ -27,6 +27,7 @@ export declare class SavedEvidenceQueryService implements ImpactQueryService {
27
27
  coverage: import("@weftlet/core").ContextPackCoverage;
28
28
  diagnostics: import("@weftlet/core").DiagnosticReport;
29
29
  exploration?: {
30
+ readonly rankingVersion?: string;
30
31
  readonly returned: number;
31
32
  readonly reachableFacts: number;
32
33
  readonly maxDepth: number;
@@ -1,3 +1,4 @@
1
+ import { type ScipSourceProject } from './scip-project.js';
1
2
  export interface ContextGraphConfig {
2
3
  readonly schemaVersion: 1;
3
4
  readonly repositoryRoot?: string;
@@ -12,6 +13,8 @@ export interface ContextGraphConfig {
12
13
  readonly contractConsumerRoots?: readonly string[];
13
14
  readonly contractAliases?: Readonly<Record<string, readonly string[]>>;
14
15
  readonly evidenceRoots: readonly string[];
16
+ /** Exact committed JSON routing declarations; opt-in, not approved intent or runtime evidence. */
17
+ readonly routingSources?: readonly string[];
15
18
  readonly semantic: {
16
19
  readonly provider: 'codex-cli' | 'module';
17
20
  /** Codex authentication only; secrets stay in the process environment. */
@@ -41,6 +44,7 @@ export interface ContextGraphConfig {
41
44
  readonly version: string;
42
45
  readonly command: string;
43
46
  readonly args: readonly string[];
47
+ readonly sourceProject?: ScipSourceProject;
44
48
  };
45
49
  readonly ontology: {
46
50
  readonly backend?: 'rdf' | 'opencaesar';
@@ -2,6 +2,7 @@
2
2
  import { existsSync } from 'node:fs';
3
3
  import { readFile } from 'node:fs/promises';
4
4
  import { dirname, resolve } from 'node:path';
5
+ import { assertScipSourceProject, assertSourceProjectInvocation } from './scip-project.js';
5
6
  function isRecord(value) {
6
7
  return typeof value === 'object' && value !== null && !Array.isArray(value);
7
8
  }
@@ -33,6 +34,13 @@ function validateConfig(value, absolutePath) {
33
34
  if (value.approvedIntentSources !== undefined) {
34
35
  requireConfig(isStringArray(value.approvedIntentSources), 'approvedIntentSources must be an array of strings', absolutePath);
35
36
  }
37
+ if (value.routingSources !== undefined) {
38
+ requireConfig(isStringArray(value.routingSources) && value.routingSources.length <= 16 &&
39
+ new Set(value.routingSources).size === value.routingSources.length &&
40
+ value.routingSources.every((path) => path.endsWith('.json') && path === path.trim() && path.length <= 1024 &&
41
+ !/[\\:*?#[\]{}%\x00-\x1f\x7f-\x9f]/.test(path) &&
42
+ path.split('/').every((part) => part.length > 0 && part !== '.' && part !== '..')), 'routingSources must contain at most 16 unique exact repository-relative .json paths', absolutePath);
43
+ }
36
44
  for (const field of [
37
45
  'semanticSources',
38
46
  'codeRoots',
@@ -131,6 +139,10 @@ function validateConfig(value, absolutePath) {
131
139
  requireConfig(isNonEmptyString(value.scip.version), 'scip.version is required', absolutePath);
132
140
  requireConfig(isNonEmptyString(value.scip.command), 'scip.command is required', absolutePath);
133
141
  requireConfig(isStringArray(value.scip.args), 'scip.args must be an array of strings', absolutePath);
142
+ if (value.scip.sourceProject !== undefined) {
143
+ assertScipSourceProject(value.scip.sourceProject);
144
+ assertSourceProjectInvocation(value.scip.command, value.scip.args);
145
+ }
134
146
  }
135
147
  requireConfig(isRecord(value.ontology), 'ontology must be an object', absolutePath);
136
148
  if (isRecord(value.ontology)) {
@@ -0,0 +1,9 @@
1
+ /** Explicit source-only indexing, independent of a consumer's build tsconfig. */
2
+ export interface ScipSourceProject {
3
+ readonly include: readonly string[];
4
+ readonly aliases?: Readonly<Record<string, string>>;
5
+ }
6
+ /** Also used by direct indexer callers, which need not pass through loadConfig. */
7
+ export declare function assertScipSourceProject(value: unknown): asserts value is ScipSourceProject;
8
+ export declare function assertSourceProjectInvocation(command: string, args: readonly string[]): void;
9
+ export declare function scipSourcePatterns(project: ScipSourceProject | undefined): readonly string[];
@@ -0,0 +1,40 @@
1
+ function repositoryPath(value) {
2
+ return typeof value === 'string' && value.length > 0 && value.length <= 512 &&
3
+ !/[\\:\x00-\x1f]/.test(value) &&
4
+ value.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..');
5
+ }
6
+ /** Also used by direct indexer callers, which need not pass through loadConfig. */
7
+ export function assertScipSourceProject(value) {
8
+ const error = (reason) => { throw new Error(`Invalid scip.sourceProject: ${reason}`); };
9
+ if (!value || typeof value !== 'object' || Array.isArray(value))
10
+ error('expected an object');
11
+ const record = value;
12
+ if (Object.keys(record).some((key) => !['include', 'aliases'].includes(key)))
13
+ error('unknown field');
14
+ if (!Array.isArray(record.include) || record.include.length === 0 || record.include.length > 100 ||
15
+ !record.include.every(repositoryPath))
16
+ error('include requires 1–100 repository-relative paths/globs');
17
+ if (record.aliases !== undefined) {
18
+ if (!record.aliases || typeof record.aliases !== 'object' || Array.isArray(record.aliases))
19
+ error('aliases must be an object');
20
+ const entries = Object.entries(record.aliases);
21
+ if (entries.length > 100)
22
+ error('at most 100 exact aliases are supported');
23
+ for (const [name, path] of entries) {
24
+ if (!/^(?:@[\w.-]+\/)?[\w.-]+(?:\/[\w.-]+)*$/.test(name) ||
25
+ name.split('/').some((segment) => segment === '.' || segment === '..') ||
26
+ !repositoryPath(path) || /[*?]/.test(path) || !/\.(?:[cm]?ts|tsx)$/.test(path)) {
27
+ error(`alias ${name} must map an exact module name to a repository TypeScript file`);
28
+ }
29
+ }
30
+ }
31
+ }
32
+ export function assertSourceProjectInvocation(command, args) {
33
+ if (command !== 'bundled-scip-typescript' || args.length !== 2 ||
34
+ args[0] !== 'index' || args[1] !== '--no-global-caches') {
35
+ throw new Error('scip.sourceProject requires bundled-scip-typescript and args ["index", "--no-global-caches"]');
36
+ }
37
+ }
38
+ export function scipSourcePatterns(project) {
39
+ return project ? [...project.include, ...Object.values(project.aliases ?? {})] : [];
40
+ }
@@ -1,6 +1,7 @@
1
1
  // Environment allowlist adapted from the extracted semantic.ts; see docs/extraction-manifest.json.
2
2
  import { execFile } from 'node:child_process';
3
3
  import { promisify } from 'node:util';
4
+ import { resolveCodexInvocation } from './codex-command.js';
4
5
  import { SemanticProviderError } from './semantic-process.js';
5
6
  const semanticEnvironmentAllowlist = [
6
7
  'APPDATA',
@@ -35,8 +36,14 @@ export function semanticCommandEnvironment(source) {
35
36
  /** Read-only status check. Never expose output: API login status may contain key fragments. */
36
37
  export const readCodexLoginMode = async (command, environment, options = {}) => {
37
38
  try {
38
- const result = await promisify(execFile)(command, ['login', 'status'], {
39
- env: environment, timeout: Math.min(options.timeoutMs ?? 5000, 5000),
39
+ const started = performance.now();
40
+ const timeoutMs = Math.min(options.timeoutMs ?? 5000, 5000);
41
+ const invocation = await resolveCodexInvocation(command, environment, process.platform, { ...options, timeoutMs });
42
+ const remaining = Math.ceil(timeoutMs - (performance.now() - started));
43
+ if (remaining <= 0)
44
+ return 'unavailable';
45
+ const result = await promisify(execFile)(invocation.command, [...invocation.prefixArgs, 'login', 'status'], {
46
+ env: environment, timeout: remaining,
40
47
  signal: options.signal, maxBuffer: 8192, windowsHide: true,
41
48
  });
42
49
  const output = `${result.stdout}\n${result.stderr}`;
@@ -0,0 +1,8 @@
1
+ import { type SemanticRunOptions } from './semantic-process.js';
2
+ interface CodexInvocation {
3
+ readonly command: string;
4
+ readonly prefixArgs: readonly string[];
5
+ }
6
+ /** Bound discovery too: a disconnected drive in PATH must not leave a provider call waiting indefinitely. */
7
+ export declare function resolveCodexInvocation(command: string, environment: Readonly<Record<string, string | undefined>>, platform?: NodeJS.Platform, options?: SemanticRunOptions): Promise<CodexInvocation>;
8
+ export {};
@@ -0,0 +1,81 @@
1
+ import { readFile, stat } from 'node:fs/promises';
2
+ import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
3
+ import { SemanticProviderError } from './semantic-process.js';
4
+ async function isFile(path) {
5
+ try {
6
+ return (await stat(path)).isFile();
7
+ }
8
+ catch (error) {
9
+ if (['ENOENT', 'ENOTDIR'].includes(error.code ?? ''))
10
+ return false;
11
+ throw new SemanticProviderError('unavailable', 'Cannot inspect the Codex installation on PATH');
12
+ }
13
+ }
14
+ async function npmEntry(directory, shim) {
15
+ const packageRoot = basename(directory).toLowerCase() === '.bin'
16
+ ? join(dirname(directory), '@openai', 'codex')
17
+ : join(directory, 'node_modules', '@openai', 'codex');
18
+ try {
19
+ const pkg = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8'));
20
+ const bin = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.codex;
21
+ if (pkg.name !== '@openai/codex' || typeof bin !== 'string' || !/\.(?:c|m)?js$/i.test(bin))
22
+ return;
23
+ const entry = resolve(packageRoot, bin);
24
+ const within = relative(packageRoot, entry);
25
+ if (within.startsWith('..') || isAbsolute(within) || !await isFile(entry))
26
+ return;
27
+ // Recognize standard npm cmd-shim targets; never execute or evaluate the batch file.
28
+ const target = relative(directory, entry).replaceAll('\\', '/').toLowerCase();
29
+ const launcher = (await readFile(shim, 'utf8')).replaceAll('\\', '/').toLowerCase();
30
+ if (launcher.includes(`%dp0%/${target}"`) || launcher.includes(`%~dp0${target}"`))
31
+ return entry;
32
+ }
33
+ catch { /* Missing/invalid metadata or an unsupported launcher is reported without its contents. */ }
34
+ }
35
+ /** Native binaries and standard npm installations; no cmd.exe, PowerShell, or implicit cwd lookup. */
36
+ async function findWindowsCodex(environment) {
37
+ const pathKey = Object.keys(environment).find(key => key.toUpperCase() === 'PATH');
38
+ for (const item of (pathKey ? environment[pathKey] ?? '' : '').split(';')) {
39
+ const path = item.replace(/^"(.*)"$/, '$1');
40
+ if (!path)
41
+ continue;
42
+ const directory = resolve(path);
43
+ const native = join(directory, 'codex.exe');
44
+ if (await isFile(native))
45
+ return { command: native, prefixArgs: [] };
46
+ const shim = join(directory, 'codex.cmd');
47
+ if (!await isFile(shim))
48
+ continue;
49
+ const entry = await npmEntry(directory, shim);
50
+ if (entry)
51
+ return { command: process.execPath, prefixArgs: [entry] };
52
+ throw new SemanticProviderError('unavailable', 'Unsupported Codex launcher on PATH. Use a standard npm @openai/codex installation or a native codex.exe; batch wrappers are not executed');
53
+ }
54
+ throw new SemanticProviderError('unavailable', 'Codex CLI was not found on PATH; install @openai/codex with npm or add a native codex.exe');
55
+ }
56
+ /** Bound discovery too: a disconnected drive in PATH must not leave a provider call waiting indefinitely. */
57
+ export async function resolveCodexInvocation(command, environment, platform = process.platform, options = {}) {
58
+ if (options.signal?.aborted)
59
+ throw new SemanticProviderError('cancelled', 'Codex executable discovery cancelled');
60
+ if (platform !== 'win32' || command !== 'codex')
61
+ return { command, prefixArgs: [] };
62
+ let timer;
63
+ let cancel = () => { };
64
+ try {
65
+ return await Promise.race([
66
+ findWindowsCodex(environment),
67
+ new Promise((_resolve, reject) => {
68
+ timer = setTimeout(() => reject(new SemanticProviderError('timeout', 'Codex executable discovery timed out')), Math.min(options.timeoutMs ?? 5000, 5000));
69
+ cancel = () => reject(new SemanticProviderError('cancelled', 'Codex executable discovery cancelled'));
70
+ options.signal?.addEventListener('abort', cancel, { once: true });
71
+ if (options.signal?.aborted)
72
+ cancel();
73
+ }),
74
+ ]);
75
+ }
76
+ finally {
77
+ if (timer)
78
+ clearTimeout(timer);
79
+ options.signal?.removeEventListener('abort', cancel);
80
+ }
81
+ }
@@ -0,0 +1,18 @@
1
+ export interface DocumentReference {
2
+ readonly target: string;
3
+ readonly startLine: number;
4
+ readonly endLine: number;
5
+ readonly relativeTo: 'document' | 'repository';
6
+ }
7
+ export declare function documentReferences(content: string): readonly DocumentReference[];
8
+ export type ReferenceResolution = {
9
+ readonly kind: 'resolved';
10
+ readonly path: string;
11
+ } | {
12
+ readonly kind: 'ignored';
13
+ } | {
14
+ readonly kind: 'missing';
15
+ readonly path?: string;
16
+ readonly reason: string;
17
+ };
18
+ export declare function resolveDocumentReference(sourcePath: string, reference: DocumentReference, tracked: ReadonlySet<string>): ReferenceResolution;