@aiwg/cli 2026.7.19 → 2026.7.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/README.md +397 -385
  2. package/dist/src/api/index.d.ts +1 -0
  3. package/dist/src/api/index.js +1 -0
  4. package/dist/src/artifacts/browser-export.js +7 -0
  5. package/dist/src/artifacts/citation-parser.js +96 -35
  6. package/dist/src/artifacts/cli.js +59 -5
  7. package/dist/src/artifacts/discover-facets.js +15 -0
  8. package/dist/src/artifacts/discovery-eval.js +290 -0
  9. package/dist/src/artifacts/fortemi-core-query-adapter.js +1 -1
  10. package/dist/src/artifacts/fortemi-shard-export.js +1 -1
  11. package/dist/src/artifacts/index-builder.js +54 -17
  12. package/dist/src/artifacts/query-engine.js +10 -6
  13. package/dist/src/artifacts/state-transfer.js +27 -0
  14. package/dist/src/artifacts/stats.js +8 -0
  15. package/dist/src/cli/cli-extension-loader.js +73 -0
  16. package/dist/src/cli/handlers/help.js +2 -1
  17. package/dist/src/cli/handlers/index.js +6 -2
  18. package/dist/src/cli/handlers/resource-versions.js +247 -0
  19. package/dist/src/cli/handlers/sessions.js +966 -0
  20. package/dist/src/cli/handlers/skill-lint.js +49 -45
  21. package/dist/src/cli/handlers/subcommands.js +55 -3
  22. package/dist/src/cli/handlers/use.js +154 -59
  23. package/dist/src/cli/handlers/utilities.js +49 -34
  24. package/dist/src/cli/skill-usage.js +146 -24
  25. package/dist/src/config/cli.js +13 -9
  26. package/dist/src/config/project-artifacts-runtime.mjs +68 -0
  27. package/dist/src/config/project-artifacts.js +1 -68
  28. package/dist/src/extensions/commands/definitions.js +65 -2
  29. package/dist/src/extensions/manifest.js +29 -0
  30. package/dist/src/extensions/project-local-discovery.js +86 -2
  31. package/dist/src/extensions/project-local-remove.js +52 -56
  32. package/dist/src/extensions/shadow-resolver.js +3 -1
  33. package/dist/src/plugins/standalone-packager.js +143 -0
  34. package/dist/src/resources/cache-cleanup.js +67 -0
  35. package/dist/src/resources/doctor.js +107 -0
  36. package/dist/src/resources/lockfile.js +125 -0
  37. package/dist/src/resources/resolver.js +133 -0
  38. package/dist/src/resources/web-release.d.ts +8 -0
  39. package/dist/src/resources/web-release.js +159 -1
  40. package/dist/src/sessions/adapters/claude.js +357 -0
  41. package/dist/src/sessions/adapters/codex.js +521 -0
  42. package/dist/src/sessions/adapters/copilot.js +226 -0
  43. package/dist/src/sessions/adapters/cursor.js +372 -0
  44. package/dist/src/sessions/adapters/factory.js +345 -0
  45. package/dist/src/sessions/adapters/generic.js +225 -0
  46. package/dist/src/sessions/adapters/hermes.js +341 -0
  47. package/dist/src/sessions/adapters/openclaw.js +381 -0
  48. package/dist/src/sessions/adapters/opencode.js +454 -0
  49. package/dist/src/sessions/adapters/openhuman.js +315 -0
  50. package/dist/src/sessions/adapters/warp.js +160 -0
  51. package/dist/src/sessions/adapters/windsurf.js +212 -0
  52. package/dist/src/sessions/candidates.js +210 -0
  53. package/dist/src/sessions/contracts.js +310 -0
  54. package/dist/src/sessions/discovery.js +51 -0
  55. package/dist/src/sessions/fixtures.js +12 -0
  56. package/dist/src/sessions/importer.js +315 -0
  57. package/dist/src/sessions/index.js +25 -0
  58. package/dist/src/sessions/knowledge-shard.js +61 -0
  59. package/dist/src/sessions/optional-backends.js +238 -0
  60. package/dist/src/sessions/policy.js +192 -0
  61. package/dist/src/sessions/ports.js +2 -0
  62. package/dist/src/sessions/promotion.js +367 -0
  63. package/dist/src/sessions/readers.js +176 -0
  64. package/dist/src/sessions/repository.js +1551 -0
  65. package/dist/src/skills/adapters/agent-skills.js +59 -0
  66. package/dist/src/skills/adapters/local.js +19 -1
  67. package/dist/src/skills/agent-skills.js +249 -0
  68. package/dist/src/skills/cli.js +463 -7
  69. package/dist/src/skills/deployer.js +554 -0
  70. package/dist/src/skills/doctor.js +105 -0
  71. package/dist/src/skills/exporter.js +382 -0
  72. package/dist/src/skills/importer.js +921 -0
  73. package/dist/src/skills/registry.js +19 -0
  74. package/dist/src/skills/validator.js +323 -0
  75. package/dist/src/smiths/context-pipeline/aiwg-md.js +5 -1
  76. package/dist/src/smiths/context-pipeline/claude-hook.js +21 -1
  77. package/dist/src/smiths/context-pipeline/finalization.js +5 -3
  78. package/dist/src/smiths/context-pipeline/generator.js +4 -1
  79. package/dist/src/smiths/context-pipeline/parallelism-section.js +34 -1
  80. package/dist/src/smiths/context-pipeline/workspace-context.js +15 -3
  81. package/dist/src/smiths/mcpsmith/example.js +3 -1
  82. package/dist/src/smiths/mcpsmith/generator.js +3 -1
  83. package/dist/src/smiths/toolsmith/runtime-discovery.mjs +2 -1
  84. package/dist/src/storage/cli.js +3 -2
  85. package/dist/src/storage/subsystem-cli.js +7 -2
  86. package/dist/src/update/notifier.mjs +1 -1
  87. package/dist/src/update/service.mjs +123 -0
  88. package/package.json +3 -2
@@ -7,4 +7,5 @@
7
7
  */
8
8
  export { run } from '../cli/router.js';
9
9
  export * from '../resources/index.js';
10
+ export * from '../sessions/index.js';
10
11
  //# sourceMappingURL=index.d.ts.map
@@ -7,4 +7,5 @@
7
7
  */
8
8
  export { run } from '../cli/router.js';
9
9
  export * from '../resources/index.js';
10
+ export * from '../sessions/index.js';
10
11
  //# sourceMappingURL=index.js.map
@@ -508,6 +508,13 @@ function recordForEntry(cwd, entry, graphName, dependencyGraph, privacy, schemaV
508
508
  ...(entry.operationalState
509
509
  ? { operational_state: entry.operationalState }
510
510
  : {}),
511
+ ...(entry.stateTransfer
512
+ ? {
513
+ state_transfer: {
514
+ deleted_at: entry.stateTransfer.deletedAt,
515
+ },
516
+ }
517
+ : {}),
511
518
  }
512
519
  : {}),
513
520
  updated_at: entry.updated,
@@ -8,7 +8,7 @@
8
8
  * - **Incoming**: corpus papers that cite this work (column: "REF") → `cited-by` edges
9
9
  *
10
10
  * Supported node-id forms (#105):
11
- * - `REF-\d+` research-paper IDs (REF-001, REF-029, ...)
11
+ * - `REF-\d+[a-z]?` research-paper IDs (REF-001, REF-434a, ...)
12
12
  * - `PROF-[POFG]-[a-z0-9-]+` entity-profile IDs:
13
13
  * - `PROF-P-*` people, `PROF-O-*` orgs, `PROF-F-*` funders, `PROF-G-*` groups
14
14
  *
@@ -26,17 +26,43 @@ import { parseFrontmatter } from './index-builder.js';
26
26
  * Match a single node identifier (REF-* or PROF-*) anywhere in a string.
27
27
  * Used by `extractRefsFromTable` to pull every ID out of a table cell.
28
28
  */
29
- const NODE_ID_PATTERN = /(?:REF-\d+|PROF-[POFG]-[a-z0-9-]+)/g;
29
+ const NODE_ID_PATTERN = /(?:REF-\d+[a-z]?|PROF-[POFG]-[a-z0-9-]+)/g;
30
30
  /**
31
31
  * Validate that a string is a complete node identifier.
32
32
  * Used by `parseCitationSidecar` and `buildRefToPathMap` to gate
33
33
  * frontmatter `ref` values.
34
34
  */
35
- const NODE_ID_FULL = /^(?:REF-\d+|PROF-[POFG]-[a-z0-9-]+)$/;
35
+ const NODE_ID_FULL = /^(?:REF-\d+[a-z]?|PROF-[POFG]-[a-z0-9-]+)$/;
36
+ /** Explicit column aliases used by historical citation-sidecar variants. */
37
+ const CITATION_REF_COLUMNS = [
38
+ 'Inducted REF',
39
+ 'REF',
40
+ 'Corpus REF',
41
+ 'In-corpus REF',
42
+ ];
43
+ /**
44
+ * Preserve the corpus snapshot's legacy edge semantics: every node ID on a
45
+ * pipe-delimited line inside an outgoing/incoming section is a declaration.
46
+ * Explicit column parsing remains the primary path, while this compatibility
47
+ * scan covers malformed rows and prose continuations that historical corpus
48
+ * snapshots already count.
49
+ */
50
+ function extractSectionNodeIds(section) {
51
+ const refs = [];
52
+ for (const line of section.split('\n')) {
53
+ if (!line.includes('|'))
54
+ continue;
55
+ NODE_ID_PATTERN.lastIndex = 0;
56
+ const matches = line.match(NODE_ID_PATTERN);
57
+ if (matches)
58
+ refs.push(...matches);
59
+ }
60
+ return [...new Set(refs)];
61
+ }
36
62
  /**
37
63
  * Test whether a string is a valid sidecar node identifier.
38
64
  *
39
- * Accepts `REF-\d+` and `PROF-[POFG]-[a-z0-9-]+`. Returns false for any
65
+ * Accepts `REF-\d+[a-z]?` and `PROF-[POFG]-[a-z0-9-]+`. Returns false for any
40
66
  * other input (including unrelated `PROF-` prefixed strings that don't
41
67
  * match the four-letter type code form).
42
68
  */
@@ -54,34 +80,71 @@ export function isNodeId(value) {
54
80
  * @returns Array of node identifiers found
55
81
  */
56
82
  export function extractRefsFromTable(tableText, columnName) {
57
- const lines = tableText.split('\n').filter(l => l.trim().startsWith('|'));
58
- if (lines.length < 3)
59
- return []; // Need header + separator + at least one row
60
- // Parse header to find column index
61
- const headerCells = lines[0].split('|').map(c => c.trim()).filter(Boolean);
62
- const colIndex = headerCells.findIndex(h => h.toLowerCase() === columnName.toLowerCase());
63
- if (colIndex === -1)
64
- return [];
65
- // Skip header (line 0) and separator (line 1), parse data rows
83
+ const aliases = (Array.isArray(columnName) ? columnName : [columnName])
84
+ .map(name => name.trim().toLowerCase());
85
+ const tableBlocks = [];
86
+ let currentBlock = [];
87
+ for (const line of tableText.split('\n')) {
88
+ if (line.trim().startsWith('|')) {
89
+ currentBlock.push(line);
90
+ }
91
+ else if (currentBlock.length > 0) {
92
+ tableBlocks.push(currentBlock);
93
+ currentBlock = [];
94
+ }
95
+ }
96
+ if (currentBlock.length > 0)
97
+ tableBlocks.push(currentBlock);
98
+ const parseRow = (line) => {
99
+ let row = line.trim();
100
+ if (row.startsWith('|'))
101
+ row = row.slice(1);
102
+ if (row.endsWith('|'))
103
+ row = row.slice(0, -1);
104
+ return row.split('|').map(cell => cell.trim());
105
+ };
66
106
  const refs = [];
67
- for (let i = 2; i < lines.length; i++) {
68
- const cells = lines[i].split('|').map(c => c.trim()).filter(Boolean);
69
- if (colIndex >= cells.length)
107
+ for (const lines of tableBlocks) {
108
+ const headerCells = parseRow(lines[0]);
109
+ const colIndex = headerCells.findIndex(header => {
110
+ const normalized = header.toLowerCase().replace(/[`*_]/g, '').trim();
111
+ return aliases.some(alias => normalized === alias ||
112
+ normalized.startsWith(`${alias} /`) ||
113
+ normalized.startsWith(`${alias} (`));
114
+ });
115
+ if (lines.length < 3 || colIndex === -1) {
116
+ // Legacy sidecars sometimes have headerless continuation rows or a
117
+ // generic table whose rows were extended with a final REF cell. The
118
+ // corpus snapshot has always treated every in-section table REF as an
119
+ // outgoing/incoming declaration, so retain that compatibility fallback.
120
+ const dataLines = lines.length >= 3 ? lines.slice(2) : lines;
121
+ for (const line of dataLines) {
122
+ NODE_ID_PATTERN.lastIndex = 0;
123
+ const refMatches = line.match(NODE_ID_PATTERN);
124
+ if (refMatches)
125
+ refs.push(...refMatches);
126
+ }
70
127
  continue;
71
- const value = cells[colIndex].trim();
72
- // Skip empty, dash, or em-dash values
73
- if (!value || value === '—' || value === '-' || value === '–')
74
- continue;
75
- // Extract node-id pattern(s) (REF-* or PROF-*) from the cell.
76
- // Reset the lastIndex defensively — NODE_ID_PATTERN is a module-level
77
- // /g RegExp shared across calls.
78
- NODE_ID_PATTERN.lastIndex = 0;
79
- const refMatches = value.match(NODE_ID_PATTERN);
80
- if (refMatches) {
81
- refs.push(...refMatches);
128
+ }
129
+ // Skip this table's header and separator, preserving all interior cells.
130
+ for (let i = 2; i < lines.length; i++) {
131
+ const cells = parseRow(lines[i]);
132
+ if (colIndex >= cells.length)
133
+ continue;
134
+ // Historical rows may expand the named REF cell into an issue/REF pair
135
+ // or append placeholder cells without extending the header. Scan from
136
+ // the named column through the remainder of the row; columns before the
137
+ // alias (title/authors/year) remain excluded.
138
+ const value = cells.slice(colIndex).join(' | ');
139
+ if (!value || value === '—' || value === '-' || value === '–')
140
+ continue;
141
+ NODE_ID_PATTERN.lastIndex = 0;
142
+ const refMatches = value.match(NODE_ID_PATTERN);
143
+ if (refMatches)
144
+ refs.push(...refMatches);
82
145
  }
83
146
  }
84
- return refs;
147
+ return [...new Set(refs)];
85
148
  }
86
149
  /**
87
150
  * Parse a citation sidecar markdown file into structured edges.
@@ -102,17 +165,15 @@ export function parseCitationSidecar(content) {
102
165
  for (const section of sections) {
103
166
  const sectionLower = section.toLowerCase();
104
167
  if (sectionLower.startsWith('outgoing')) {
105
- // Outgoing table: extract from "Inducted REF" column
106
- cites = extractRefsFromTable(section, 'Inducted REF');
168
+ cites.push(...extractRefsFromTable(section, CITATION_REF_COLUMNS));
169
+ cites.push(...extractSectionNodeIds(section));
107
170
  }
108
171
  else if (sectionLower.startsWith('incoming')) {
109
- // Incoming table: extract from "REF" column
110
- // The incoming section may have subsections (### Corpus Cross-References)
111
- // Look for tables anywhere in this section
112
- citedBy = extractRefsFromTable(section, 'REF');
172
+ citedBy.push(...extractRefsFromTable(section, CITATION_REF_COLUMNS));
173
+ citedBy.push(...extractSectionNodeIds(section));
113
174
  }
114
175
  }
115
- return { ref, cites, citedBy };
176
+ return { ref, cites: [...new Set(cites)], citedBy: [...new Set(citedBy)] };
116
177
  }
117
178
  /**
118
179
  * Convert a CitationParseResult into TypedEdge arrays for the dependency graph.
@@ -134,7 +134,7 @@ function parseAiwgVersionFlag(args) {
134
134
  return undefined;
135
135
  const value = args[indices[0] + 1];
136
136
  if (!value || value.startsWith('--')) {
137
- console.error('Error: --aiwg-version requires an exact calendar-semver version or channel name');
137
+ console.error('Error: --aiwg-version requires an exact calendar-semver version, SemVer range, sha256 digest, or channel name');
138
138
  process.exit(1);
139
139
  }
140
140
  try {
@@ -257,6 +257,9 @@ export async function main(args) {
257
257
  case 'dedup-report':
258
258
  await handleDedup(subcommandArgs);
259
259
  break;
260
+ case 'eval-discovery':
261
+ await handleEvalDiscovery(subcommandArgs);
262
+ break;
260
263
  case 'watch':
261
264
  await handleWatch(subcommandArgs);
262
265
  break;
@@ -283,7 +286,7 @@ export async function main(args) {
283
286
  break;
284
287
  default:
285
288
  console.error(`Error: Unknown index subcommand '${subcommand}'`);
286
- console.log('Available: build, query, discover, show, export, sync, migrate-legacy, deps, stats, status, list, neighbors, set, embed, similar, dedup-report, watch');
289
+ console.log('Available: build, query, discover, show, export, sync, migrate-legacy, deps, stats, status, list, neighbors, set, embed, similar, dedup-report, eval-discovery, watch');
287
290
  process.exit(1);
288
291
  }
289
292
  }
@@ -306,6 +309,7 @@ function printIndexUsage() {
306
309
  console.log(' embed Build the semantic embedding index for a graph (opt-in deps)');
307
310
  console.log(' similar Semantic neighbors of a node (requires embed)');
308
311
  console.log(' dedup-report Near-duplicate node pairs above a similarity threshold');
312
+ console.log(' eval-discovery Benchmark operational capability discovery relevance');
309
313
  console.log(' watch Start a filesystem watcher for automatic incremental index updates');
310
314
  console.log('');
311
315
  console.log('Options:');
@@ -332,9 +336,59 @@ function printIndexUsage() {
332
336
  console.log(' aiwg index deps .aiwg/requirements/UC-001.md');
333
337
  console.log(' aiwg index stats --json');
334
338
  console.log(' aiwg index stats --graph project');
339
+ console.log(' aiwg index eval-discovery --queries test/fixtures/artifacts/discovery-relevance.jsonl --backend local --strategy lexical');
335
340
  console.log(' aiwg index neighbors --graph citation-network --node REF-008 --direction in --edge-type cites');
336
341
  console.log(' aiwg index set --graph citation-network --op intersection --node-a REF-008 --node-b REF-016 --direction in');
337
342
  }
343
+ async function handleEvalDiscovery(args) {
344
+ if (args.includes('--help') || args.includes('-h')) {
345
+ console.log('Usage: aiwg index eval-discovery --queries <jsonl> --backend <local|fortemi-core> --strategy <lexical|dense|hybrid-rrf|rerank|chunk-multivector> [options]');
346
+ console.log('');
347
+ console.log('Options:');
348
+ console.log(' --queries <path> Versioned relevance JSONL fixture (required)');
349
+ console.log(' --backend <name> local or fortemi-core (required)');
350
+ console.log(' --strategy <name> lexical, dense, hybrid-rrf, rerank, or chunk-multivector');
351
+ console.log(' --out <path> Write the JSON report to a file');
352
+ console.log(' --json Emit JSON instead of the readable summary');
353
+ return;
354
+ }
355
+ const queries = parseFlagValue(args, '--queries', 'Error: --queries requires a JSONL path');
356
+ if (!queries) {
357
+ console.error('Error: --queries is required');
358
+ process.exit(1);
359
+ }
360
+ const backend = parseFlagValue(args, '--backend', 'Error: --backend requires local or fortemi-core');
361
+ if (backend !== 'local' && backend !== 'fortemi-core') {
362
+ console.error('Error: --backend must be local or fortemi-core');
363
+ process.exit(1);
364
+ }
365
+ const strategy = parseFlagValue(args, '--strategy', 'Error: --strategy requires a value') ?? 'lexical';
366
+ const { DISCOVERY_EVAL_STRATEGIES, evaluateDiscovery, formatDiscoveryEvalSummary } = await import('./discovery-eval.js');
367
+ if (!DISCOVERY_EVAL_STRATEGIES.includes(strategy)) {
368
+ console.error(`Error: --strategy must be ${DISCOVERY_EVAL_STRATEGIES.join(', ')}`);
369
+ process.exit(1);
370
+ }
371
+ try {
372
+ const report = await evaluateDiscovery({
373
+ cwd: process.cwd(),
374
+ fixturePath: path.resolve(queries),
375
+ backend,
376
+ strategy: strategy,
377
+ });
378
+ const serialized = `${JSON.stringify(report, null, 2)}\n`;
379
+ const out = parseFlagValue(args, '--out', 'Error: --out requires a path');
380
+ if (out) {
381
+ const fs = await import('node:fs');
382
+ fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
383
+ fs.writeFileSync(path.resolve(out), serialized);
384
+ }
385
+ console.log(args.includes('--json') ? serialized.trimEnd() : formatDiscoveryEvalSummary(report));
386
+ }
387
+ catch (error) {
388
+ console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
389
+ process.exit(1);
390
+ }
391
+ }
338
392
  /**
339
393
  * Handle 'index watch' command — filesystem watcher daemon for auto-index updates.
340
394
  *
@@ -1308,7 +1362,7 @@ async function handleDiscover(args) {
1308
1362
  if (!phrase) {
1309
1363
  console.error('Error: aiwg index discover requires a search phrase');
1310
1364
  console.log('');
1311
- console.log('Usage: aiwg index discover "<phrase>" [--type <kinds>] [--limit N] [--json|--format json|text] [--pretty|--compact] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <exact-or-channel>] [--offline]');
1365
+ console.log('Usage: aiwg index discover "<phrase>" [--type <kinds>] [--limit N] [--json|--format json|text] [--pretty|--compact] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]');
1312
1366
  console.log('');
1313
1367
  console.log('Examples:');
1314
1368
  console.log(' aiwg index discover "create intake"');
@@ -1388,8 +1442,8 @@ async function handleShow(args) {
1388
1442
  }
1389
1443
  const HELP_TEXT = [
1390
1444
  '',
1391
- 'Usage: aiwg show <type> <name> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <exact-or-channel>] [--offline]',
1392
- ' aiwg show metadata <id-or-name-or-path> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <exact-or-channel>] [--offline]',
1445
+ 'Usage: aiwg show <type> <name> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]',
1446
+ ' aiwg show metadata <id-or-name-or-path> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]',
1393
1447
  ' aiwg index show <type> <name> ...',
1394
1448
  '',
1395
1449
  `Types: ${OPERATIONAL_SHOW_TYPES.join(' | ')}`,
@@ -108,6 +108,21 @@ export const DISCOVER_FACETS = [
108
108
  ],
109
109
  capabilities: ['new-project', 'new-bundle'],
110
110
  },
111
+ {
112
+ facet: 'feature-domain',
113
+ label: 'Project-local bundle lifecycle',
114
+ intents: [
115
+ 'project-local bundle',
116
+ 'create project-local bundle',
117
+ 'deploy project-local bundle',
118
+ 'doctor project-local bundle',
119
+ 'promote bundle',
120
+ 'promote project-local',
121
+ 'graduate project-local bundle',
122
+ 'graduate to upstream',
123
+ ],
124
+ capabilities: ['new-bundle', 'use', 'aiwg-doctor', 'promote'],
125
+ },
111
126
  {
112
127
  facet: 'provider-capability',
113
128
  label: 'Provider capability routing (native vs emulated)',
@@ -0,0 +1,290 @@
1
+ import fs from 'node:fs';
2
+ import os from 'node:os';
3
+ import path from 'node:path';
4
+ import { performance } from 'node:perf_hooks';
5
+ import { OPERATIONAL_DISCOVERY_TYPES } from './types.js';
6
+ import { loadGraphIndexFile } from './index-reader.js';
7
+ import { loadFortemiCoreExport, loadFortemiCoreMetadataEntries, scoreStaticRecord } from './fortemi-core-query-adapter.js';
8
+ import { discoverCapability } from './query-engine.js';
9
+ export const DISCOVERY_EVAL_SCHEMA = 'aiwg.discovery-relevance.v1';
10
+ export const DISCOVERY_EVAL_REPORT_SCHEMA = 'aiwg.discovery-eval-report.v1';
11
+ export const DISCOVERY_EVAL_STRATEGIES = ['lexical', 'dense', 'hybrid-rrf', 'rerank', 'chunk-multivector'];
12
+ const round = (value, digits = 6) => {
13
+ const scale = 10 ** digits;
14
+ return Math.round(value * scale) / scale;
15
+ };
16
+ function percentile(values, p) {
17
+ if (values.length === 0)
18
+ return 0;
19
+ const sorted = [...values].sort((a, b) => a - b);
20
+ return sorted[Math.ceil(p * sorted.length) - 1] ?? sorted[sorted.length - 1];
21
+ }
22
+ function stringArray(value, label, line) {
23
+ if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== 'string' || item.trim() === '')) {
24
+ throw new Error(`Discovery relevance fixture line ${line}: ${label} must be a non-empty string array`);
25
+ }
26
+ return value;
27
+ }
28
+ export function parseDiscoveryRelevanceJsonl(content) {
29
+ const records = [];
30
+ const ids = new Set();
31
+ for (const [offset, raw] of content.split(/\r?\n/).entries()) {
32
+ if (raw.trim() === '')
33
+ continue;
34
+ const line = offset + 1;
35
+ let value;
36
+ try {
37
+ value = JSON.parse(raw);
38
+ }
39
+ catch (error) {
40
+ throw new Error(`Discovery relevance fixture line ${line}: malformed JSON (${error instanceof Error ? error.message : String(error)})`);
41
+ }
42
+ if (value.schema !== DISCOVERY_EVAL_SCHEMA)
43
+ throw new Error(`Discovery relevance fixture line ${line}: unsupported schema`);
44
+ if (typeof value.id !== 'string' || value.id.trim() === '')
45
+ throw new Error(`Discovery relevance fixture line ${line}: id must be a non-empty string`);
46
+ if (ids.has(value.id))
47
+ throw new Error(`Discovery relevance fixture line ${line}: duplicate query id '${value.id}'`);
48
+ ids.add(value.id);
49
+ if (typeof value.query !== 'string' || value.query.trim() === '')
50
+ throw new Error(`Discovery relevance fixture line ${line}: query must be a non-empty string`);
51
+ if (!OPERATIONAL_DISCOVERY_TYPES.includes(String(value.target_type))) {
52
+ throw new Error(`Discovery relevance fixture line ${line}: invalid target_type '${String(value.target_type)}'`);
53
+ }
54
+ const classes = ['exact-name', 'capability', 'process-step', 'hard-negative', 'cross-type'];
55
+ if (!classes.includes(String(value.query_class)))
56
+ throw new Error(`Discovery relevance fixture line ${line}: invalid query_class '${String(value.query_class)}'`);
57
+ records.push({
58
+ schema: DISCOVERY_EVAL_SCHEMA, id: value.id, query: value.query,
59
+ target_type: value.target_type,
60
+ relevant_ids: stringArray(value.relevant_ids, 'relevant_ids', line),
61
+ hard_negative_ids: stringArray(value.hard_negative_ids, 'hard_negative_ids', line),
62
+ query_class: value.query_class,
63
+ ...(typeof value.notes === 'string' ? { notes: value.notes } : {}),
64
+ });
65
+ }
66
+ if (records.length === 0)
67
+ throw new Error('Discovery relevance fixture contains no queries');
68
+ return records;
69
+ }
70
+ export function validateOperationalCoverage(records, minimum = 10) {
71
+ for (const type of OPERATIONAL_DISCOVERY_TYPES) {
72
+ const count = records.filter((record) => record.target_type === type).length;
73
+ if (count < minimum)
74
+ throw new Error(`Discovery relevance fixture requires at least ${minimum} '${type}' queries; found ${count}`);
75
+ }
76
+ }
77
+ function terms(text) {
78
+ return [...new Set(text.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().split(/\s+/).filter((term) => term.length > 1))];
79
+ }
80
+ function identity(entry) { return `${entry.type}:${entry.name ?? entry.title}`.toLowerCase(); }
81
+ function matches(item, expected) {
82
+ const needle = expected.toLowerCase();
83
+ return item.id.toLowerCase() === needle || `${item.type}:${item.name}`.toLowerCase() === needle;
84
+ }
85
+ function fields(entry) {
86
+ return [entry.name ?? '', entry.title, entry.capability ?? '', entry.summary, ...(entry.triggers ?? []), ...(entry.searchTerms ?? []), entry.path].filter(Boolean);
87
+ }
88
+ function cosine(left, right) {
89
+ const a = terms(left);
90
+ const b = terms(right);
91
+ if (!a.length || !b.length)
92
+ return 0;
93
+ const set = new Set(b);
94
+ return a.filter((term) => set.has(term)).length / Math.sqrt(a.length * b.length);
95
+ }
96
+ function prototypeRank(entries, query, strategy, limit) {
97
+ const dense = entries.map((entry) => ({ entry, score: cosine(query, fields(entry).join(' ')) }));
98
+ const lexical = entries.map((entry) => {
99
+ const queryTerms = terms(query);
100
+ const values = fields(entry).map((field) => field.toLowerCase());
101
+ const hits = queryTerms.filter((term) => values.some((value) => value.includes(term))).length;
102
+ return { entry, score: (values.some((value) => value === query.toLowerCase()) ? 1 : 0) + (queryTerms.length ? hits / queryTerms.length : 0) };
103
+ });
104
+ let ranked;
105
+ if (strategy === 'dense')
106
+ ranked = dense;
107
+ else if (strategy === 'chunk-multivector')
108
+ ranked = entries.map((entry) => ({ entry, score: Math.max(...fields(entry).map((field) => cosine(query, field)), 0) }));
109
+ else if (strategy === 'rerank')
110
+ ranked = [...lexical].sort((a, b) => b.score - a.score).slice(0, 50)
111
+ .map((candidate) => ({ entry: candidate.entry, score: candidate.score + 0.35 * cosine(query, fields(candidate.entry).join(' ')) }));
112
+ else {
113
+ const fused = new Map();
114
+ for (const list of [lexical, dense]) {
115
+ [...list].sort((a, b) => b.score - a.score || identity(a.entry).localeCompare(identity(b.entry))).forEach((candidate, rank) => {
116
+ const key = identity(candidate.entry);
117
+ const current = fused.get(key) ?? { entry: candidate.entry, score: 0 };
118
+ current.score += 1 / (61 + rank);
119
+ fused.set(key, current);
120
+ });
121
+ }
122
+ ranked = [...fused.values()];
123
+ }
124
+ return ranked.filter((candidate) => candidate.score > 0)
125
+ .sort((a, b) => b.score - a.score || identity(a.entry).localeCompare(identity(b.entry))).slice(0, limit)
126
+ .map(({ entry, score }) => ({ id: identity(entry), type: entry.type, name: entry.name ?? entry.title, score: round(score) }));
127
+ }
128
+ async function currentRank(cwd, query, backend, limit) {
129
+ const output = [];
130
+ const original = console.log;
131
+ console.log = (...args) => output.push(args.map(String).join(' '));
132
+ try {
133
+ await discoverCapability(cwd, { phrase: query.query, typeFilter: [query.target_type], graph: 'framework', backend, limit, json: true, jsonPretty: false, includePaths: false });
134
+ }
135
+ finally {
136
+ console.log = original;
137
+ }
138
+ const envelope = JSON.parse(output.join(''));
139
+ return envelope.results.map((item) => ({ id: item.id, type: item.type, name: item.name ?? item.title, score: item.score }));
140
+ }
141
+ function fortemiStaticRank(exported, query, limit) {
142
+ const queryTerms = query.query.toLowerCase().split(/[^a-z0-9-]+/).map((term) => term.trim()).filter((term) => term.length > 2);
143
+ return exported.items
144
+ .filter((record) => (record.search?.type ?? record.type.replace(/^aiwg:/, '')) === query.target_type)
145
+ .map((record) => ({ record, ...scoreStaticRecord(record, queryTerms) }))
146
+ .filter((item) => item.score > 0)
147
+ .sort((a, b) => b.score - a.score || a.record.source.path.localeCompare(b.record.source.path))
148
+ .slice(0, limit)
149
+ .map(({ record, score }) => ({
150
+ id: `${query.target_type}:${record.name ?? record.search?.name ?? record.title}`.toLowerCase(),
151
+ type: query.target_type,
152
+ name: record.name ?? record.search?.name ?? record.title,
153
+ score: round(score),
154
+ }));
155
+ }
156
+ function loadEntries(cwd, backend) {
157
+ if (backend === 'fortemi-core') {
158
+ const loaded = loadFortemiCoreMetadataEntries(cwd, 'framework');
159
+ if (!loaded.entries.length)
160
+ throw new Error(loaded.reason ?? 'Fortemi Core framework index is empty');
161
+ return loaded.entries;
162
+ }
163
+ const index = loadGraphIndexFile(cwd, 'metadata.json', 'framework');
164
+ if (!index)
165
+ throw new Error('Local framework index is missing; run `aiwg index build --graph framework`');
166
+ return Object.values(index.entries);
167
+ }
168
+ export function calculateDiscoveryMetrics(records, resultSets) {
169
+ const ranks = records.map((record, index) => {
170
+ const rank = (resultSets[index] ?? []).findIndex((item) => record.relevant_ids.some((id) => matches(item, id)));
171
+ return rank < 0 ? null : rank + 1;
172
+ });
173
+ const hit = (k) => ranks.filter((rank) => rank !== null && rank <= k).length / records.length;
174
+ const perType = {};
175
+ for (const type of OPERATIONAL_DISCOVERY_TYPES) {
176
+ const selected = records.map((record, index) => ({ record, index })).filter(({ record }) => record.target_type === type);
177
+ perType[type] = round(selected.filter(({ index }) => ranks[index] !== null && ranks[index] <= 3).length / selected.length);
178
+ }
179
+ return {
180
+ query_count: records.length, hit_at_1: round(hit(1)), hit_at_3: round(hit(3)), hit_at_5: round(hit(5)),
181
+ mrr: round(ranks.reduce((sum, rank) => sum + (rank === null ? 0 : 1 / rank), 0) / records.length),
182
+ ndcg_at_10: round(ranks.reduce((sum, rank) => sum + (rank === null || rank > 10 ? 0 : 1 / Math.log2(rank + 1)), 0) / records.length),
183
+ per_type_recall_at_3: perType,
184
+ hard_negative_intrusion_at_5: round(records.filter((record, index) => (resultSets[index] ?? []).slice(0, 5)
185
+ .some((item) => record.hard_negative_ids.some((id) => matches(item, id)))).length / records.length),
186
+ };
187
+ }
188
+ function indexBytes(backend) {
189
+ const root = path.join(process.env.XDG_DATA_HOME ?? path.join(os.homedir(), '.local', 'share'), 'aiwg', 'index', ...(backend === 'local' ? ['framework'] : ['fortemi-core', 'framework']));
190
+ let total = 0;
191
+ const visit = (target) => {
192
+ if (!fs.existsSync(target))
193
+ return;
194
+ const stat = fs.statSync(target);
195
+ if (stat.isFile())
196
+ total += stat.size;
197
+ else
198
+ for (const child of fs.readdirSync(target))
199
+ visit(path.join(target, child));
200
+ };
201
+ visit(root);
202
+ return total;
203
+ }
204
+ export async function evaluateDiscovery(options) {
205
+ const records = parseDiscoveryRelevanceJsonl(fs.readFileSync(options.fixturePath, 'utf8'));
206
+ validateOperationalCoverage(records);
207
+ const limit = options.limit ?? 10;
208
+ const entries = loadEntries(options.cwd, options.backend);
209
+ const fortemiExport = options.backend === 'fortemi-core' && options.strategy === 'lexical'
210
+ ? loadFortemiCoreExport(options.cwd, 'framework')
211
+ : null;
212
+ if (fortemiExport && !fortemiExport.exported) {
213
+ throw new Error(fortemiExport.reason ?? 'Fortemi Core framework export is unavailable');
214
+ }
215
+ const resultSets = [];
216
+ const latencies = [];
217
+ let peakRss = process.memoryUsage().rss;
218
+ for (const record of records) {
219
+ const start = performance.now();
220
+ resultSets.push(options.strategy === 'lexical'
221
+ ? options.backend === 'fortemi-core'
222
+ ? fortemiStaticRank(fortemiExport.exported, record, limit)
223
+ : await currentRank(options.cwd, record, options.backend, limit)
224
+ : prototypeRank(entries.filter((entry) => entry.type === record.target_type), record.query, options.strategy, limit));
225
+ latencies.push(performance.now() - start);
226
+ peakRss = Math.max(peakRss, process.memoryUsage().rss);
227
+ }
228
+ const metrics = calculateDiscoveryMetrics(records, resultSets);
229
+ const cpus = os.cpus();
230
+ let parity;
231
+ if (options.backend === 'fortemi-core' && options.strategy === 'lexical') {
232
+ const localSets = [];
233
+ for (const record of records)
234
+ localSets.push(await currentRank(options.cwd, record, 'local', limit));
235
+ const differingQueries = [];
236
+ let topOneAgreements = 0;
237
+ let topFiveOverlap = 0;
238
+ for (let index = 0; index < records.length; index++) {
239
+ const fortemi = resultSets[index];
240
+ const local = localSets[index];
241
+ const key = (item) => item ? `${item.type}:${item.name}`.toLowerCase() : '';
242
+ if (key(fortemi[0]) === key(local[0]))
243
+ topOneAgreements++;
244
+ else
245
+ differingQueries.push(records[index].id);
246
+ const localFive = new Set(local.slice(0, 5).map((item) => key(item)));
247
+ topFiveOverlap += fortemi.slice(0, 5).filter((item) => localFive.has(key(item))).length / Math.max(1, Math.min(5, local.length, fortemi.length));
248
+ }
249
+ parity = {
250
+ compared_to: 'local:lexical',
251
+ top_1_agreement: round(topOneAgreements / records.length),
252
+ top_5_overlap: round(topFiveOverlap / records.length),
253
+ differing_queries: differingQueries,
254
+ };
255
+ }
256
+ return {
257
+ schema: DISCOVERY_EVAL_REPORT_SCHEMA,
258
+ corpus: { path: path.relative(options.cwd, options.fixturePath).replace(/\\/g, '/'), schema: DISCOVERY_EVAL_SCHEMA, query_count: records.length },
259
+ configuration: { backend: options.backend, strategy: options.strategy, limit, graph: 'framework' },
260
+ hardware: { platform: os.platform(), arch: os.arch(), cpu_model: cpus[0]?.model ?? 'unknown', logical_cpus: cpus.length, total_memory_bytes: os.totalmem(), node: process.version },
261
+ metrics,
262
+ performance: { p50_latency_ms: round(percentile(latencies, 0.5), 3), p95_latency_ms: round(percentile(latencies, 0.95), 3), index_bytes: indexBytes(options.backend), peak_resident_memory_bytes: peakRss },
263
+ ...(parity ? { parity } : {}),
264
+ adoption_gate: {
265
+ baseline: 'local:lexical', no_per_type_hit_at_3_regression: null, aggregate_mrr_improvement: null,
266
+ mrr_95pct_confidence_interval: null, latency_ceiling_ms: options.latencyCeilingMs ?? 250,
267
+ storage_ceiling_ratio: options.storageCeilingRatio ?? 2, clears_gate: null,
268
+ decision: 'Run the full strategy matrix and compare against local:lexical before adoption.',
269
+ },
270
+ queries: records.map((record, index) => {
271
+ const rank = resultSets[index].findIndex((item) => record.relevant_ids.some((id) => matches(item, id)));
272
+ return { id: record.id, target_type: record.target_type, relevant_rank: rank < 0 ? null : rank + 1, reciprocal_rank: rank < 0 ? 0 : round(1 / (rank + 1)), latency_ms: round(latencies[index], 3), results: resultSets[index] };
273
+ }),
274
+ };
275
+ }
276
+ export function formatDiscoveryEvalSummary(report) {
277
+ const m = report.metrics;
278
+ return [
279
+ `Discovery evaluation: ${report.configuration.backend}:${report.configuration.strategy}`,
280
+ `Corpus: ${report.corpus.query_count} queries (${report.corpus.path})`, '',
281
+ 'Metric Value', `Hit@1 ${m.hit_at_1.toFixed(4)}`, `Hit@3 ${m.hit_at_3.toFixed(4)}`,
282
+ `Hit@5 ${m.hit_at_5.toFixed(4)}`, `MRR ${m.mrr.toFixed(4)}`, `nDCG@10 ${m.ndcg_at_10.toFixed(4)}`,
283
+ `Hard-negative@5 ${m.hard_negative_intrusion_at_5.toFixed(4)}`, `p50 latency ${report.performance.p50_latency_ms.toFixed(3)} ms`,
284
+ `p95 latency ${report.performance.p95_latency_ms.toFixed(3)} ms`, `Index bytes ${report.performance.index_bytes}`,
285
+ `Peak resident memory ${report.performance.peak_resident_memory_bytes}`, '', 'Per-type Hit@3:',
286
+ ...Object.entries(m.per_type_recall_at_3).map(([type, value]) => ` ${type.padEnd(10)} ${value.toFixed(4)}`), '',
287
+ `Decision: ${report.adoption_gate.decision}`,
288
+ ].join('\n');
289
+ }
290
+ //# sourceMappingURL=discovery-eval.js.map
@@ -19,7 +19,7 @@ function includesAny(text, terms) {
19
19
  }
20
20
  return matches;
21
21
  }
22
- function scoreStaticRecord(record, terms) {
22
+ export function scoreStaticRecord(record, terms) {
23
23
  const fields = [
24
24
  { name: "title", text: record.title, weight: 3 },
25
25
  { name: "name", text: record.name ?? record.search?.name ?? "", weight: 3 },
@@ -5,7 +5,7 @@ async function loadFortemiShardConverter() {
5
5
  let core;
6
6
  try {
7
7
  core = (await import(
8
- /* @vite-ignore */ "@fortemi/core/aiwg-index"));
8
+ /* @vite-ignore */ "@fortemi/core/aiwg-index-shard"));
9
9
  }
10
10
  catch {
11
11
  throw new Error("Portable Fortemi shard export requires @fortemi/core with aiwgFortemiIndexToKnowledgeShard.");