@aiwg/cli 2026.7.19 → 2026.7.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +397 -385
- package/dist/src/artifacts/cli.js +59 -5
- package/dist/src/artifacts/discover-facets.js +15 -0
- package/dist/src/artifacts/discovery-eval.js +290 -0
- package/dist/src/artifacts/fortemi-core-query-adapter.js +1 -1
- package/dist/src/artifacts/fortemi-shard-export.js +1 -1
- package/dist/src/artifacts/query-engine.js +10 -6
- package/dist/src/cli/handlers/help.js +2 -1
- package/dist/src/cli/handlers/index.js +3 -1
- package/dist/src/cli/handlers/resource-versions.js +247 -0
- package/dist/src/cli/handlers/subcommands.js +55 -3
- package/dist/src/cli/handlers/use.js +15 -3
- package/dist/src/cli/handlers/utilities.js +27 -26
- package/dist/src/config/cli.js +13 -9
- package/dist/src/config/project-artifacts-runtime.mjs +68 -0
- package/dist/src/config/project-artifacts.js +1 -68
- package/dist/src/extensions/commands/definitions.js +36 -2
- package/dist/src/extensions/project-local-discovery.js +86 -2
- package/dist/src/extensions/project-local-remove.js +52 -56
- package/dist/src/extensions/shadow-resolver.js +3 -1
- package/dist/src/plugins/standalone-packager.js +143 -0
- package/dist/src/resources/cache-cleanup.js +67 -0
- package/dist/src/resources/doctor.js +107 -0
- package/dist/src/resources/lockfile.js +125 -0
- package/dist/src/resources/resolver.js +133 -0
- package/dist/src/resources/web-release.d.ts +8 -0
- package/dist/src/resources/web-release.js +159 -1
- package/dist/src/smiths/context-pipeline/aiwg-md.js +5 -1
- package/dist/src/smiths/context-pipeline/claude-hook.js +21 -1
- package/dist/src/smiths/context-pipeline/finalization.js +5 -3
- package/dist/src/smiths/context-pipeline/generator.js +4 -1
- package/dist/src/smiths/context-pipeline/parallelism-section.js +34 -1
- package/dist/src/smiths/context-pipeline/workspace-context.js +15 -3
- package/dist/src/smiths/mcpsmith/example.js +3 -1
- package/dist/src/smiths/mcpsmith/generator.js +3 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +2 -1
- package/dist/src/storage/cli.js +3 -2
- package/dist/src/storage/subsystem-cli.js +7 -2
- package/dist/src/update/notifier.mjs +1 -1
- package/dist/src/update/service.mjs +123 -0
- package/package.json +3 -2
|
@@ -134,7 +134,7 @@ function parseAiwgVersionFlag(args) {
|
|
|
134
134
|
return undefined;
|
|
135
135
|
const value = args[indices[0] + 1];
|
|
136
136
|
if (!value || value.startsWith('--')) {
|
|
137
|
-
console.error('Error: --aiwg-version requires an exact calendar-semver version or channel name');
|
|
137
|
+
console.error('Error: --aiwg-version requires an exact calendar-semver version, SemVer range, sha256 digest, or channel name');
|
|
138
138
|
process.exit(1);
|
|
139
139
|
}
|
|
140
140
|
try {
|
|
@@ -257,6 +257,9 @@ export async function main(args) {
|
|
|
257
257
|
case 'dedup-report':
|
|
258
258
|
await handleDedup(subcommandArgs);
|
|
259
259
|
break;
|
|
260
|
+
case 'eval-discovery':
|
|
261
|
+
await handleEvalDiscovery(subcommandArgs);
|
|
262
|
+
break;
|
|
260
263
|
case 'watch':
|
|
261
264
|
await handleWatch(subcommandArgs);
|
|
262
265
|
break;
|
|
@@ -283,7 +286,7 @@ export async function main(args) {
|
|
|
283
286
|
break;
|
|
284
287
|
default:
|
|
285
288
|
console.error(`Error: Unknown index subcommand '${subcommand}'`);
|
|
286
|
-
console.log('Available: build, query, discover, show, export, sync, migrate-legacy, deps, stats, status, list, neighbors, set, embed, similar, dedup-report, watch');
|
|
289
|
+
console.log('Available: build, query, discover, show, export, sync, migrate-legacy, deps, stats, status, list, neighbors, set, embed, similar, dedup-report, eval-discovery, watch');
|
|
287
290
|
process.exit(1);
|
|
288
291
|
}
|
|
289
292
|
}
|
|
@@ -306,6 +309,7 @@ function printIndexUsage() {
|
|
|
306
309
|
console.log(' embed Build the semantic embedding index for a graph (opt-in deps)');
|
|
307
310
|
console.log(' similar Semantic neighbors of a node (requires embed)');
|
|
308
311
|
console.log(' dedup-report Near-duplicate node pairs above a similarity threshold');
|
|
312
|
+
console.log(' eval-discovery Benchmark operational capability discovery relevance');
|
|
309
313
|
console.log(' watch Start a filesystem watcher for automatic incremental index updates');
|
|
310
314
|
console.log('');
|
|
311
315
|
console.log('Options:');
|
|
@@ -332,9 +336,59 @@ function printIndexUsage() {
|
|
|
332
336
|
console.log(' aiwg index deps .aiwg/requirements/UC-001.md');
|
|
333
337
|
console.log(' aiwg index stats --json');
|
|
334
338
|
console.log(' aiwg index stats --graph project');
|
|
339
|
+
console.log(' aiwg index eval-discovery --queries test/fixtures/artifacts/discovery-relevance.jsonl --backend local --strategy lexical');
|
|
335
340
|
console.log(' aiwg index neighbors --graph citation-network --node REF-008 --direction in --edge-type cites');
|
|
336
341
|
console.log(' aiwg index set --graph citation-network --op intersection --node-a REF-008 --node-b REF-016 --direction in');
|
|
337
342
|
}
|
|
343
|
+
async function handleEvalDiscovery(args) {
|
|
344
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
345
|
+
console.log('Usage: aiwg index eval-discovery --queries <jsonl> --backend <local|fortemi-core> --strategy <lexical|dense|hybrid-rrf|rerank|chunk-multivector> [options]');
|
|
346
|
+
console.log('');
|
|
347
|
+
console.log('Options:');
|
|
348
|
+
console.log(' --queries <path> Versioned relevance JSONL fixture (required)');
|
|
349
|
+
console.log(' --backend <name> local or fortemi-core (required)');
|
|
350
|
+
console.log(' --strategy <name> lexical, dense, hybrid-rrf, rerank, or chunk-multivector');
|
|
351
|
+
console.log(' --out <path> Write the JSON report to a file');
|
|
352
|
+
console.log(' --json Emit JSON instead of the readable summary');
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const queries = parseFlagValue(args, '--queries', 'Error: --queries requires a JSONL path');
|
|
356
|
+
if (!queries) {
|
|
357
|
+
console.error('Error: --queries is required');
|
|
358
|
+
process.exit(1);
|
|
359
|
+
}
|
|
360
|
+
const backend = parseFlagValue(args, '--backend', 'Error: --backend requires local or fortemi-core');
|
|
361
|
+
if (backend !== 'local' && backend !== 'fortemi-core') {
|
|
362
|
+
console.error('Error: --backend must be local or fortemi-core');
|
|
363
|
+
process.exit(1);
|
|
364
|
+
}
|
|
365
|
+
const strategy = parseFlagValue(args, '--strategy', 'Error: --strategy requires a value') ?? 'lexical';
|
|
366
|
+
const { DISCOVERY_EVAL_STRATEGIES, evaluateDiscovery, formatDiscoveryEvalSummary } = await import('./discovery-eval.js');
|
|
367
|
+
if (!DISCOVERY_EVAL_STRATEGIES.includes(strategy)) {
|
|
368
|
+
console.error(`Error: --strategy must be ${DISCOVERY_EVAL_STRATEGIES.join(', ')}`);
|
|
369
|
+
process.exit(1);
|
|
370
|
+
}
|
|
371
|
+
try {
|
|
372
|
+
const report = await evaluateDiscovery({
|
|
373
|
+
cwd: process.cwd(),
|
|
374
|
+
fixturePath: path.resolve(queries),
|
|
375
|
+
backend,
|
|
376
|
+
strategy: strategy,
|
|
377
|
+
});
|
|
378
|
+
const serialized = `${JSON.stringify(report, null, 2)}\n`;
|
|
379
|
+
const out = parseFlagValue(args, '--out', 'Error: --out requires a path');
|
|
380
|
+
if (out) {
|
|
381
|
+
const fs = await import('node:fs');
|
|
382
|
+
fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
|
|
383
|
+
fs.writeFileSync(path.resolve(out), serialized);
|
|
384
|
+
}
|
|
385
|
+
console.log(args.includes('--json') ? serialized.trimEnd() : formatDiscoveryEvalSummary(report));
|
|
386
|
+
}
|
|
387
|
+
catch (error) {
|
|
388
|
+
console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
389
|
+
process.exit(1);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
338
392
|
/**
|
|
339
393
|
* Handle 'index watch' command — filesystem watcher daemon for auto-index updates.
|
|
340
394
|
*
|
|
@@ -1308,7 +1362,7 @@ async function handleDiscover(args) {
|
|
|
1308
1362
|
if (!phrase) {
|
|
1309
1363
|
console.error('Error: aiwg index discover requires a search phrase');
|
|
1310
1364
|
console.log('');
|
|
1311
|
-
console.log('Usage: aiwg index discover "<phrase>" [--type <kinds>] [--limit N] [--json|--format json|text] [--pretty|--compact] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <
|
|
1365
|
+
console.log('Usage: aiwg index discover "<phrase>" [--type <kinds>] [--limit N] [--json|--format json|text] [--pretty|--compact] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]');
|
|
1312
1366
|
console.log('');
|
|
1313
1367
|
console.log('Examples:');
|
|
1314
1368
|
console.log(' aiwg index discover "create intake"');
|
|
@@ -1388,8 +1442,8 @@ async function handleShow(args) {
|
|
|
1388
1442
|
}
|
|
1389
1443
|
const HELP_TEXT = [
|
|
1390
1444
|
'',
|
|
1391
|
-
'Usage: aiwg show <type> <name> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <
|
|
1392
|
-
' aiwg show metadata <id-or-name-or-path> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <
|
|
1445
|
+
'Usage: aiwg show <type> <name> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]',
|
|
1446
|
+
' aiwg show metadata <id-or-name-or-path> [--json] [--first] [--graph <name>] [--backend local|fortemi-core] [--resource-source local|web|auto] [--aiwg-version <version|range|digest|channel>] [--offline]',
|
|
1393
1447
|
' aiwg index show <type> <name> ...',
|
|
1394
1448
|
'',
|
|
1395
1449
|
`Types: ${OPERATIONAL_SHOW_TYPES.join(' | ')}`,
|
|
@@ -108,6 +108,21 @@ export const DISCOVER_FACETS = [
|
|
|
108
108
|
],
|
|
109
109
|
capabilities: ['new-project', 'new-bundle'],
|
|
110
110
|
},
|
|
111
|
+
{
|
|
112
|
+
facet: 'feature-domain',
|
|
113
|
+
label: 'Project-local bundle lifecycle',
|
|
114
|
+
intents: [
|
|
115
|
+
'project-local bundle',
|
|
116
|
+
'create project-local bundle',
|
|
117
|
+
'deploy project-local bundle',
|
|
118
|
+
'doctor project-local bundle',
|
|
119
|
+
'promote bundle',
|
|
120
|
+
'promote project-local',
|
|
121
|
+
'graduate project-local bundle',
|
|
122
|
+
'graduate to upstream',
|
|
123
|
+
],
|
|
124
|
+
capabilities: ['new-bundle', 'use', 'aiwg-doctor', 'promote'],
|
|
125
|
+
},
|
|
111
126
|
{
|
|
112
127
|
facet: 'provider-capability',
|
|
113
128
|
label: 'Provider capability routing (native vs emulated)',
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { performance } from 'node:perf_hooks';
|
|
5
|
+
import { OPERATIONAL_DISCOVERY_TYPES } from './types.js';
|
|
6
|
+
import { loadGraphIndexFile } from './index-reader.js';
|
|
7
|
+
import { loadFortemiCoreExport, loadFortemiCoreMetadataEntries, scoreStaticRecord } from './fortemi-core-query-adapter.js';
|
|
8
|
+
import { discoverCapability } from './query-engine.js';
|
|
9
|
+
export const DISCOVERY_EVAL_SCHEMA = 'aiwg.discovery-relevance.v1';
|
|
10
|
+
export const DISCOVERY_EVAL_REPORT_SCHEMA = 'aiwg.discovery-eval-report.v1';
|
|
11
|
+
export const DISCOVERY_EVAL_STRATEGIES = ['lexical', 'dense', 'hybrid-rrf', 'rerank', 'chunk-multivector'];
|
|
12
|
+
const round = (value, digits = 6) => {
|
|
13
|
+
const scale = 10 ** digits;
|
|
14
|
+
return Math.round(value * scale) / scale;
|
|
15
|
+
};
|
|
16
|
+
function percentile(values, p) {
|
|
17
|
+
if (values.length === 0)
|
|
18
|
+
return 0;
|
|
19
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
20
|
+
return sorted[Math.ceil(p * sorted.length) - 1] ?? sorted[sorted.length - 1];
|
|
21
|
+
}
|
|
22
|
+
function stringArray(value, label, line) {
|
|
23
|
+
if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== 'string' || item.trim() === '')) {
|
|
24
|
+
throw new Error(`Discovery relevance fixture line ${line}: ${label} must be a non-empty string array`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
export function parseDiscoveryRelevanceJsonl(content) {
|
|
29
|
+
const records = [];
|
|
30
|
+
const ids = new Set();
|
|
31
|
+
for (const [offset, raw] of content.split(/\r?\n/).entries()) {
|
|
32
|
+
if (raw.trim() === '')
|
|
33
|
+
continue;
|
|
34
|
+
const line = offset + 1;
|
|
35
|
+
let value;
|
|
36
|
+
try {
|
|
37
|
+
value = JSON.parse(raw);
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
throw new Error(`Discovery relevance fixture line ${line}: malformed JSON (${error instanceof Error ? error.message : String(error)})`);
|
|
41
|
+
}
|
|
42
|
+
if (value.schema !== DISCOVERY_EVAL_SCHEMA)
|
|
43
|
+
throw new Error(`Discovery relevance fixture line ${line}: unsupported schema`);
|
|
44
|
+
if (typeof value.id !== 'string' || value.id.trim() === '')
|
|
45
|
+
throw new Error(`Discovery relevance fixture line ${line}: id must be a non-empty string`);
|
|
46
|
+
if (ids.has(value.id))
|
|
47
|
+
throw new Error(`Discovery relevance fixture line ${line}: duplicate query id '${value.id}'`);
|
|
48
|
+
ids.add(value.id);
|
|
49
|
+
if (typeof value.query !== 'string' || value.query.trim() === '')
|
|
50
|
+
throw new Error(`Discovery relevance fixture line ${line}: query must be a non-empty string`);
|
|
51
|
+
if (!OPERATIONAL_DISCOVERY_TYPES.includes(String(value.target_type))) {
|
|
52
|
+
throw new Error(`Discovery relevance fixture line ${line}: invalid target_type '${String(value.target_type)}'`);
|
|
53
|
+
}
|
|
54
|
+
const classes = ['exact-name', 'capability', 'process-step', 'hard-negative', 'cross-type'];
|
|
55
|
+
if (!classes.includes(String(value.query_class)))
|
|
56
|
+
throw new Error(`Discovery relevance fixture line ${line}: invalid query_class '${String(value.query_class)}'`);
|
|
57
|
+
records.push({
|
|
58
|
+
schema: DISCOVERY_EVAL_SCHEMA, id: value.id, query: value.query,
|
|
59
|
+
target_type: value.target_type,
|
|
60
|
+
relevant_ids: stringArray(value.relevant_ids, 'relevant_ids', line),
|
|
61
|
+
hard_negative_ids: stringArray(value.hard_negative_ids, 'hard_negative_ids', line),
|
|
62
|
+
query_class: value.query_class,
|
|
63
|
+
...(typeof value.notes === 'string' ? { notes: value.notes } : {}),
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
if (records.length === 0)
|
|
67
|
+
throw new Error('Discovery relevance fixture contains no queries');
|
|
68
|
+
return records;
|
|
69
|
+
}
|
|
70
|
+
export function validateOperationalCoverage(records, minimum = 10) {
|
|
71
|
+
for (const type of OPERATIONAL_DISCOVERY_TYPES) {
|
|
72
|
+
const count = records.filter((record) => record.target_type === type).length;
|
|
73
|
+
if (count < minimum)
|
|
74
|
+
throw new Error(`Discovery relevance fixture requires at least ${minimum} '${type}' queries; found ${count}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function terms(text) {
|
|
78
|
+
return [...new Set(text.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().split(/\s+/).filter((term) => term.length > 1))];
|
|
79
|
+
}
|
|
80
|
+
function identity(entry) { return `${entry.type}:${entry.name ?? entry.title}`.toLowerCase(); }
|
|
81
|
+
function matches(item, expected) {
|
|
82
|
+
const needle = expected.toLowerCase();
|
|
83
|
+
return item.id.toLowerCase() === needle || `${item.type}:${item.name}`.toLowerCase() === needle;
|
|
84
|
+
}
|
|
85
|
+
function fields(entry) {
|
|
86
|
+
return [entry.name ?? '', entry.title, entry.capability ?? '', entry.summary, ...(entry.triggers ?? []), ...(entry.searchTerms ?? []), entry.path].filter(Boolean);
|
|
87
|
+
}
|
|
88
|
+
function cosine(left, right) {
|
|
89
|
+
const a = terms(left);
|
|
90
|
+
const b = terms(right);
|
|
91
|
+
if (!a.length || !b.length)
|
|
92
|
+
return 0;
|
|
93
|
+
const set = new Set(b);
|
|
94
|
+
return a.filter((term) => set.has(term)).length / Math.sqrt(a.length * b.length);
|
|
95
|
+
}
|
|
96
|
+
function prototypeRank(entries, query, strategy, limit) {
|
|
97
|
+
const dense = entries.map((entry) => ({ entry, score: cosine(query, fields(entry).join(' ')) }));
|
|
98
|
+
const lexical = entries.map((entry) => {
|
|
99
|
+
const queryTerms = terms(query);
|
|
100
|
+
const values = fields(entry).map((field) => field.toLowerCase());
|
|
101
|
+
const hits = queryTerms.filter((term) => values.some((value) => value.includes(term))).length;
|
|
102
|
+
return { entry, score: (values.some((value) => value === query.toLowerCase()) ? 1 : 0) + (queryTerms.length ? hits / queryTerms.length : 0) };
|
|
103
|
+
});
|
|
104
|
+
let ranked;
|
|
105
|
+
if (strategy === 'dense')
|
|
106
|
+
ranked = dense;
|
|
107
|
+
else if (strategy === 'chunk-multivector')
|
|
108
|
+
ranked = entries.map((entry) => ({ entry, score: Math.max(...fields(entry).map((field) => cosine(query, field)), 0) }));
|
|
109
|
+
else if (strategy === 'rerank')
|
|
110
|
+
ranked = [...lexical].sort((a, b) => b.score - a.score).slice(0, 50)
|
|
111
|
+
.map((candidate) => ({ entry: candidate.entry, score: candidate.score + 0.35 * cosine(query, fields(candidate.entry).join(' ')) }));
|
|
112
|
+
else {
|
|
113
|
+
const fused = new Map();
|
|
114
|
+
for (const list of [lexical, dense]) {
|
|
115
|
+
[...list].sort((a, b) => b.score - a.score || identity(a.entry).localeCompare(identity(b.entry))).forEach((candidate, rank) => {
|
|
116
|
+
const key = identity(candidate.entry);
|
|
117
|
+
const current = fused.get(key) ?? { entry: candidate.entry, score: 0 };
|
|
118
|
+
current.score += 1 / (61 + rank);
|
|
119
|
+
fused.set(key, current);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
ranked = [...fused.values()];
|
|
123
|
+
}
|
|
124
|
+
return ranked.filter((candidate) => candidate.score > 0)
|
|
125
|
+
.sort((a, b) => b.score - a.score || identity(a.entry).localeCompare(identity(b.entry))).slice(0, limit)
|
|
126
|
+
.map(({ entry, score }) => ({ id: identity(entry), type: entry.type, name: entry.name ?? entry.title, score: round(score) }));
|
|
127
|
+
}
|
|
128
|
+
async function currentRank(cwd, query, backend, limit) {
|
|
129
|
+
const output = [];
|
|
130
|
+
const original = console.log;
|
|
131
|
+
console.log = (...args) => output.push(args.map(String).join(' '));
|
|
132
|
+
try {
|
|
133
|
+
await discoverCapability(cwd, { phrase: query.query, typeFilter: [query.target_type], graph: 'framework', backend, limit, json: true, jsonPretty: false, includePaths: false });
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
console.log = original;
|
|
137
|
+
}
|
|
138
|
+
const envelope = JSON.parse(output.join(''));
|
|
139
|
+
return envelope.results.map((item) => ({ id: item.id, type: item.type, name: item.name ?? item.title, score: item.score }));
|
|
140
|
+
}
|
|
141
|
+
function fortemiStaticRank(exported, query, limit) {
|
|
142
|
+
const queryTerms = query.query.toLowerCase().split(/[^a-z0-9-]+/).map((term) => term.trim()).filter((term) => term.length > 2);
|
|
143
|
+
return exported.items
|
|
144
|
+
.filter((record) => (record.search?.type ?? record.type.replace(/^aiwg:/, '')) === query.target_type)
|
|
145
|
+
.map((record) => ({ record, ...scoreStaticRecord(record, queryTerms) }))
|
|
146
|
+
.filter((item) => item.score > 0)
|
|
147
|
+
.sort((a, b) => b.score - a.score || a.record.source.path.localeCompare(b.record.source.path))
|
|
148
|
+
.slice(0, limit)
|
|
149
|
+
.map(({ record, score }) => ({
|
|
150
|
+
id: `${query.target_type}:${record.name ?? record.search?.name ?? record.title}`.toLowerCase(),
|
|
151
|
+
type: query.target_type,
|
|
152
|
+
name: record.name ?? record.search?.name ?? record.title,
|
|
153
|
+
score: round(score),
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
function loadEntries(cwd, backend) {
|
|
157
|
+
if (backend === 'fortemi-core') {
|
|
158
|
+
const loaded = loadFortemiCoreMetadataEntries(cwd, 'framework');
|
|
159
|
+
if (!loaded.entries.length)
|
|
160
|
+
throw new Error(loaded.reason ?? 'Fortemi Core framework index is empty');
|
|
161
|
+
return loaded.entries;
|
|
162
|
+
}
|
|
163
|
+
const index = loadGraphIndexFile(cwd, 'metadata.json', 'framework');
|
|
164
|
+
if (!index)
|
|
165
|
+
throw new Error('Local framework index is missing; run `aiwg index build --graph framework`');
|
|
166
|
+
return Object.values(index.entries);
|
|
167
|
+
}
|
|
168
|
+
export function calculateDiscoveryMetrics(records, resultSets) {
|
|
169
|
+
const ranks = records.map((record, index) => {
|
|
170
|
+
const rank = (resultSets[index] ?? []).findIndex((item) => record.relevant_ids.some((id) => matches(item, id)));
|
|
171
|
+
return rank < 0 ? null : rank + 1;
|
|
172
|
+
});
|
|
173
|
+
const hit = (k) => ranks.filter((rank) => rank !== null && rank <= k).length / records.length;
|
|
174
|
+
const perType = {};
|
|
175
|
+
for (const type of OPERATIONAL_DISCOVERY_TYPES) {
|
|
176
|
+
const selected = records.map((record, index) => ({ record, index })).filter(({ record }) => record.target_type === type);
|
|
177
|
+
perType[type] = round(selected.filter(({ index }) => ranks[index] !== null && ranks[index] <= 3).length / selected.length);
|
|
178
|
+
}
|
|
179
|
+
return {
|
|
180
|
+
query_count: records.length, hit_at_1: round(hit(1)), hit_at_3: round(hit(3)), hit_at_5: round(hit(5)),
|
|
181
|
+
mrr: round(ranks.reduce((sum, rank) => sum + (rank === null ? 0 : 1 / rank), 0) / records.length),
|
|
182
|
+
ndcg_at_10: round(ranks.reduce((sum, rank) => sum + (rank === null || rank > 10 ? 0 : 1 / Math.log2(rank + 1)), 0) / records.length),
|
|
183
|
+
per_type_recall_at_3: perType,
|
|
184
|
+
hard_negative_intrusion_at_5: round(records.filter((record, index) => (resultSets[index] ?? []).slice(0, 5)
|
|
185
|
+
.some((item) => record.hard_negative_ids.some((id) => matches(item, id)))).length / records.length),
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
function indexBytes(backend) {
|
|
189
|
+
const root = path.join(process.env.XDG_DATA_HOME ?? path.join(os.homedir(), '.local', 'share'), 'aiwg', 'index', ...(backend === 'local' ? ['framework'] : ['fortemi-core', 'framework']));
|
|
190
|
+
let total = 0;
|
|
191
|
+
const visit = (target) => {
|
|
192
|
+
if (!fs.existsSync(target))
|
|
193
|
+
return;
|
|
194
|
+
const stat = fs.statSync(target);
|
|
195
|
+
if (stat.isFile())
|
|
196
|
+
total += stat.size;
|
|
197
|
+
else
|
|
198
|
+
for (const child of fs.readdirSync(target))
|
|
199
|
+
visit(path.join(target, child));
|
|
200
|
+
};
|
|
201
|
+
visit(root);
|
|
202
|
+
return total;
|
|
203
|
+
}
|
|
204
|
+
export async function evaluateDiscovery(options) {
|
|
205
|
+
const records = parseDiscoveryRelevanceJsonl(fs.readFileSync(options.fixturePath, 'utf8'));
|
|
206
|
+
validateOperationalCoverage(records);
|
|
207
|
+
const limit = options.limit ?? 10;
|
|
208
|
+
const entries = loadEntries(options.cwd, options.backend);
|
|
209
|
+
const fortemiExport = options.backend === 'fortemi-core' && options.strategy === 'lexical'
|
|
210
|
+
? loadFortemiCoreExport(options.cwd, 'framework')
|
|
211
|
+
: null;
|
|
212
|
+
if (fortemiExport && !fortemiExport.exported) {
|
|
213
|
+
throw new Error(fortemiExport.reason ?? 'Fortemi Core framework export is unavailable');
|
|
214
|
+
}
|
|
215
|
+
const resultSets = [];
|
|
216
|
+
const latencies = [];
|
|
217
|
+
let peakRss = process.memoryUsage().rss;
|
|
218
|
+
for (const record of records) {
|
|
219
|
+
const start = performance.now();
|
|
220
|
+
resultSets.push(options.strategy === 'lexical'
|
|
221
|
+
? options.backend === 'fortemi-core'
|
|
222
|
+
? fortemiStaticRank(fortemiExport.exported, record, limit)
|
|
223
|
+
: await currentRank(options.cwd, record, options.backend, limit)
|
|
224
|
+
: prototypeRank(entries.filter((entry) => entry.type === record.target_type), record.query, options.strategy, limit));
|
|
225
|
+
latencies.push(performance.now() - start);
|
|
226
|
+
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
227
|
+
}
|
|
228
|
+
const metrics = calculateDiscoveryMetrics(records, resultSets);
|
|
229
|
+
const cpus = os.cpus();
|
|
230
|
+
let parity;
|
|
231
|
+
if (options.backend === 'fortemi-core' && options.strategy === 'lexical') {
|
|
232
|
+
const localSets = [];
|
|
233
|
+
for (const record of records)
|
|
234
|
+
localSets.push(await currentRank(options.cwd, record, 'local', limit));
|
|
235
|
+
const differingQueries = [];
|
|
236
|
+
let topOneAgreements = 0;
|
|
237
|
+
let topFiveOverlap = 0;
|
|
238
|
+
for (let index = 0; index < records.length; index++) {
|
|
239
|
+
const fortemi = resultSets[index];
|
|
240
|
+
const local = localSets[index];
|
|
241
|
+
const key = (item) => item ? `${item.type}:${item.name}`.toLowerCase() : '';
|
|
242
|
+
if (key(fortemi[0]) === key(local[0]))
|
|
243
|
+
topOneAgreements++;
|
|
244
|
+
else
|
|
245
|
+
differingQueries.push(records[index].id);
|
|
246
|
+
const localFive = new Set(local.slice(0, 5).map((item) => key(item)));
|
|
247
|
+
topFiveOverlap += fortemi.slice(0, 5).filter((item) => localFive.has(key(item))).length / Math.max(1, Math.min(5, local.length, fortemi.length));
|
|
248
|
+
}
|
|
249
|
+
parity = {
|
|
250
|
+
compared_to: 'local:lexical',
|
|
251
|
+
top_1_agreement: round(topOneAgreements / records.length),
|
|
252
|
+
top_5_overlap: round(topFiveOverlap / records.length),
|
|
253
|
+
differing_queries: differingQueries,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
return {
|
|
257
|
+
schema: DISCOVERY_EVAL_REPORT_SCHEMA,
|
|
258
|
+
corpus: { path: path.relative(options.cwd, options.fixturePath).replace(/\\/g, '/'), schema: DISCOVERY_EVAL_SCHEMA, query_count: records.length },
|
|
259
|
+
configuration: { backend: options.backend, strategy: options.strategy, limit, graph: 'framework' },
|
|
260
|
+
hardware: { platform: os.platform(), arch: os.arch(), cpu_model: cpus[0]?.model ?? 'unknown', logical_cpus: cpus.length, total_memory_bytes: os.totalmem(), node: process.version },
|
|
261
|
+
metrics,
|
|
262
|
+
performance: { p50_latency_ms: round(percentile(latencies, 0.5), 3), p95_latency_ms: round(percentile(latencies, 0.95), 3), index_bytes: indexBytes(options.backend), peak_resident_memory_bytes: peakRss },
|
|
263
|
+
...(parity ? { parity } : {}),
|
|
264
|
+
adoption_gate: {
|
|
265
|
+
baseline: 'local:lexical', no_per_type_hit_at_3_regression: null, aggregate_mrr_improvement: null,
|
|
266
|
+
mrr_95pct_confidence_interval: null, latency_ceiling_ms: options.latencyCeilingMs ?? 250,
|
|
267
|
+
storage_ceiling_ratio: options.storageCeilingRatio ?? 2, clears_gate: null,
|
|
268
|
+
decision: 'Run the full strategy matrix and compare against local:lexical before adoption.',
|
|
269
|
+
},
|
|
270
|
+
queries: records.map((record, index) => {
|
|
271
|
+
const rank = resultSets[index].findIndex((item) => record.relevant_ids.some((id) => matches(item, id)));
|
|
272
|
+
return { id: record.id, target_type: record.target_type, relevant_rank: rank < 0 ? null : rank + 1, reciprocal_rank: rank < 0 ? 0 : round(1 / (rank + 1)), latency_ms: round(latencies[index], 3), results: resultSets[index] };
|
|
273
|
+
}),
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
export function formatDiscoveryEvalSummary(report) {
|
|
277
|
+
const m = report.metrics;
|
|
278
|
+
return [
|
|
279
|
+
`Discovery evaluation: ${report.configuration.backend}:${report.configuration.strategy}`,
|
|
280
|
+
`Corpus: ${report.corpus.query_count} queries (${report.corpus.path})`, '',
|
|
281
|
+
'Metric Value', `Hit@1 ${m.hit_at_1.toFixed(4)}`, `Hit@3 ${m.hit_at_3.toFixed(4)}`,
|
|
282
|
+
`Hit@5 ${m.hit_at_5.toFixed(4)}`, `MRR ${m.mrr.toFixed(4)}`, `nDCG@10 ${m.ndcg_at_10.toFixed(4)}`,
|
|
283
|
+
`Hard-negative@5 ${m.hard_negative_intrusion_at_5.toFixed(4)}`, `p50 latency ${report.performance.p50_latency_ms.toFixed(3)} ms`,
|
|
284
|
+
`p95 latency ${report.performance.p95_latency_ms.toFixed(3)} ms`, `Index bytes ${report.performance.index_bytes}`,
|
|
285
|
+
`Peak resident memory ${report.performance.peak_resident_memory_bytes}`, '', 'Per-type Hit@3:',
|
|
286
|
+
...Object.entries(m.per_type_recall_at_3).map(([type, value]) => ` ${type.padEnd(10)} ${value.toFixed(4)}`), '',
|
|
287
|
+
`Decision: ${report.adoption_gate.decision}`,
|
|
288
|
+
].join('\n');
|
|
289
|
+
}
|
|
290
|
+
//# sourceMappingURL=discovery-eval.js.map
|
|
@@ -19,7 +19,7 @@ function includesAny(text, terms) {
|
|
|
19
19
|
}
|
|
20
20
|
return matches;
|
|
21
21
|
}
|
|
22
|
-
function scoreStaticRecord(record, terms) {
|
|
22
|
+
export function scoreStaticRecord(record, terms) {
|
|
23
23
|
const fields = [
|
|
24
24
|
{ name: "title", text: record.title, weight: 3 },
|
|
25
25
|
{ name: "name", text: record.name ?? record.search?.name ?? "", weight: 3 },
|
|
@@ -5,7 +5,7 @@ async function loadFortemiShardConverter() {
|
|
|
5
5
|
let core;
|
|
6
6
|
try {
|
|
7
7
|
core = (await import(
|
|
8
|
-
/* @vite-ignore */ "@fortemi/core/aiwg-index"));
|
|
8
|
+
/* @vite-ignore */ "@fortemi/core/aiwg-index-shard"));
|
|
9
9
|
}
|
|
10
10
|
catch {
|
|
11
11
|
throw new Error("Portable Fortemi shard export requires @fortemi/core with aiwgFortemiIndexToKnowledgeShard.");
|
|
@@ -1740,14 +1740,18 @@ export async function showArtifact(cwd, params) {
|
|
|
1740
1740
|
try {
|
|
1741
1741
|
if (!webRelease)
|
|
1742
1742
|
throw new Error('verified web release context is unavailable');
|
|
1743
|
-
const {
|
|
1744
|
-
const
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1743
|
+
const { logicalIdFromFirstPartyPath, resolveAiwgResourceBytes } = await import('../resources/resolver.js');
|
|
1744
|
+
const logicalId = logicalIdFromFirstPartyPath(entry.path);
|
|
1745
|
+
if (!logicalId)
|
|
1746
|
+
throw new Error(`indexed path is not a first-party AIWG resource: ${entry.path}`);
|
|
1747
|
+
const resolved = await resolveAiwgResourceBytes(logicalId, {
|
|
1748
|
+
source: 'web',
|
|
1749
|
+
frameworkRoot: aiwgRoot ?? cwd,
|
|
1750
|
+
webRelease,
|
|
1751
|
+
webReleaseOptions: params.webReleaseOptions,
|
|
1748
1752
|
offline: params.offline,
|
|
1749
1753
|
});
|
|
1750
|
-
content = bytes.toString('utf8');
|
|
1754
|
+
content = resolved.bytes.toString('utf8');
|
|
1751
1755
|
}
|
|
1752
1756
|
catch (error) {
|
|
1753
1757
|
console.error(`Error: AIWG web resource show failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
@@ -80,6 +80,7 @@ function displayHelp() {
|
|
|
80
80
|
helpGroup('DISCOVERY', [
|
|
81
81
|
['discover "<phrase>"', 'Find skills/agents/commands/rules by capability'],
|
|
82
82
|
['show <type> <name>', 'Stream the body of an indexed artifact'],
|
|
83
|
+
['versions <list|resolve|show>', 'Browse and resolve signed AIWG web resource releases'],
|
|
83
84
|
['index <subcommand>', 'Manage the artifact index (build/query/discover/deps/stats)'],
|
|
84
85
|
['artifacts move --to <path>', 'Move/rename the project AIWG artifact root and reindex'],
|
|
85
86
|
]);
|
|
@@ -120,7 +121,7 @@ function displayHelp() {
|
|
|
120
121
|
['doctor', 'Check installation health'],
|
|
121
122
|
['version', 'Show version and channel info'],
|
|
122
123
|
['refresh', 'Update AIWG and redeploy frameworks (formerly: sync)'],
|
|
123
|
-
['update', '
|
|
124
|
+
['update', 'Update the active installation and re-deploy installed frameworks (alias: upgrade)'],
|
|
124
125
|
['help', 'Show this help message'],
|
|
125
126
|
]);
|
|
126
127
|
helpGroup('CHANNEL', [
|
|
@@ -54,6 +54,7 @@ import { cockpitHandler } from './cockpit.js';
|
|
|
54
54
|
import { commandLogHandler } from './command-log.js';
|
|
55
55
|
import { skillUsageHandler } from './skill-usage.js';
|
|
56
56
|
import { modelsHandler } from './models.js';
|
|
57
|
+
import { versionsHandler } from './resource-versions.js';
|
|
57
58
|
// Re-export individual handlers
|
|
58
59
|
export {
|
|
59
60
|
// Maintenance
|
|
@@ -65,7 +66,7 @@ newBundleHandler, quickrefHandler, newProjectHandler,
|
|
|
65
66
|
// Workspace
|
|
66
67
|
statusHandler, wizardHandler, migrateWorkspaceHandler, rollbackWorkspaceHandler,
|
|
67
68
|
// Subcommands
|
|
68
|
-
mcpHandler, catalogHandler, modelsHandler, indexHandler, artifactsHandler, corpusHandler, discoverHandler, showHandler, featuresHandler, skillsHandler, configHandler, opsHandler, storageHandler, activityLogHandler, commandLogHandler, skillUsageHandler, kbHandler, memoryHandler, reflectionsHandler, provenanceHandler, researchStoreHandler, researchQueryHandler, runtimeInfoHandler, agentcardHandler,
|
|
69
|
+
mcpHandler, catalogHandler, modelsHandler, versionsHandler, indexHandler, artifactsHandler, corpusHandler, discoverHandler, showHandler, featuresHandler, skillsHandler, configHandler, opsHandler, storageHandler, activityLogHandler, commandLogHandler, skillUsageHandler, kbHandler, memoryHandler, reflectionsHandler, provenanceHandler, researchStoreHandler, researchQueryHandler, runtimeInfoHandler, agentcardHandler,
|
|
69
70
|
// Agentic Tools (RLM)
|
|
70
71
|
chunkHandler, fanoutHandler, rlmPrepHandler, rlmSearchHandler, rlmStatusCliHandler, rlmCacheHandler,
|
|
71
72
|
// Utilities
|
|
@@ -139,6 +140,7 @@ export const allHandlers = [
|
|
|
139
140
|
mcpHandler,
|
|
140
141
|
catalogHandler,
|
|
141
142
|
modelsHandler,
|
|
143
|
+
versionsHandler,
|
|
142
144
|
indexHandler,
|
|
143
145
|
artifactsHandler,
|
|
144
146
|
corpusHandler,
|