@aiwg/cli 2026.8.8 → 2026.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -8
- package/THIRD_PARTY_NOTICES.md +35 -0
- package/agentic/code/providers/capability-matrix.yaml +3 -3
- package/bin/aiwg.mjs +125 -0
- package/dist/src/artifacts/backends/graphology-backend.js +4 -3
- package/dist/src/artifacts/backends/sqlite-backend.js +4 -5
- package/dist/src/artifacts/cli.js +2 -2
- package/dist/src/artifacts/corpus-tools/cli.js +27 -0
- package/dist/src/artifacts/corpus-tools/profile-embed.js +3 -2
- package/dist/src/artifacts/corpus-tools/retrieval-lab.js +356 -0
- package/dist/src/artifacts/discover-facets.js +2 -2
- package/dist/src/artifacts/embedding-index.js +9 -8
- package/dist/src/artifacts/graph-backend.js +2 -2
- package/dist/src/artifacts/query-engine.js +21 -7
- package/dist/src/artifacts/repair.js +47 -0
- package/dist/src/artifacts/types.js +3 -3
- package/dist/src/cli/command-log.js +2 -2
- package/dist/src/cli/handlers/artifacts.js +50 -1
- package/dist/src/cli/handlers/cost-report.js +71 -0
- package/dist/src/cli/handlers/evidence.js +78 -0
- package/dist/src/cli/handlers/help.js +9 -0
- package/dist/src/cli/handlers/index.js +8 -3
- package/dist/src/cli/handlers/local-executor.js +4 -3
- package/dist/src/cli/handlers/refresh.js +20 -8
- package/dist/src/cli/handlers/regenerate.js +3 -3
- package/dist/src/cli/handlers/serve.js +15 -36
- package/dist/src/cli/handlers/setup-manifest.js +8 -1
- package/dist/src/cli/handlers/use.js +256 -70
- package/dist/src/cli/handlers/utilities.js +149 -0
- package/dist/src/cli/handlers/workspace.js +10 -0
- package/dist/src/cli/help-generator.js +2 -1
- package/dist/src/cli/router.js +4 -1
- package/dist/src/cli/services/deployment-verification.js +596 -0
- package/dist/src/cli/skill-usage.js +2 -2
- package/dist/src/cli/workflow-orchestrator.js +1 -1
- package/dist/src/cli/workspace-signals.js +2 -2
- package/dist/src/config/aiwg-config.js +54 -27
- package/dist/src/config/cli.js +3 -3
- package/dist/src/config/project-artifacts-health.js +2 -0
- package/dist/src/config/project-artifacts-health.mjs +123 -0
- package/dist/src/config/project-artifacts-runtime.mjs +16 -0
- package/dist/src/config/project-artifacts.js +2 -1
- package/dist/src/cost/fleet-report.js +329 -0
- package/dist/src/evidence/bundle.js +256 -0
- package/dist/src/extensions/commands/definitions.js +77 -25
- package/dist/src/extensions/deployment-registration.js +6 -4
- package/dist/src/features/catalog.js +26 -0
- package/dist/src/features/cli.js +1 -3
- package/dist/src/features/runtime.js +17 -1
- package/dist/src/issues/cli.js +91 -7
- package/dist/src/mcp/server.mjs +1 -1
- package/dist/src/ops/registry.js +2 -2
- package/dist/src/policy/authorization.js +2 -2
- package/dist/src/providers/capability-matrix.yaml +3 -3
- package/dist/src/providers/provider-definitions.js +7 -5
- package/dist/src/providers/provider-definitions.mjs +1 -1
- package/dist/src/serve/pty-bridge.js +2 -8
- package/dist/src/serve/screen-reader.js +3 -6
- package/dist/src/smiths/context-pipeline/aiwg-md.js +2 -2
- package/dist/src/smiths/context-pipeline/finalization.js +18 -5
- package/dist/src/smiths/context-pipeline/generator.js +2 -2
- package/dist/src/smiths/context-pipeline/workspace-context.js +16 -17
- package/package.json +2 -1
- package/tools/agents/deploy-agents.mjs +10 -11
- package/tools/agents/providers/base.mjs +47 -5
- package/tools/agents/providers/openclaw.mjs +5 -2
- package/tools/agents/providers/windsurf.mjs +13 -24
- package/tools/skills/deploy-skills-codex.mjs +21 -5
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/** Experimental hybrid corpus-retrieval benchmark. Never replaces research-query. @issue #2038 */
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { performance } from 'node:perf_hooks';
|
|
6
|
+
import { bm25Rank, tokenizeText } from '../fulltext.js';
|
|
7
|
+
import { loadGraphIndexFile } from '../index-reader.js';
|
|
8
|
+
import { runResearchQuery } from '../../research/query-cli.js';
|
|
9
|
+
export const RETRIEVAL_LAB_QUERY_SCHEMA = 'aiwg.corpus-retrieval-query/v1';
|
|
10
|
+
export const RETRIEVAL_LAB_CONCEPT_SCHEMA = 'aiwg.concept-scheme/v1';
|
|
11
|
+
export const RETRIEVAL_LAB_REPORT_SCHEMA = 'aiwg.corpus-retrieval-report/v1';
|
|
12
|
+
export const RETRIEVAL_LAB_STRATEGIES = ['research-query', 'direct-rg', 'vector', 'bm25', 'graph-ppr', 'hybrid-rrf'];
|
|
13
|
+
const round = (value, digits = 6) => {
|
|
14
|
+
const scale = 10 ** digits;
|
|
15
|
+
return Math.round(value * scale) / scale;
|
|
16
|
+
};
|
|
17
|
+
function stringArray(value, label, line) {
|
|
18
|
+
if (!Array.isArray(value) || value.length === 0 || value.some(item => typeof item !== 'string' || !item.trim())) {
|
|
19
|
+
throw new Error(`Retrieval query line ${line}: ${label} must be a non-empty string array`);
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
export function parseRetrievalQueries(content) {
|
|
24
|
+
const queries = [];
|
|
25
|
+
const ids = new Set();
|
|
26
|
+
for (const [offset, raw] of content.split(/\r?\n/).entries()) {
|
|
27
|
+
if (!raw.trim())
|
|
28
|
+
continue;
|
|
29
|
+
const line = offset + 1;
|
|
30
|
+
let value;
|
|
31
|
+
try {
|
|
32
|
+
value = JSON.parse(raw);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
throw new Error(`Retrieval query line ${line}: malformed JSON`);
|
|
36
|
+
}
|
|
37
|
+
if (value.schema !== RETRIEVAL_LAB_QUERY_SCHEMA)
|
|
38
|
+
throw new Error(`Retrieval query line ${line}: unsupported schema`);
|
|
39
|
+
if (typeof value.id !== 'string' || !value.id.trim() || ids.has(value.id))
|
|
40
|
+
throw new Error(`Retrieval query line ${line}: id must be unique and non-empty`);
|
|
41
|
+
if (typeof value.question !== 'string' || !value.question.trim())
|
|
42
|
+
throw new Error(`Retrieval query line ${line}: question must be non-empty`);
|
|
43
|
+
ids.add(value.id);
|
|
44
|
+
queries.push({
|
|
45
|
+
schema: RETRIEVAL_LAB_QUERY_SCHEMA,
|
|
46
|
+
id: value.id,
|
|
47
|
+
question: value.question,
|
|
48
|
+
expected_ids: stringArray(value.expected_ids, 'expected_ids', line),
|
|
49
|
+
expected_evidence: stringArray(value.expected_evidence, 'expected_evidence', line),
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (!queries.length)
|
|
53
|
+
throw new Error('Retrieval query fixture contains no queries');
|
|
54
|
+
return queries;
|
|
55
|
+
}
|
|
56
|
+
function normalizedScheme(scheme) {
|
|
57
|
+
return {
|
|
58
|
+
schema: RETRIEVAL_LAB_CONCEPT_SCHEMA,
|
|
59
|
+
id: scheme.id,
|
|
60
|
+
concepts: [...scheme.concepts].map(concept => ({
|
|
61
|
+
id: concept.id,
|
|
62
|
+
prefLabel: concept.prefLabel,
|
|
63
|
+
...(concept.altLabels?.length ? { altLabels: [...concept.altLabels].sort() } : {}),
|
|
64
|
+
...(concept.broader?.length ? { broader: [...concept.broader].sort() } : {}),
|
|
65
|
+
...(concept.narrower?.length ? { narrower: [...concept.narrower].sort() } : {}),
|
|
66
|
+
...(concept.related?.length ? { related: [...concept.related].sort() } : {}),
|
|
67
|
+
})).sort((a, b) => a.id.localeCompare(b.id)),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
export function parseConceptScheme(content) {
|
|
71
|
+
let value;
|
|
72
|
+
try {
|
|
73
|
+
value = JSON.parse(content);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
throw new Error('Concept scheme is malformed JSON');
|
|
77
|
+
}
|
|
78
|
+
if (value.schema !== RETRIEVAL_LAB_CONCEPT_SCHEMA || typeof value.id !== 'string' || !value.id || !Array.isArray(value.concepts) || !value.concepts.length) {
|
|
79
|
+
throw new Error(`Concept scheme must use ${RETRIEVAL_LAB_CONCEPT_SCHEMA}`);
|
|
80
|
+
}
|
|
81
|
+
const ids = new Set();
|
|
82
|
+
for (const concept of value.concepts) {
|
|
83
|
+
const normalizedId = concept?.id?.toLowerCase();
|
|
84
|
+
if (!concept || typeof concept.id !== 'string' || !concept.id || typeof concept.prefLabel !== 'string' || !concept.prefLabel || ids.has(normalizedId)) {
|
|
85
|
+
throw new Error('Concept scheme contains a malformed or duplicate concept');
|
|
86
|
+
}
|
|
87
|
+
ids.add(normalizedId);
|
|
88
|
+
for (const relation of ['altLabels', 'broader', 'narrower', 'related']) {
|
|
89
|
+
if (concept[relation] !== undefined && (!Array.isArray(concept[relation]) || concept[relation].some(item => typeof item !== 'string'))) {
|
|
90
|
+
throw new Error(`Concept ${concept.id} has malformed ${relation}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
for (const concept of value.concepts) {
|
|
95
|
+
for (const relation of ['broader', 'narrower', 'related']) {
|
|
96
|
+
for (const target of concept[relation] ?? [])
|
|
97
|
+
if (!ids.has(target.toLowerCase()))
|
|
98
|
+
throw new Error(`Concept ${concept.id} references unknown ${relation} target ${target}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const scheme = normalizedScheme(value);
|
|
102
|
+
return { scheme, hash: createHash('sha256').update(JSON.stringify(scheme)).digest('hex') };
|
|
103
|
+
}
|
|
104
|
+
function documentId(entry) {
|
|
105
|
+
const text = [entry.name, entry.title, entry.path].filter(Boolean).join(' ');
|
|
106
|
+
return text.match(/\bREF-\d+\b/i)?.[0]?.toUpperCase() ?? path.basename(entry.path).replace(/\.[^.]+$/, '');
|
|
107
|
+
}
|
|
108
|
+
function isResearch(entry) {
|
|
109
|
+
const candidate = `${entry.type} ${entry.path}`.toLowerCase().replaceAll('\\', '/');
|
|
110
|
+
return /research-ref|research-profile|research-view|research-synthesis|kb-page|\/research\/|\/kb\//.test(candidate);
|
|
111
|
+
}
|
|
112
|
+
function readBody(root, entry) {
|
|
113
|
+
try {
|
|
114
|
+
return fs.readFileSync(path.resolve(root, entry.path), 'utf8');
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return '';
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function featureVector(text, dimensions = 128) {
|
|
121
|
+
const vector = new Array(dimensions).fill(0);
|
|
122
|
+
for (const token of tokenizeText(text)) {
|
|
123
|
+
const digest = createHash('sha256').update(token).digest();
|
|
124
|
+
const index = digest.readUInt16BE(0) % dimensions;
|
|
125
|
+
vector[index] += (digest[2] & 1) ? 1 : -1;
|
|
126
|
+
}
|
|
127
|
+
const norm = Math.sqrt(vector.reduce((sum, item) => sum + item * item, 0));
|
|
128
|
+
return norm ? vector.map(item => item / norm) : vector;
|
|
129
|
+
}
|
|
130
|
+
function cosine(left, right) {
|
|
131
|
+
return left.reduce((sum, value, index) => sum + value * (right[index] ?? 0), 0);
|
|
132
|
+
}
|
|
133
|
+
function conceptText(concept) {
|
|
134
|
+
return [concept.prefLabel, ...(concept.altLabels ?? [])].join(' ');
|
|
135
|
+
}
|
|
136
|
+
function loadDocuments(root, scheme) {
|
|
137
|
+
const index = loadGraphIndexFile(root, 'metadata.json', 'project');
|
|
138
|
+
if (!index)
|
|
139
|
+
throw new Error('Project corpus index is missing; run `aiwg index build --graph project`');
|
|
140
|
+
const concepts = new Map(scheme.concepts.map(concept => [concept.id.toLowerCase(), concept]));
|
|
141
|
+
return Object.values(index.entries).filter(isResearch).map(entry => {
|
|
142
|
+
const body = readBody(root, entry);
|
|
143
|
+
const documentConcepts = entry.tags.map(tag => tag.toLowerCase()).filter(tag => concepts.has(tag));
|
|
144
|
+
const expanded = documentConcepts.map(id => conceptText(concepts.get(id))).join(' ');
|
|
145
|
+
const text = [entry.title, entry.name ?? '', entry.summary, entry.tags.join(' '), expanded, body].join('\n');
|
|
146
|
+
return { entry, id: documentId(entry), body, text, concepts: documentConcepts, vector: featureVector(text) };
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
function matchedTerms(query, text) {
|
|
150
|
+
const available = new Set(tokenizeText(text));
|
|
151
|
+
return [...new Set(tokenizeText(query))].filter(term => available.has(term)).sort();
|
|
152
|
+
}
|
|
153
|
+
function hit(document, score, query, graphConcepts = []) {
|
|
154
|
+
return { id: document.id, path: document.entry.path, title: document.entry.title, score: round(score), matched_terms: matchedTerms(query, document.text), graph_concepts: graphConcepts.sort() };
|
|
155
|
+
}
|
|
156
|
+
function directRank(documents, query) {
|
|
157
|
+
const terms = [...new Set(tokenizeText(query))];
|
|
158
|
+
return documents.map(document => ({ document, score: terms.reduce((sum, term) => sum + (document.text.toLowerCase().includes(term) ? 1 : 0), 0) }))
|
|
159
|
+
.filter(item => item.score > 0).sort((a, b) => b.score - a.score || a.document.id.localeCompare(b.document.id))
|
|
160
|
+
.map(item => hit(item.document, item.score / Math.max(1, terms.length), query));
|
|
161
|
+
}
|
|
162
|
+
function vectorRank(documents, query) {
|
|
163
|
+
const queryVector = featureVector(query);
|
|
164
|
+
return documents.map(document => ({ document, score: Math.max(0, cosine(queryVector, document.vector)) }))
|
|
165
|
+
.filter(item => item.score > 0).sort((a, b) => b.score - a.score || a.document.id.localeCompare(b.document.id))
|
|
166
|
+
.map(item => hit(item.document, item.score, query));
|
|
167
|
+
}
|
|
168
|
+
function lexicalRank(documents, query) {
|
|
169
|
+
const byId = new Map(documents.map(document => [document.id, document]));
|
|
170
|
+
return bm25Rank(documents.map(document => ({ id: document.id, text: document.text })), query)
|
|
171
|
+
.map(item => hit(byId.get(item.id), item.score, query));
|
|
172
|
+
}
|
|
173
|
+
function graphRank(documents, scheme, query, vector, lexical) {
|
|
174
|
+
const concepts = new Map(scheme.concepts.map(concept => [concept.id.toLowerCase(), concept]));
|
|
175
|
+
const documentById = new Map(documents.map(document => [document.id, document]));
|
|
176
|
+
const restart = new Map();
|
|
177
|
+
for (const concept of concepts.values()) {
|
|
178
|
+
const overlap = matchedTerms(query, conceptText(concept)).length;
|
|
179
|
+
if (overlap)
|
|
180
|
+
restart.set(concept.id.toLowerCase(), overlap);
|
|
181
|
+
}
|
|
182
|
+
for (const candidate of [...vector.slice(0, 5), ...lexical.slice(0, 5)]) {
|
|
183
|
+
for (const concept of documentById.get(candidate.id)?.concepts ?? [])
|
|
184
|
+
restart.set(concept, (restart.get(concept) ?? 0) + candidate.score * 0.25);
|
|
185
|
+
}
|
|
186
|
+
const totalRestart = [...restart.values()].reduce((sum, value) => sum + value, 0);
|
|
187
|
+
if (!totalRestart)
|
|
188
|
+
return [];
|
|
189
|
+
for (const [id, value] of restart)
|
|
190
|
+
restart.set(id, value / totalRestart);
|
|
191
|
+
let scores = new Map(restart);
|
|
192
|
+
const weights = { broader: 0.8, narrower: 1, related: 0.6 };
|
|
193
|
+
for (let iteration = 0; iteration < 20; iteration++) {
|
|
194
|
+
const next = new Map();
|
|
195
|
+
for (const [id, value] of restart)
|
|
196
|
+
next.set(id, value * 0.15);
|
|
197
|
+
for (const [source, score] of scores) {
|
|
198
|
+
const concept = concepts.get(source);
|
|
199
|
+
if (!concept)
|
|
200
|
+
continue;
|
|
201
|
+
const edges = Object.keys(weights).flatMap(type => (concept[type] ?? []).map(target => ({ target: target.toLowerCase(), weight: weights[type] })));
|
|
202
|
+
const denominator = edges.reduce((sum, edge) => sum + edge.weight, 0);
|
|
203
|
+
if (!denominator) {
|
|
204
|
+
next.set(source, (next.get(source) ?? 0) + score * 0.85);
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
for (const edge of edges) {
|
|
208
|
+
const target = concepts.get(edge.target);
|
|
209
|
+
const degree = (target?.broader?.length ?? 0) + (target?.narrower?.length ?? 0) + (target?.related?.length ?? 0);
|
|
210
|
+
const specificity = 1 / Math.log2(2 + degree);
|
|
211
|
+
next.set(edge.target, (next.get(edge.target) ?? 0) + score * 0.85 * (edge.weight / denominator) * specificity);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
scores = next;
|
|
215
|
+
}
|
|
216
|
+
return documents.map(document => {
|
|
217
|
+
const contributing = document.concepts.filter(id => (scores.get(id) ?? 0) > 0);
|
|
218
|
+
const score = contributing.reduce((sum, id) => sum + (scores.get(id) ?? 0), 0) / Math.max(1, document.concepts.length);
|
|
219
|
+
return { document, score, contributing };
|
|
220
|
+
}).filter(item => item.score > 0).sort((a, b) => b.score - a.score || a.document.id.localeCompare(b.document.id))
|
|
221
|
+
.map(item => hit(item.document, item.score, query, item.contributing));
|
|
222
|
+
}
|
|
223
|
+
function rrf(lists, documents, query) {
|
|
224
|
+
const scores = new Map();
|
|
225
|
+
for (const list of lists)
|
|
226
|
+
list.forEach((item, rank) => scores.set(item.id, (scores.get(item.id) ?? 0) + 1 / (60 + rank + 1)));
|
|
227
|
+
const byId = new Map(documents.map(document => [document.id, document]));
|
|
228
|
+
return [...scores.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([id, score]) => {
|
|
229
|
+
const document = byId.get(id);
|
|
230
|
+
const graphConcepts = lists[2]?.find(item => item.id === id)?.graph_concepts ?? [];
|
|
231
|
+
return hit(document, score, query, graphConcepts);
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
function percentile(values, percentileValue) {
|
|
235
|
+
if (!values.length)
|
|
236
|
+
return 0;
|
|
237
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
238
|
+
return sorted[Math.ceil(sorted.length * percentileValue) - 1] ?? sorted[sorted.length - 1];
|
|
239
|
+
}
|
|
240
|
+
function strategyReport(queries, results, latencies) {
|
|
241
|
+
const ranks = queries.map((query, index) => {
|
|
242
|
+
const rank = results[index].findIndex(item => query.expected_ids.some(expected => expected.toLowerCase() === item.id.toLowerCase()));
|
|
243
|
+
return rank < 0 ? null : rank + 1;
|
|
244
|
+
});
|
|
245
|
+
const at = (limit) => round(ranks.filter(rank => rank !== null && rank <= limit).length / queries.length);
|
|
246
|
+
return {
|
|
247
|
+
metrics: {
|
|
248
|
+
hit_at_1: at(1), hit_at_3: at(3), hit_at_5: at(5),
|
|
249
|
+
mrr: round(ranks.reduce((sum, rank) => sum + (rank ? 1 / rank : 0), 0) / queries.length),
|
|
250
|
+
p95_latency_ms: round(percentile(latencies, 0.95), 3),
|
|
251
|
+
},
|
|
252
|
+
failures: queries.map((query, index) => ({ query, results: results[index] })).filter(item => !item.results.slice(0, 5).some(result => item.query.expected_ids.some(expected => expected.toLowerCase() === result.id.toLowerCase())))
|
|
253
|
+
.map(item => ({ query_id: item.query.id, expected_ids: item.query.expected_ids, returned_ids: item.results.slice(0, 5).map(result => result.id) })),
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
function selectionDiagnostics(results) {
|
|
257
|
+
const scores = results.slice(0, 5).map(item => item.score).filter(score => score > 0);
|
|
258
|
+
const total = scores.reduce((sum, score) => sum + score, 0);
|
|
259
|
+
if (!total)
|
|
260
|
+
return { confidence: 0, dispersion: 1 };
|
|
261
|
+
const probabilities = scores.map(score => score / total);
|
|
262
|
+
const entropy = -probabilities.reduce((sum, value) => sum + value * Math.log(value), 0);
|
|
263
|
+
const maxEntropy = Math.log(Math.max(2, probabilities.length));
|
|
264
|
+
return { confidence: round(probabilities[0]), dispersion: round(entropy / maxEntropy) };
|
|
265
|
+
}
|
|
266
|
+
export async function runRetrievalLab(options) {
|
|
267
|
+
const queries = parseRetrievalQueries(fs.readFileSync(options.queriesPath, 'utf8'));
|
|
268
|
+
const { scheme, hash } = parseConceptScheme(fs.readFileSync(options.conceptsPath, 'utf8'));
|
|
269
|
+
if (options.expectedSchemeHash && options.expectedSchemeHash.toLowerCase() !== hash) {
|
|
270
|
+
throw new Error(`Concept scheme drift invalidated benchmark: expected ${options.expectedSchemeHash.toLowerCase()}, observed ${hash}`);
|
|
271
|
+
}
|
|
272
|
+
const documents = loadDocuments(options.root, scheme);
|
|
273
|
+
if (!documents.length)
|
|
274
|
+
throw new Error('Project corpus index contains no research documents');
|
|
275
|
+
const limit = options.limit ?? 10;
|
|
276
|
+
const matrix = Object.fromEntries(RETRIEVAL_LAB_STRATEGIES.map(strategy => [strategy, []]));
|
|
277
|
+
const latencies = Object.fromEntries(RETRIEVAL_LAB_STRATEGIES.map(strategy => [strategy, []]));
|
|
278
|
+
for (const query of queries) {
|
|
279
|
+
let started = performance.now();
|
|
280
|
+
const current = await runResearchQuery(options.root, { question: query.question, backend: 'local', graph: 'project', depth: 'thorough', maxSources: limit });
|
|
281
|
+
matrix['research-query'].push(current.sources.map(source => ({ id: source.id, path: source.path, title: source.title, score: source.score, matched_terms: matchedTerms(query.question, [source.title, source.summary, source.tags.join(' ')].join(' ')), graph_concepts: [] })));
|
|
282
|
+
latencies['research-query'].push(performance.now() - started);
|
|
283
|
+
started = performance.now();
|
|
284
|
+
const direct = directRank(documents, query.question).slice(0, limit);
|
|
285
|
+
latencies['direct-rg'].push(performance.now() - started);
|
|
286
|
+
matrix['direct-rg'].push(direct);
|
|
287
|
+
const hybridStarted = performance.now();
|
|
288
|
+
started = hybridStarted;
|
|
289
|
+
const vector = vectorRank(documents, query.question).slice(0, limit);
|
|
290
|
+
latencies.vector.push(performance.now() - started);
|
|
291
|
+
matrix.vector.push(vector);
|
|
292
|
+
started = performance.now();
|
|
293
|
+
const lexical = lexicalRank(documents, query.question).slice(0, limit);
|
|
294
|
+
latencies.bm25.push(performance.now() - started);
|
|
295
|
+
matrix.bm25.push(lexical);
|
|
296
|
+
started = performance.now();
|
|
297
|
+
const graph = graphRank(documents, scheme, query.question, vector, lexical).slice(0, limit);
|
|
298
|
+
latencies['graph-ppr'].push(performance.now() - started);
|
|
299
|
+
matrix['graph-ppr'].push(graph);
|
|
300
|
+
const hybrid = rrf([vector, lexical, graph], documents, query.question).slice(0, limit);
|
|
301
|
+
latencies['hybrid-rrf'].push(performance.now() - hybridStarted);
|
|
302
|
+
matrix['hybrid-rrf'].push(hybrid);
|
|
303
|
+
}
|
|
304
|
+
const strategies = Object.fromEntries(RETRIEVAL_LAB_STRATEGIES.map(strategy => [strategy, strategyReport(queries, matrix[strategy], latencies[strategy])]));
|
|
305
|
+
const hybrid = strategies['hybrid-rrf'].metrics;
|
|
306
|
+
const baselines = [strategies['research-query'].metrics, strategies['direct-rg'].metrics];
|
|
307
|
+
const qualityBeats = baselines.every(baseline => hybrid.hit_at_5 >= baseline.hit_at_5 && hybrid.mrr >= baseline.mrr)
|
|
308
|
+
&& baselines.some(baseline => hybrid.hit_at_5 > baseline.hit_at_5 || hybrid.mrr > baseline.mrr);
|
|
309
|
+
const acceptableLatency = hybrid.p95_latency_ms <= (options.latencyCeilingMs ?? 250);
|
|
310
|
+
const clears = qualityBeats && acceptableLatency;
|
|
311
|
+
const documentById = new Map(documents.map(document => [document.id, document]));
|
|
312
|
+
return {
|
|
313
|
+
schema: RETRIEVAL_LAB_REPORT_SCHEMA,
|
|
314
|
+
status: 'complete',
|
|
315
|
+
configuration: {
|
|
316
|
+
graph: 'project', vector: 'local-feature-hash-v1', lexical: 'bm25', graph_walk: 'typed-ppr-specificity-restart', fusion: 'rrf-k60',
|
|
317
|
+
concept_scheme_id: scheme.id, concept_scheme_hash: hash, expected_concept_scheme_hash: options.expectedSchemeHash?.toLowerCase() ?? null,
|
|
318
|
+
},
|
|
319
|
+
query_count: queries.length,
|
|
320
|
+
document_count: documents.length,
|
|
321
|
+
strategies,
|
|
322
|
+
queries: queries.map((query, index) => {
|
|
323
|
+
const results = Object.fromEntries(RETRIEVAL_LAB_STRATEGIES.map(strategy => [strategy, matrix[strategy][index]]));
|
|
324
|
+
const top = results['hybrid-rrf'].slice(0, 5);
|
|
325
|
+
const unsupported = top.filter(item => !item.matched_terms.length && !item.graph_concepts.length).map(item => item.id);
|
|
326
|
+
const selectedText = top.map(item => documentById.get(item.id)?.text ?? '').join('\n').toLowerCase();
|
|
327
|
+
const missingEvidence = query.expected_evidence.filter(evidence => !selectedText.includes(evidence.toLowerCase()));
|
|
328
|
+
return {
|
|
329
|
+
id: query.id, expected_ids: query.expected_ids, results,
|
|
330
|
+
source_selection: selectionDiagnostics(results['hybrid-rrf']),
|
|
331
|
+
faithfulness: { passed: unsupported.length === 0 && missingEvidence.length === 0, unsupported_top_ids: unsupported, missing_expected_evidence: missingEvidence },
|
|
332
|
+
};
|
|
333
|
+
}),
|
|
334
|
+
adoption_gate: {
|
|
335
|
+
replaces_current_query: false,
|
|
336
|
+
baseline: 'research-query', quality_beats_baselines: qualityBeats, acceptable_latency: acceptableLatency, clears_gate: clears,
|
|
337
|
+
decision: clears
|
|
338
|
+
? 'Candidate cleared the lab gate; current research-query remains unchanged pending an explicit adoption decision.'
|
|
339
|
+
: 'HOLD: hybrid retrieval did not beat both baselines within the latency ceiling; current research-query remains unchanged.',
|
|
340
|
+
},
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
export function renderRetrievalLab(report) {
|
|
344
|
+
const lines = [
|
|
345
|
+
`Corpus retrieval lab: ${report.query_count} queries / ${report.document_count} documents`,
|
|
346
|
+
`Concept scheme: ${report.configuration.concept_scheme_id} (${report.configuration.concept_scheme_hash})`, '',
|
|
347
|
+
'Strategy Hit@1 Hit@3 Hit@5 MRR p95 ms',
|
|
348
|
+
];
|
|
349
|
+
for (const strategy of RETRIEVAL_LAB_STRATEGIES) {
|
|
350
|
+
const metrics = report.strategies[strategy].metrics;
|
|
351
|
+
lines.push(`${strategy.padEnd(16)} ${metrics.hit_at_1.toFixed(3)} ${metrics.hit_at_3.toFixed(3)} ${metrics.hit_at_5.toFixed(3)} ${metrics.mrr.toFixed(3)} ${metrics.p95_latency_ms.toFixed(3)}`);
|
|
352
|
+
}
|
|
353
|
+
lines.push('', `Decision: ${report.adoption_gate.decision}`);
|
|
354
|
+
return `${lines.join('\n')}\n`;
|
|
355
|
+
}
|
|
356
|
+
//# sourceMappingURL=retrieval-lab.js.map
|
|
@@ -213,7 +213,7 @@ function facetActivation(phrase, entry) {
|
|
|
213
213
|
matchedIntent: raw,
|
|
214
214
|
};
|
|
215
215
|
}
|
|
216
|
-
if (p.includes(intent) || intent.includes(p)) {
|
|
216
|
+
if (p.includes(intent) || (pTokens.length > 1 && intent.includes(p))) {
|
|
217
217
|
if (FLOOR_STRONG > best) {
|
|
218
218
|
best = FLOOR_STRONG;
|
|
219
219
|
matchedIntent = raw;
|
|
@@ -223,7 +223,7 @@ function facetActivation(phrase, entry) {
|
|
|
223
223
|
const iTokens = intent.split(' ').filter(Boolean);
|
|
224
224
|
if (iTokens.length > 1 && pTokens.length > 0) {
|
|
225
225
|
const hits = iTokens.filter((t) => pTokens.includes(t)).length;
|
|
226
|
-
if (hits >= Math.ceil(iTokens.length / 2) && FLOOR_OVERLAP > best) {
|
|
226
|
+
if (hits >= 2 && hits >= Math.ceil(iTokens.length / 2) && FLOOR_OVERLAP > best) {
|
|
227
227
|
best = FLOOR_OVERLAP;
|
|
228
228
|
matchedIntent = raw;
|
|
229
229
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* dense vectors using a small local model and stores them in an HNSW index for
|
|
7
7
|
* fast similarity queries.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
9
|
+
* Enable with: aiwg features install embeddings
|
|
10
10
|
*
|
|
11
11
|
* @implements #730
|
|
12
12
|
* @source @src/artifacts/types.ts
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import fs from 'fs';
|
|
16
16
|
import path from 'path';
|
|
17
|
+
import { loadFeaturePackage } from '../features/runtime.js';
|
|
17
18
|
/**
|
|
18
19
|
* Default embedding model (all-MiniLM-L6-v2: ~22MB, 384 dims, ~5ms/embedding on CPU)
|
|
19
20
|
*/
|
|
@@ -141,13 +142,13 @@ async function embedEntryVector(embed, entry, options) {
|
|
|
141
142
|
export async function checkEmbeddingDeps() {
|
|
142
143
|
const missing = [];
|
|
143
144
|
try {
|
|
144
|
-
await (
|
|
145
|
+
await loadFeaturePackage('@xenova/transformers');
|
|
145
146
|
}
|
|
146
147
|
catch {
|
|
147
148
|
missing.push('@xenova/transformers');
|
|
148
149
|
}
|
|
149
150
|
try {
|
|
150
|
-
await (
|
|
151
|
+
await loadFeaturePackage('hnswlib-node');
|
|
151
152
|
}
|
|
152
153
|
catch {
|
|
153
154
|
missing.push('hnswlib-node');
|
|
@@ -168,9 +169,9 @@ export async function checkEmbeddingDeps() {
|
|
|
168
169
|
* @returns Number of entries embedded
|
|
169
170
|
*/
|
|
170
171
|
export async function buildEmbeddingIndex(entries, outputDir, model = DEFAULT_EMBEDDING_MODEL, options = {}) {
|
|
171
|
-
const transformersMod = await (
|
|
172
|
+
const transformersMod = await loadFeaturePackage('@xenova/transformers');
|
|
172
173
|
const { pipeline } = transformersMod;
|
|
173
|
-
const hnswlib = await (
|
|
174
|
+
const hnswlib = await loadFeaturePackage('hnswlib-node');
|
|
174
175
|
const HierarchicalNSW = hnswlib.HierarchicalNSW ?? hnswlib.default?.HierarchicalNSW;
|
|
175
176
|
if (!HierarchicalNSW) {
|
|
176
177
|
throw new Error('hnswlib-node: HierarchicalNSW not found in module exports');
|
|
@@ -233,9 +234,9 @@ export async function semanticQuery(query, indexDir, topK = 10) {
|
|
|
233
234
|
if (!manifest) {
|
|
234
235
|
throw new Error(`No embedding index found at ${indexDir}/embeddings/`);
|
|
235
236
|
}
|
|
236
|
-
const transformersMod = await (
|
|
237
|
+
const transformersMod = await loadFeaturePackage('@xenova/transformers');
|
|
237
238
|
const { pipeline } = transformersMod;
|
|
238
|
-
const hnswlib = await (
|
|
239
|
+
const hnswlib = await loadFeaturePackage('hnswlib-node');
|
|
239
240
|
const HierarchicalNSW = hnswlib.HierarchicalNSW ?? hnswlib.default?.HierarchicalNSW;
|
|
240
241
|
if (!HierarchicalNSW) {
|
|
241
242
|
throw new Error('hnswlib-node: HierarchicalNSW not found in module exports');
|
|
@@ -320,7 +321,7 @@ export async function dedupReport(indexDir, threshold = 0.92, topK = 5) {
|
|
|
320
321
|
if (!manifest) {
|
|
321
322
|
throw new Error(`No embedding index found at ${indexDir}/embeddings/`);
|
|
322
323
|
}
|
|
323
|
-
const hnswlib = await (
|
|
324
|
+
const hnswlib = await loadFeaturePackage('hnswlib-node');
|
|
324
325
|
const HierarchicalNSW = hnswlib.HierarchicalNSW ?? hnswlib.default?.HierarchicalNSW;
|
|
325
326
|
if (!HierarchicalNSW) {
|
|
326
327
|
throw new Error('hnswlib-node: HierarchicalNSW not found in module exports');
|
|
@@ -32,7 +32,7 @@ export async function createGraphBackend(type = 'json') {
|
|
|
32
32
|
return GraphologyBackend.create();
|
|
33
33
|
}
|
|
34
34
|
catch {
|
|
35
|
-
throw new Error('graphology backend
|
|
35
|
+
throw new Error('graphology backend is unavailable; run `aiwg features install graph`');
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
38
|
case 'sqlite': {
|
|
@@ -41,7 +41,7 @@ export async function createGraphBackend(type = 'json') {
|
|
|
41
41
|
return new SqliteGraphBackend();
|
|
42
42
|
}
|
|
43
43
|
catch {
|
|
44
|
-
throw new Error('sqlite backend
|
|
44
|
+
throw new Error('sqlite backend is unavailable; run `aiwg features install sqlite`');
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
default:
|
|
@@ -18,8 +18,8 @@ import { parseFrontmatter } from './index-builder.js';
|
|
|
18
18
|
import { applyFacetFusion, diagnoseFacetActivations, } from './discover-facets.js';
|
|
19
19
|
import { recordTypeForEntry, stableRecordId, } from './browser-export.js';
|
|
20
20
|
import { loadProviderModelMetadata } from '../models/provider-models.js';
|
|
21
|
+
import { projectAiwgPath, projectControlPath } from '../config/project-artifacts.js';
|
|
21
22
|
import { operationalStateQueryProjection, } from './operational-state.js';
|
|
22
|
-
import { projectAiwgPath } from '../config/project-artifacts.js';
|
|
23
23
|
function normalizeIndexedPath(entryPath) {
|
|
24
24
|
return entryPath.replace(/\\/g, '/');
|
|
25
25
|
}
|
|
@@ -82,6 +82,22 @@ function graphScope(graph) {
|
|
|
82
82
|
return 'codebase';
|
|
83
83
|
return 'custom';
|
|
84
84
|
}
|
|
85
|
+
function containsTokenSequence(haystack, needle) {
|
|
86
|
+
if (needle.length === 0 || haystack.length < needle.length)
|
|
87
|
+
return false;
|
|
88
|
+
for (let i = 0; i <= haystack.length - needle.length; i++) {
|
|
89
|
+
let matched = true;
|
|
90
|
+
for (let j = 0; j < needle.length; j++) {
|
|
91
|
+
if (haystack[i + j] !== needle[j]) {
|
|
92
|
+
matched = false;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (matched)
|
|
97
|
+
return true;
|
|
98
|
+
}
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
85
101
|
function withIndexProvenance(entry, graph) {
|
|
86
102
|
return { ...entry, indexGraph: graph, indexScope: graphScope(graph) };
|
|
87
103
|
}
|
|
@@ -375,11 +391,9 @@ function scoreEntryDetailed(entry, text, opts = {}) {
|
|
|
375
391
|
});
|
|
376
392
|
return finish(1.0008, 1.0008);
|
|
377
393
|
}
|
|
378
|
-
else if (
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
rawLower.includes(trigger)) {
|
|
382
|
-
const triggerInsideQuery = lower.includes(trigger) || rawLower.includes(trigger);
|
|
394
|
+
else if (containsTokenSequence(triggerTokens, tokens) ||
|
|
395
|
+
containsTokenSequence(tokens, triggerTokens)) {
|
|
396
|
+
const triggerInsideQuery = containsTokenSequence(tokens, triggerTokens);
|
|
383
397
|
const containedCoverage = triggerInsideQuery
|
|
384
398
|
? queryCoverage
|
|
385
399
|
: triggerTokens.length > 0
|
|
@@ -756,7 +770,7 @@ const DEFAULT_DISCOVER_TYPES = [...OPERATIONAL_DISCOVERY_TYPES];
|
|
|
756
770
|
const DEFAULT_CAPABILITY_GRAPHS = ['project', 'user', 'framework'];
|
|
757
771
|
function projectAllowsUserIndices(cwd) {
|
|
758
772
|
try {
|
|
759
|
-
const configPath =
|
|
773
|
+
const configPath = projectControlPath(cwd, 'aiwg.config');
|
|
760
774
|
if (!fs.existsSync(configPath))
|
|
761
775
|
return true;
|
|
762
776
|
const parsed = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { PROJECT_CONTROL_PLANE_FILES, auditProjectArtifactHealth, } from '../config/project-artifacts.js';
|
|
4
|
+
export async function repairProjectArtifacts(options) {
|
|
5
|
+
const projectDir = path.resolve(options.projectDir);
|
|
6
|
+
const before = auditProjectArtifactHealth(projectDir);
|
|
7
|
+
const copied = [];
|
|
8
|
+
const removed = [];
|
|
9
|
+
if (!before.external_configured) {
|
|
10
|
+
throw new Error('No external AIWG artifact corpus is configured for this project.');
|
|
11
|
+
}
|
|
12
|
+
if (!before.external_reachable) {
|
|
13
|
+
throw new Error(`External AIWG artifact corpus is unavailable: ${before.artifact_root}`);
|
|
14
|
+
}
|
|
15
|
+
if (before.divergent_control_files.length || before.divergent_local_corpus_files.length) {
|
|
16
|
+
throw new Error(`Automatic repair refused because local and external content diverges: ${[
|
|
17
|
+
...before.divergent_control_files,
|
|
18
|
+
...before.divergent_local_corpus_files,
|
|
19
|
+
].join(', ')}`);
|
|
20
|
+
}
|
|
21
|
+
for (const relativePath of before.missing_local_control_files) {
|
|
22
|
+
const source = path.join(before.artifact_root, relativePath);
|
|
23
|
+
if (!PROJECT_CONTROL_PLANE_FILES.includes(relativePath) || !before.control_files.find((item) => item.file === relativePath)?.external) {
|
|
24
|
+
throw new Error(`Cannot restore missing local control-plane file from the external corpus: ${relativePath}`);
|
|
25
|
+
}
|
|
26
|
+
copied.push(relativePath);
|
|
27
|
+
if (options.apply) {
|
|
28
|
+
const destination = path.join(before.local_control_root, relativePath);
|
|
29
|
+
await mkdir(path.dirname(destination), { recursive: true });
|
|
30
|
+
await writeFile(destination, await readFile(source));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
for (const relativePath of before.duplicated_local_corpus_files) {
|
|
34
|
+
removed.push(relativePath);
|
|
35
|
+
if (options.apply) {
|
|
36
|
+
await rm(path.join(before.local_control_root, relativePath), { force: false });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
applied: options.apply === true,
|
|
41
|
+
before,
|
|
42
|
+
after: options.apply ? auditProjectArtifactHealth(projectDir) : before,
|
|
43
|
+
copied,
|
|
44
|
+
removed,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=repair.js.map
|
|
@@ -12,7 +12,7 @@ import fs from 'fs';
|
|
|
12
12
|
import os from 'node:os';
|
|
13
13
|
import path from 'node:path';
|
|
14
14
|
import { load as loadYaml } from 'js-yaml';
|
|
15
|
-
import { DEFAULT_PROJECT_AIWG_DIR, projectAiwgPath, resolveProjectAiwgDir, } from '../config/project-artifacts.js';
|
|
15
|
+
import { DEFAULT_PROJECT_AIWG_DIR, projectAiwgPath, projectControlPath, resolveProjectAiwgDir, } from '../config/project-artifacts.js';
|
|
16
16
|
/**
|
|
17
17
|
* Operational artifact kinds that belong on the broad capability-discovery
|
|
18
18
|
* surface. This is intentionally narrower than every indexed artifact type:
|
|
@@ -411,7 +411,7 @@ export function loadUserGraphConfigs(cwd, diagnostics) {
|
|
|
411
411
|
let fromDeprecatedYaml = false;
|
|
412
412
|
// (a) Canonical: .aiwg/aiwg.config (JSON).
|
|
413
413
|
try {
|
|
414
|
-
const aiwgConfigPath =
|
|
414
|
+
const aiwgConfigPath = projectControlPath(cwd, 'aiwg.config');
|
|
415
415
|
if (fs.existsSync(aiwgConfigPath)) {
|
|
416
416
|
const parsed = JSON.parse(fs.readFileSync(aiwgConfigPath, 'utf-8'));
|
|
417
417
|
const idx = parsed.index;
|
|
@@ -451,7 +451,7 @@ export function loadUserGraphConfigs(cwd, diagnostics) {
|
|
|
451
451
|
return loaded;
|
|
452
452
|
if (fromDeprecatedYaml && !yamlIndexDeprecationWarned) {
|
|
453
453
|
yamlIndexDeprecationWarned = true;
|
|
454
|
-
process.stderr.write('[aiwg index] note: index.graphs in .aiwg/config.yaml is deprecated — move the index block into .aiwg/aiwg.config (see docs/cli
|
|
454
|
+
process.stderr.write('[aiwg index] note: index.graphs in .aiwg/config.yaml is deprecated — move the index block into .aiwg/aiwg.config (see docs/cli/reference.md, #1491).\n');
|
|
455
455
|
}
|
|
456
456
|
for (const [name, def] of Object.entries(graphs)) {
|
|
457
457
|
if (name in BUILTIN_GRAPH_CONFIGS) {
|
|
@@ -4,7 +4,7 @@ import { createHash } from 'crypto';
|
|
|
4
4
|
import path from 'path';
|
|
5
5
|
import os from 'os';
|
|
6
6
|
import { readAiwgConfig } from '../config/aiwg-config.js';
|
|
7
|
-
import { PROJECT_AIWG_LOCATION_FILE, projectAiwgPath } from '../config/project-artifacts.js';
|
|
7
|
+
import { PROJECT_AIWG_LOCATION_FILE, projectAiwgPath, projectControlPath, } from '../config/project-artifacts.js';
|
|
8
8
|
const DEFAULT_MAX_BYTES = 1_048_576;
|
|
9
9
|
const DEFAULT_REPORT_LIMIT = 20;
|
|
10
10
|
export async function maybeAppendCommandLog(input) {
|
|
@@ -157,7 +157,7 @@ async function findProjectRoot(startDir) {
|
|
|
157
157
|
while (current !== path.dirname(current)) {
|
|
158
158
|
if (existsSync(path.join(current, '.aiwg')) ||
|
|
159
159
|
existsSync(path.join(current, PROJECT_AIWG_LOCATION_FILE)) ||
|
|
160
|
-
existsSync(
|
|
160
|
+
existsSync(projectControlPath(current, 'aiwg.config'))) {
|
|
161
161
|
return current;
|
|
162
162
|
}
|
|
163
163
|
current = path.dirname(current);
|