@ngockhoale/ukit 2.6.9 → 2.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/manifests/platform.full.yaml +11 -0
- package/package.json +1 -1
- package/src/cli/commands/feedback.js +97 -0
- package/src/cli/commands/memory.js +250 -1
- package/src/cli/commands/metrics.js +235 -0
- package/src/cli/index.js +14 -0
- package/src/core/codeintel/retriever.js +65 -0
- package/src/core/memory/store.js +7 -2
- package/src/core/runtimeConfig.js +53 -0
- package/src/diagnostics/failurePatterns.js +154 -0
- package/src/diagnostics/feedbackEvents.js +196 -0
- package/src/diagnostics/laneStats.js +111 -0
- package/src/diagnostics/ledgerFiles.js +47 -0
- package/src/diagnostics/routeOutcomes.js +146 -0
- package/src/diagnostics/skillAccuracy.js +158 -0
- package/src/learning/patternProposals.js +151 -0
- package/src/learning/tuning.js +213 -0
- package/templates/.claude/hooks/block-dangerous.sh +29 -5
- package/templates/.claude/hooks/context-hardcap-gate.sh +4 -1
- package/templates/.claude/hooks/protect-files.sh +26 -3
- package/templates/.claude/hooks/session-episode.sh +84 -0
- package/templates/.claude/settings.json +11 -0
- package/templates/.claude/ukit/index/route-task.mjs +63 -6
- package/templates/.claude/ukit/runtime/hook-chain-runner.mjs +9 -1
- package/templates/.claude/ukit/runtime/hook-input.sh +26 -3
- package/templates/docs/UKIT_INTERNALS.md +17 -0
- package/templates/ukit/storage/config.json +18 -0
|
@@ -1325,6 +1325,17 @@ items:
|
|
|
1325
1325
|
packs:
|
|
1326
1326
|
- core
|
|
1327
1327
|
|
|
1328
|
+
- id: hook-session-episode
|
|
1329
|
+
type: hook
|
|
1330
|
+
sourceTemplate: .claude/hooks/session-episode.sh
|
|
1331
|
+
targetPath: .claude/hooks/session-episode.sh
|
|
1332
|
+
requires: []
|
|
1333
|
+
mergeStrategy: overwrite_with_backup
|
|
1334
|
+
variables: []
|
|
1335
|
+
enabledByDefault: true
|
|
1336
|
+
packs:
|
|
1337
|
+
- core
|
|
1338
|
+
|
|
1328
1339
|
- id: ukit-index-anchor-search-script
|
|
1329
1340
|
type: config
|
|
1330
1341
|
sourceTemplate: .claude/ukit/index/anchor-search.mjs
|
package/package.json
CHANGED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// `ukit feedback` — manual wrong-route label capture (SPEC C32 §3c).
|
|
2
|
+
//
|
|
3
|
+
// `ukit feedback "<text>" [--target <file>]` appends
|
|
4
|
+
// `{ts, text, targetFile?, sessionId?}` to
|
|
5
|
+
// `.ukit/storage/cache/feedback-manual.jsonl` and prints `feedback recorded`.
|
|
6
|
+
// `sessionId` = UKIT_SESSION_ID env only (SessionEnd hook exports it) — omitted
|
|
7
|
+
// when unset. `--list` prints a byKind roll-up via collectFeedbackEvents and
|
|
8
|
+
// never writes. Missing/empty text or unknown flag → usage on stderr, exit 1.
|
|
9
|
+
|
|
10
|
+
import fs from 'node:fs/promises';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { collectFeedbackEvents } from '../../diagnostics/feedbackEvents.js';
|
|
13
|
+
|
|
14
|
+
const HELP_FLAGS = new Set(['--help', '-h', 'help']);
|
|
15
|
+
|
|
16
|
+
function printUsage(stream = console.log) {
|
|
17
|
+
stream('Usage: ukit feedback "<text>" [--target <file>]');
|
|
18
|
+
stream(' ukit feedback --list');
|
|
19
|
+
stream('');
|
|
20
|
+
stream('Record a manual wrong-route correction label, or inspect collected');
|
|
21
|
+
stream('feedback events (rescue / re-route / repeat-stall / user-correction).');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async function appendManualLine(projectRoot, record) {
|
|
25
|
+
const filePath = path.join(projectRoot, '.ukit', 'storage', 'cache', 'feedback-manual.jsonl');
|
|
26
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
27
|
+
await fs.appendFile(filePath, JSON.stringify(record) + '\n');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function runFeedback({ projectRoot, argv = [] }) {
|
|
31
|
+
const args = argv ?? [];
|
|
32
|
+
|
|
33
|
+
if (args.some((a) => HELP_FLAGS.has(a))) {
|
|
34
|
+
printUsage();
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (args.includes('--list')) {
|
|
39
|
+
const result = await collectFeedbackEvents(projectRoot);
|
|
40
|
+
console.log('feedback-events');
|
|
41
|
+
console.log(` audit rows: ${result.auditRowsScanned} ledgers: ${result.ledgersScanned}`);
|
|
42
|
+
for (const [kind, count] of Object.entries(result.byKind)) {
|
|
43
|
+
console.log(` ${kind}: ${count}`);
|
|
44
|
+
}
|
|
45
|
+
for (const event of result.events.slice(0, 5)) {
|
|
46
|
+
console.log(` - ${event.kind} · ${event.detail}`);
|
|
47
|
+
}
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let text = null;
|
|
52
|
+
let targetFile;
|
|
53
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
54
|
+
const arg = args[i];
|
|
55
|
+
if (arg === '--target') {
|
|
56
|
+
targetFile = args[i + 1];
|
|
57
|
+
i += 1;
|
|
58
|
+
if (typeof targetFile !== 'string' || targetFile.startsWith('--')) {
|
|
59
|
+
console.error('Missing value for --target.');
|
|
60
|
+
printUsage(console.error);
|
|
61
|
+
process.exitCode = 1;
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (arg.startsWith('--')) {
|
|
67
|
+
console.error(`Unknown flag: ${arg}`);
|
|
68
|
+
printUsage(console.error);
|
|
69
|
+
process.exitCode = 1;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (text == null) {
|
|
73
|
+
text = arg;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (!text || !text.trim()) {
|
|
78
|
+
console.error('Missing feedback text.');
|
|
79
|
+
printUsage(console.error);
|
|
80
|
+
process.exitCode = 1;
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const record = { ts: Date.now(), text: text.trim() };
|
|
85
|
+
if (typeof targetFile === 'string' && targetFile) record.targetFile = targetFile;
|
|
86
|
+
const sessionId = process.env.UKIT_SESSION_ID;
|
|
87
|
+
if (sessionId) record.sessionId = sessionId;
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
await appendManualLine(projectRoot, record);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
console.error(`Failed to record feedback: ${error?.message ?? error}`);
|
|
93
|
+
process.exitCode = 1;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
console.log('feedback recorded');
|
|
97
|
+
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
runProjectHygiene,
|
|
9
9
|
} from '../../core/memory/store.js';
|
|
10
10
|
import {
|
|
11
|
+
addRecord,
|
|
11
12
|
getRecord,
|
|
12
13
|
loadRecords,
|
|
13
14
|
queryRecords,
|
|
@@ -20,7 +21,9 @@ import { inspectRuntimeConfig } from '../../core/runtimeConfig.js';
|
|
|
20
21
|
import { buildRuntimePaths } from '../../core/runtimePaths.js';
|
|
21
22
|
import { pathExists } from '../../core/fileOps.js';
|
|
22
23
|
import { detectProjectContext } from '../../context/detectProjectContext.js';
|
|
24
|
+
import { listLedgerFiles, LEDGER_DIR_REL } from '../../diagnostics/ledgerFiles.js';
|
|
23
25
|
import fs from 'node:fs/promises';
|
|
26
|
+
import path from 'node:path';
|
|
24
27
|
|
|
25
28
|
const HELP_FLAGS = new Set(['--help', '-h', 'help']);
|
|
26
29
|
|
|
@@ -177,6 +180,192 @@ async function runMemoryV2(projectRoot, args) {
|
|
|
177
180
|
throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|promote|stale|migrate.`);
|
|
178
181
|
}
|
|
179
182
|
|
|
183
|
+
// ---- `ukit memory promote` — MEMORY.md `## ukit-learned` lane (SPEC §7a) ----
|
|
184
|
+
|
|
185
|
+
const LEARNED_START = '<!-- ukit-learned:start -->';
|
|
186
|
+
const LEARNED_END = '<!-- ukit-learned:end -->';
|
|
187
|
+
const MEM_REF_RE = /<!--\s*mem:(mem_[0-9a-f]{12})\s*-->/g;
|
|
188
|
+
|
|
189
|
+
// Approved promotable records: active project_rule/procedure belonging to the
|
|
190
|
+
// project, not bulk-migrated v1 leftovers (created_by 'migration'), and not
|
|
191
|
+
// still-pending candidates (meta.legacyStatus 'pending').
|
|
192
|
+
async function collectPromotableRecords(projectRoot, projectId) {
|
|
193
|
+
const records = await loadRecords(projectRoot);
|
|
194
|
+
return records.filter((r) =>
|
|
195
|
+
(r.type === 'project_rule' || r.type === 'procedure')
|
|
196
|
+
&& r.status === 'active'
|
|
197
|
+
&& r.created_by !== 'migration'
|
|
198
|
+
&& r.meta?.legacyStatus !== 'pending'
|
|
199
|
+
&& (r.project_id == null || r.project_id === projectId));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function renderLearnedBlock(records) {
|
|
203
|
+
const lines = [LEARNED_START, '## ukit-learned'];
|
|
204
|
+
for (const r of records) {
|
|
205
|
+
lines.push(`- [${r.type}] ${r.text} <!-- mem:${r.id} -->`);
|
|
206
|
+
}
|
|
207
|
+
lines.push(LEARNED_END);
|
|
208
|
+
return lines.join('\n');
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Ids already referenced in mem:<id> comments OUTSIDE the managed block are
|
|
212
|
+
// user-promoted manually — promote must not duplicate them inside the block.
|
|
213
|
+
function externallyReferencedIds(content) {
|
|
214
|
+
const withoutBlock = content.replace(
|
|
215
|
+
new RegExp(`${escapeRe(LEARNED_START)}[\\s\\S]*?${escapeRe(LEARNED_END)}`, 'g'),
|
|
216
|
+
'',
|
|
217
|
+
);
|
|
218
|
+
const ids = new Set();
|
|
219
|
+
for (const m of withoutBlock.matchAll(MEM_REF_RE)) ids.add(m[1]);
|
|
220
|
+
return ids;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function escapeRe(s) {
|
|
224
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async function writeTextAtomic(filePath, text) {
|
|
228
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
229
|
+
const tmp = `${filePath}.tmp-${process.pid}-${Math.random().toString(16).slice(2)}`;
|
|
230
|
+
await fs.writeFile(tmp, text);
|
|
231
|
+
await fs.rename(tmp, filePath);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function runMemoryPromote(projectRoot, args) {
|
|
235
|
+
const dryRun = args.includes('--dry-run');
|
|
236
|
+
const projectId = (await detectProjectContext(projectRoot)).project.name;
|
|
237
|
+
const memoryMdPath = path.join(projectRoot, 'docs', 'MEMORY.md');
|
|
238
|
+
|
|
239
|
+
let existing = '';
|
|
240
|
+
try {
|
|
241
|
+
existing = await fs.readFile(memoryMdPath, 'utf8');
|
|
242
|
+
} catch {
|
|
243
|
+
existing = '';
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const outsideIds = externallyReferencedIds(existing);
|
|
247
|
+
const promotable = (await collectPromotableRecords(projectRoot, projectId))
|
|
248
|
+
.filter((r) => !outsideIds.has(r.id));
|
|
249
|
+
|
|
250
|
+
const block = renderLearnedBlock(promotable);
|
|
251
|
+
|
|
252
|
+
if (dryRun) {
|
|
253
|
+
console.log('[UKit] memory promote (dry-run) — rendered block:');
|
|
254
|
+
console.log(block);
|
|
255
|
+
console.log(`[UKit] promoted: ${promotable.length} records → docs/MEMORY.md (dry-run)`);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
let next;
|
|
260
|
+
const startIdx = existing.indexOf(LEARNED_START);
|
|
261
|
+
const endIdx = existing.indexOf(LEARNED_END);
|
|
262
|
+
if (startIdx >= 0 && endIdx > startIdx) {
|
|
263
|
+
next = `${existing.slice(0, startIdx)}${block}${existing.slice(endIdx + LEARNED_END.length)}`;
|
|
264
|
+
} else if (existing.length > 0) {
|
|
265
|
+
next = `${existing.replace(/\s*$/, '')}\n\n${block}\n`;
|
|
266
|
+
} else {
|
|
267
|
+
next = `${block}\n`;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
await writeTextAtomic(memoryMdPath, next);
|
|
271
|
+
console.log(`[UKit] promoted: ${promotable.length} records → docs/MEMORY.md`);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// ---- `ukit memory episode` — session episode from exec-ledger (SPEC §7b) ----
|
|
275
|
+
|
|
276
|
+
// Mirrors execution-ledger.mjs safeSegment — src/ cannot import the runtime
|
|
277
|
+
// module, so the segment rule is duplicated deliberately.
|
|
278
|
+
function safeSegment(value) {
|
|
279
|
+
return String(value || 'default')
|
|
280
|
+
.trim()
|
|
281
|
+
.replace(/[^a-zA-Z0-9._-]/g, '_')
|
|
282
|
+
.slice(0, 96) || 'default';
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
async function readJsonIfExists(filePath) {
|
|
286
|
+
try {
|
|
287
|
+
return JSON.parse(await fs.readFile(filePath, 'utf8'));
|
|
288
|
+
} catch {
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// Resolution order: --session <id> → UKIT_SESSION_ID env → most-recent ledger.
|
|
294
|
+
async function resolveLedger(projectRoot, sessionFlag) {
|
|
295
|
+
const dir = path.join(projectRoot, LEDGER_DIR_REL);
|
|
296
|
+
const candidate = sessionFlag ?? process.env.UKIT_SESSION_ID;
|
|
297
|
+
if (candidate) {
|
|
298
|
+
const file = `${safeSegment(candidate)}.json`;
|
|
299
|
+
const ledger = await readJsonIfExists(path.join(dir, file));
|
|
300
|
+
return ledger ? { ledger, ledgerKey: file } : { ledger: null, ledgerKey: file };
|
|
301
|
+
}
|
|
302
|
+
const [name] = await listLedgerFiles(dir, 1);
|
|
303
|
+
if (!name) return { ledger: null, ledgerKey: null };
|
|
304
|
+
const ledger = await readJsonIfExists(path.join(dir, name));
|
|
305
|
+
return ledger ? { ledger, ledgerKey: name } : { ledger: null, ledgerKey: name };
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function episodeText(ledger, sessionId) {
|
|
309
|
+
const write = ledger.writeSucceeded === true ? 'writeOk' : 'writeFail';
|
|
310
|
+
const verify = ledger.verificationAttempted !== true
|
|
311
|
+
? 'not-run'
|
|
312
|
+
: ledger.verificationSucceeded === true ? 'ok' : 'fail';
|
|
313
|
+
const receipts = Array.isArray(ledger.receipts) ? ledger.receipts : [];
|
|
314
|
+
const lastCommand = receipts.length > 0
|
|
315
|
+
? String(receipts[receipts.length - 1].command ?? '').split('\n')[0].trim() || 'n/a'
|
|
316
|
+
: 'n/a';
|
|
317
|
+
const text = `Session ${sessionId}: ${write}, verify=${verify}, `
|
|
318
|
+
+ `${receipts.length} receipts, last command ${lastCommand}`;
|
|
319
|
+
return text.slice(0, 300);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
async function runMemoryEpisode(projectRoot, args) {
|
|
323
|
+
const dryRun = args.includes('--dry-run');
|
|
324
|
+
const sessionFlag = extractFlag(args, '--session').value;
|
|
325
|
+
|
|
326
|
+
const { config } = await inspectRuntimeConfig(projectRoot);
|
|
327
|
+
if (config?.memoryV2?.enabled === false) {
|
|
328
|
+
console.log('[UKit] episode: skipped (memoryV2 disabled)');
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const { ledger, ledgerKey } = await resolveLedger(projectRoot, sessionFlag);
|
|
333
|
+
if (!ledger) {
|
|
334
|
+
console.log('[UKit] episode: nothing to record');
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const sessionId = ledger.sessionId ?? ledger.sessionKey
|
|
339
|
+
?? (ledgerKey ? ledgerKey.replace(/\.json$/, '') : 'unknown');
|
|
340
|
+
const text = episodeText(ledger, sessionId);
|
|
341
|
+
|
|
342
|
+
if (dryRun) {
|
|
343
|
+
console.log(`[UKit] episode (dry-run): ${text}`);
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const existing = await loadRecords(projectRoot);
|
|
348
|
+
if (existing.some((r) => r.meta?.ledgerKey === ledgerKey)) {
|
|
349
|
+
console.log('[UKit] episode: already recorded');
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const ttlDays = Number(config?.memoryV2?.episodeTtlDays) || 90;
|
|
354
|
+
const projectId = (await detectProjectContext(projectRoot)).project.name;
|
|
355
|
+
const record = await addRecord(projectRoot, {
|
|
356
|
+
type: 'episode',
|
|
357
|
+
scope: 'session',
|
|
358
|
+
text,
|
|
359
|
+
provenance: 'exec-ledger',
|
|
360
|
+
confidence: 0.6,
|
|
361
|
+
createdBy: 'episode-hook',
|
|
362
|
+
projectId,
|
|
363
|
+
validUntil: Date.now() + ttlDays * 24 * 60 * 60 * 1000,
|
|
364
|
+
meta: { sessionId, ledgerKey },
|
|
365
|
+
});
|
|
366
|
+
console.log(`[UKit] episode: recorded ${record.id} for session ${sessionId}`);
|
|
367
|
+
}
|
|
368
|
+
|
|
180
369
|
export async function runMemory({ projectRoot, argv = [] }) {
|
|
181
370
|
const runtimePaths = buildRuntimePaths(projectRoot);
|
|
182
371
|
if (!(await pathExists(runtimePaths.runtimeRoot))) {
|
|
@@ -242,6 +431,53 @@ export async function runMemory({ projectRoot, argv = [] }) {
|
|
|
242
431
|
return;
|
|
243
432
|
}
|
|
244
433
|
|
|
434
|
+
if (subcommand === 'learn') {
|
|
435
|
+
const applyFlag = rest.includes('--apply');
|
|
436
|
+
const knownFlags = new Set(['--apply']);
|
|
437
|
+
const minFlag = extractFlag(rest, '--min');
|
|
438
|
+
const projectFlag = extractFlag(minFlag.rest, '--project');
|
|
439
|
+
for (const arg of projectFlag.rest) {
|
|
440
|
+
if (arg.startsWith('--') && !knownFlags.has(arg)) {
|
|
441
|
+
throw new Error(`Unknown flag for memory learn: ${arg}`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
const minCount = minFlag.value != null ? Number(minFlag.value) : undefined;
|
|
445
|
+
if (minFlag.value != null && (!Number.isFinite(minCount) || minCount < 1)) {
|
|
446
|
+
throw new Error(`Invalid --min value: ${minFlag.value}`);
|
|
447
|
+
}
|
|
448
|
+
const projectId = projectFlag.value ?? (await detectProjectContext(projectRoot)).project.name;
|
|
449
|
+
|
|
450
|
+
const { proposeFromPatterns } = await import('../../learning/patternProposals.js');
|
|
451
|
+
const result = await proposeFromPatterns(projectRoot, projectId, {
|
|
452
|
+
minCount,
|
|
453
|
+
dryRun: !applyFlag,
|
|
454
|
+
});
|
|
455
|
+
if (result.error) {
|
|
456
|
+
console.log(`[UKit] memory learn: ${result.error}`);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
if (!applyFlag) {
|
|
460
|
+
console.log(`[UKit] memory learn (dry-run) — ${result.patternsScanned} pattern(s) scanned, ${result.eligible} eligible.`);
|
|
461
|
+
for (const entry of result.proposed) {
|
|
462
|
+
console.log(` [${entry.status}] ${entry.signature} — ${entry.text}`);
|
|
463
|
+
}
|
|
464
|
+
console.log('[UKit] Run `ukit memory learn --apply` to write pending candidates.');
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
const proposed = result.proposed.filter((p) => p.status === 'proposed').length;
|
|
469
|
+
const duplicates = result.proposed.filter((p) => p.status === 'duplicate').length;
|
|
470
|
+
const skipped = result.proposed.filter((p) => p.status === 'skipped').length;
|
|
471
|
+
for (const entry of result.proposed) {
|
|
472
|
+
console.log(` [${entry.status}] ${entry.signature} — ${entry.text}`);
|
|
473
|
+
}
|
|
474
|
+
console.log(`[UKit] memory learn: proposed ${proposed}, duplicate ${duplicates}, skipped ${skipped} for project ${projectId}.`);
|
|
475
|
+
if (proposed > 0) {
|
|
476
|
+
console.log('[UKit] Review with `ukit memory list --pending`, then `ukit memory approve <id>`.');
|
|
477
|
+
}
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
|
|
245
481
|
if (subcommand === 'approve' || subcommand === 'reject') {
|
|
246
482
|
const projectFlagIndex = rest.indexOf('--project');
|
|
247
483
|
const explicitProjectId = projectFlagIndex >= 0 ? rest[projectFlagIndex + 1] : null;
|
|
@@ -327,6 +563,16 @@ export async function runMemory({ projectRoot, argv = [] }) {
|
|
|
327
563
|
return;
|
|
328
564
|
}
|
|
329
565
|
|
|
566
|
+
if (subcommand === 'promote') {
|
|
567
|
+
await runMemoryPromote(projectRoot, rest);
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
if (subcommand === 'episode') {
|
|
572
|
+
await runMemoryEpisode(projectRoot, rest);
|
|
573
|
+
return;
|
|
574
|
+
}
|
|
575
|
+
|
|
330
576
|
if (subcommand === 'forget') {
|
|
331
577
|
const memoryId = rest.join(' ').trim();
|
|
332
578
|
if (!memoryId) {
|
|
@@ -353,7 +599,7 @@ export async function runMemory({ projectRoot, argv = [] }) {
|
|
|
353
599
|
|
|
354
600
|
export function printMemoryHelp() {
|
|
355
601
|
console.log('UKit Memory Commands');
|
|
356
|
-
console.log('Usage: ukit memory <list|search|recall|forget|export|propose|approve|reject|hygiene> [args]');
|
|
602
|
+
console.log('Usage: ukit memory <list|search|recall|forget|export|propose|approve|reject|learn|promote|episode|hygiene> [args]');
|
|
357
603
|
console.log('');
|
|
358
604
|
console.log('Subcommands:');
|
|
359
605
|
console.log(' list List memory items in shared .ukit/storage/memory');
|
|
@@ -365,5 +611,8 @@ export function printMemoryHelp() {
|
|
|
365
611
|
console.log(' propose "<text>" [--category <name>] [--project <id>] Propose a project convention for human approval');
|
|
366
612
|
console.log(' approve <id> [--project <id>] Approve a pending pattern candidate into project conventions');
|
|
367
613
|
console.log(' reject <id> [--project <id>] Reject a pending pattern candidate');
|
|
614
|
+
console.log(' learn [--apply] [--min <n>] [--project <id>] Propose pending candidates from mined failure patterns (dry-run by default)');
|
|
615
|
+
console.log(' promote [--dry-run] Render approved rules/procedures into the ## ukit-learned block in docs/MEMORY.md');
|
|
616
|
+
console.log(' episode [--dry-run] [--session <id>] Write a session episode record from the exec-ledger');
|
|
368
617
|
console.log(' hygiene [--project <id>] Run decision-conflict resolution + session archiving now');
|
|
369
618
|
}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// `ukit metrics` — read-only telemetry roll-up (SPEC §6, SI-102).
|
|
2
|
+
// Route outcomes + failure patterns (lazy, degrade to n/a) + v2 memory count.
|
|
3
|
+
// Never exits non-zero on missing telemetry; unknown flag → exit 1.
|
|
4
|
+
|
|
5
|
+
import fs from 'node:fs/promises';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { collectRouteOutcomes } from '../../diagnostics/routeOutcomes.js';
|
|
8
|
+
import { buildRuntimePaths } from '../../core/runtimePaths.js';
|
|
9
|
+
|
|
10
|
+
const HELP_FLAGS = new Set(['--help', '-h', 'help']);
|
|
11
|
+
const KNOWN_FLAGS = new Set(['--json']);
|
|
12
|
+
|
|
13
|
+
const MODE_FIELDS = ['routes', 'joined', 'writeOk', 'writeFail', 'verifyAttempted', 'verifyOk', 'verifyFail', 'rescue', 'stalled'];
|
|
14
|
+
|
|
15
|
+
function printUsage() {
|
|
16
|
+
console.log('Usage: ukit metrics [--json]');
|
|
17
|
+
console.log('');
|
|
18
|
+
console.log('Print a read-only roll-up of UKit telemetry:');
|
|
19
|
+
console.log(' route-outcomes route-audit ↔ exec-ledger join stats per mode/taskType');
|
|
20
|
+
console.log(' failure-patterns mined failure signatures (n/a when unavailable)');
|
|
21
|
+
console.log(' memory v2 record count by status');
|
|
22
|
+
console.log('');
|
|
23
|
+
console.log('Options:');
|
|
24
|
+
console.log(' --json Print the full roll-up object as JSON');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function collectMemorySummary(projectRoot) {
|
|
28
|
+
try {
|
|
29
|
+
const { memoryV2RecordsPath } = buildRuntimePaths(projectRoot);
|
|
30
|
+
const raw = await fs.readFile(memoryV2RecordsPath, 'utf8');
|
|
31
|
+
const doc = JSON.parse(raw);
|
|
32
|
+
const records = Array.isArray(doc) ? doc : (Array.isArray(doc?.records) ? doc.records : []);
|
|
33
|
+
const byStatus = {};
|
|
34
|
+
for (const record of records) {
|
|
35
|
+
const status = record?.status ?? 'unknown';
|
|
36
|
+
byStatus[status] = (byStatus[status] ?? 0) + 1;
|
|
37
|
+
}
|
|
38
|
+
return { total: records.length, byStatus };
|
|
39
|
+
} catch {
|
|
40
|
+
return { total: 0, byStatus: {} };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function collectFailurePatterns(projectRoot) {
|
|
45
|
+
try {
|
|
46
|
+
const mod = await import('../../diagnostics/failurePatterns.js');
|
|
47
|
+
if (typeof mod?.mineFailurePatterns !== 'function') return null;
|
|
48
|
+
return await mod.mineFailurePatterns(projectRoot);
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function lazyCollect(modulePath, exportName, projectRoot) {
|
|
55
|
+
try {
|
|
56
|
+
const mod = await import(modulePath);
|
|
57
|
+
if (typeof mod?.[exportName] !== 'function') return null;
|
|
58
|
+
return await mod[exportName](projectRoot);
|
|
59
|
+
} catch {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function printRouteOutcomes(routeOutcomes) {
|
|
65
|
+
console.log('route-outcomes');
|
|
66
|
+
console.log(` ledgers: ${routeOutcomes.ledgersScanned} audit rows: ${routeOutcomes.auditRowsScanned} joined: ${routeOutcomes.joined} coverage: ${(routeOutcomes.joinCoverage * 100).toFixed(1)}% unmatched audit: ${routeOutcomes.unmatchedAudit} unmatched ledger: ${routeOutcomes.unmatchedLedger}`);
|
|
67
|
+
const header = ['mode', ...MODE_FIELDS];
|
|
68
|
+
console.log(` ${header.join('\t')}`);
|
|
69
|
+
for (const [mode, bucket] of Object.entries(routeOutcomes.byMode)) {
|
|
70
|
+
console.log(` ${[mode, ...MODE_FIELDS.map((field) => bucket[field] ?? 0)].join('\t')}`);
|
|
71
|
+
}
|
|
72
|
+
if (Object.keys(routeOutcomes.byMode).length === 0) {
|
|
73
|
+
console.log(' (no route-audit data)');
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function printFailurePatterns(failurePatterns) {
|
|
78
|
+
console.log('failure-patterns');
|
|
79
|
+
if (!failurePatterns) {
|
|
80
|
+
console.log(' n/a');
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const patterns = Array.isArray(failurePatterns.patterns) ? failurePatterns.patterns : [];
|
|
84
|
+
if (patterns.length === 0) {
|
|
85
|
+
console.log(' (none)');
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
for (const pattern of patterns) {
|
|
89
|
+
console.log(` ${pattern.count}× ${pattern.signature} sessions=${pattern.sessions}`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function printMemory(memory) {
|
|
94
|
+
const statusPairs = Object.entries(memory.byStatus).map(([status, count]) => `${status}=${count}`).join(' ');
|
|
95
|
+
console.log(`memory: v2 records=${memory.total}${statusPairs ? ` (${statusPairs})` : ''}`);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function printFeedback(feedback) {
|
|
99
|
+
console.log('feedback');
|
|
100
|
+
if (!feedback) {
|
|
101
|
+
console.log(' n/a');
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const kinds = Object.entries(feedback.byKind ?? {})
|
|
105
|
+
.map(([kind, count]) => `${kind}=${count}`).join(' ');
|
|
106
|
+
console.log(` ${kinds || '(none)'}`);
|
|
107
|
+
const events = Array.isArray(feedback.events) ? feedback.events.slice(0, 5) : [];
|
|
108
|
+
for (const event of events) {
|
|
109
|
+
console.log(` ${event.kind}: ${event.detail ?? ''}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function printSkills(skillAccuracy) {
|
|
114
|
+
console.log('skills');
|
|
115
|
+
if (!skillAccuracy) {
|
|
116
|
+
console.log(' n/a');
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const skills = Object.entries(skillAccuracy.skills ?? {});
|
|
120
|
+
if (skills.length === 0) {
|
|
121
|
+
console.log(' (none)');
|
|
122
|
+
} else {
|
|
123
|
+
console.log(' skill\ttriggers\tjoined\taccuracy');
|
|
124
|
+
for (const [id, bucket] of skills) {
|
|
125
|
+
const accuracy = bucket.accuracy === null ? 'n/a' : bucket.accuracy.toFixed(2);
|
|
126
|
+
console.log(` ${id}\t${bucket.triggers}\t${bucket.joined}\t${accuracy}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const low = Array.isArray(skillAccuracy.lowAccuracy) ? skillAccuracy.lowAccuracy : [];
|
|
130
|
+
if (low.length > 0) {
|
|
131
|
+
console.log(` lowAccuracy: ${low.map((e) => `${e.id}=${e.accuracy.toFixed(2)}`).join(' ')}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function collectLearningSuggestions(projectRoot) {
|
|
136
|
+
try {
|
|
137
|
+
const raw = await fs.readFile(
|
|
138
|
+
path.join(projectRoot, '.ukit', 'storage', 'learning', 'suggestions.json'), 'utf8');
|
|
139
|
+
const doc = JSON.parse(raw);
|
|
140
|
+
const suggestions = Array.isArray(doc?.suggestions) ? doc.suggestions : [];
|
|
141
|
+
return {
|
|
142
|
+
pending: suggestions.length,
|
|
143
|
+
targets: suggestions.map((s) => s?.target).filter(Boolean),
|
|
144
|
+
generatedAt: doc?.generatedAt ?? null,
|
|
145
|
+
};
|
|
146
|
+
} catch {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function printLearning(learning) {
|
|
152
|
+
console.log('learning');
|
|
153
|
+
if (!learning) {
|
|
154
|
+
console.log(' n/a');
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
console.log(` pending=${learning.pending}${learning.generatedAt ? ` generatedAt=${learning.generatedAt}` : ''}`);
|
|
158
|
+
for (const target of learning.targets.slice(0, 10)) {
|
|
159
|
+
console.log(` - ${target}`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function printLaneStats(laneStats) {
|
|
164
|
+
console.log('retriever-lanes');
|
|
165
|
+
if (!laneStats) {
|
|
166
|
+
console.log(' n/a');
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
if (laneStats.empty) {
|
|
170
|
+
console.log(' (no retriever-lanes.jsonl data)');
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
const lanes = Object.entries(laneStats.lanes ?? {});
|
|
174
|
+
if (lanes.length === 0) {
|
|
175
|
+
console.log(' (none)');
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
console.log(' lane\tretrievals\ttotalHits\texclusive\tavgWeight\tcontribution');
|
|
179
|
+
for (const [lane, bucket] of lanes) {
|
|
180
|
+
console.log(` ${lane}\t${bucket.retrievals}\t${bucket.totalHits}\t${bucket.exclusive}\t${bucket.avgWeight.toFixed(2)}\t${bucket.contribution.toFixed(2)}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export async function runMetrics({ projectRoot, argv = [] }) {
|
|
185
|
+
if (argv.some((flag) => HELP_FLAGS.has(flag))) {
|
|
186
|
+
printUsage();
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
const unknown = argv.filter((flag) => !KNOWN_FLAGS.has(flag));
|
|
190
|
+
if (unknown.length > 0) {
|
|
191
|
+
console.error(`[UKit] Unknown metrics flag(s): ${unknown.join(', ')}`);
|
|
192
|
+
printUsage();
|
|
193
|
+
process.exitCode = 1;
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const routeOutcomes = await collectRouteOutcomes(projectRoot).catch((error) => ({
|
|
198
|
+
ledgersScanned: 0,
|
|
199
|
+
auditRowsScanned: 0,
|
|
200
|
+
joined: 0,
|
|
201
|
+
unmatchedAudit: 0,
|
|
202
|
+
unmatchedLedger: 0,
|
|
203
|
+
byMode: {},
|
|
204
|
+
byTaskType: {},
|
|
205
|
+
joinCoverage: 0,
|
|
206
|
+
error: error?.message ?? String(error),
|
|
207
|
+
}));
|
|
208
|
+
const failurePatterns = await collectFailurePatterns(projectRoot);
|
|
209
|
+
const memory = await collectMemorySummary(projectRoot);
|
|
210
|
+
const feedback = await lazyCollect('../../diagnostics/feedbackEvents.js', 'collectFeedbackEvents', projectRoot);
|
|
211
|
+
const skillAccuracy = await lazyCollect('../../diagnostics/skillAccuracy.js', 'collectSkillAccuracy', projectRoot);
|
|
212
|
+
const laneStats = await lazyCollect('../../diagnostics/laneStats.js', 'collectLaneStats', projectRoot);
|
|
213
|
+
const learning = await collectLearningSuggestions(projectRoot);
|
|
214
|
+
|
|
215
|
+
const rollup = { routeOutcomes, failurePatterns, memory, feedback, skillAccuracy, laneStats, learning };
|
|
216
|
+
|
|
217
|
+
if (argv.includes('--json')) {
|
|
218
|
+
console.log(JSON.stringify(rollup, null, 2));
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
printRouteOutcomes(routeOutcomes);
|
|
223
|
+
console.log('');
|
|
224
|
+
printFailurePatterns(failurePatterns);
|
|
225
|
+
console.log('');
|
|
226
|
+
printFeedback(feedback);
|
|
227
|
+
console.log('');
|
|
228
|
+
printSkills(skillAccuracy);
|
|
229
|
+
console.log('');
|
|
230
|
+
printLaneStats(laneStats);
|
|
231
|
+
console.log('');
|
|
232
|
+
printLearning(learning);
|
|
233
|
+
console.log('');
|
|
234
|
+
printMemory(memory);
|
|
235
|
+
}
|
package/src/cli/index.js
CHANGED
|
@@ -7,6 +7,8 @@ import { runStatus } from './commands/status.js';
|
|
|
7
7
|
import { runMemory } from './commands/memory.js';
|
|
8
8
|
import { runUpdate } from './commands/update.js';
|
|
9
9
|
import { runCode } from './commands/code.js';
|
|
10
|
+
import { runMetrics } from './commands/metrics.js';
|
|
11
|
+
import { runFeedback } from './commands/feedback.js';
|
|
10
12
|
|
|
11
13
|
const GLOBAL_FLAGS = new Set(['--help', '-h', '--version', '-v']);
|
|
12
14
|
|
|
@@ -65,6 +67,16 @@ export async function runCli({ argv, packageRoot, projectRoot, packageVersion })
|
|
|
65
67
|
return;
|
|
66
68
|
}
|
|
67
69
|
|
|
70
|
+
if (command === 'metrics') {
|
|
71
|
+
await runMetrics({ projectRoot, argv: commandArgv });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (command === 'feedback') {
|
|
76
|
+
await runFeedback({ projectRoot, argv: commandArgv });
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
68
80
|
if (command === 'update') {
|
|
69
81
|
await runUpdate({ packageVersion, argv: commandArgv });
|
|
70
82
|
return;
|
|
@@ -98,6 +110,8 @@ export async function runCli({ argv, packageRoot, projectRoot, packageVersion })
|
|
|
98
110
|
console.log(' code Context compiler surface (peek/search/context/impact)');
|
|
99
111
|
console.log(' status Show UKit runtime status');
|
|
100
112
|
console.log(' memory Inspect shared UKit memory');
|
|
113
|
+
console.log(' metrics Telemetry roll-up (route outcomes, failure patterns, memory)');
|
|
114
|
+
console.log(' feedback Record or list wrong-route feedback labels');
|
|
101
115
|
console.log(' update Upgrade the global UKit CLI to the latest version');
|
|
102
116
|
console.log(' version Show UKit version');
|
|
103
117
|
console.log('');
|