@ngockhoale/ukit 2.6.9 → 2.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/manifests/platform.full.yaml +11 -0
- package/package.json +1 -1
- package/src/cli/commands/feedback.js +97 -0
- package/src/cli/commands/memory.js +250 -1
- package/src/cli/commands/metrics.js +235 -0
- package/src/cli/index.js +14 -0
- package/src/core/codeintel/retriever.js +65 -0
- package/src/core/memory/store.js +7 -2
- package/src/core/runtimeConfig.js +53 -0
- package/src/diagnostics/failurePatterns.js +154 -0
- package/src/diagnostics/feedbackEvents.js +196 -0
- package/src/diagnostics/laneStats.js +111 -0
- package/src/diagnostics/ledgerFiles.js +47 -0
- package/src/diagnostics/routeOutcomes.js +146 -0
- package/src/diagnostics/skillAccuracy.js +158 -0
- package/src/learning/patternProposals.js +151 -0
- package/src/learning/tuning.js +213 -0
- package/templates/.claude/hooks/block-dangerous.sh +29 -5
- package/templates/.claude/hooks/context-hardcap-gate.sh +4 -1
- package/templates/.claude/hooks/protect-files.sh +26 -3
- package/templates/.claude/hooks/session-episode.sh +84 -0
- package/templates/.claude/settings.json +11 -0
- package/templates/.claude/ukit/index/route-task.mjs +63 -6
- package/templates/.claude/ukit/runtime/hook-chain-runner.mjs +9 -1
- package/templates/.claude/ukit/runtime/hook-input.sh +26 -3
- package/templates/docs/UKIT_INTERNALS.md +17 -0
- package/templates/ukit/storage/config.json +18 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import fs from 'node:fs/promises';
|
|
2
|
+
import fsSync from 'node:fs';
|
|
2
3
|
import path from 'node:path';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
3
5
|
|
|
4
6
|
import { getArtifactPath, INDEX_ARTIFACTS, INDEX_SCHEMA_VERSION, normalizeRelative } from '../../index/paths.js';
|
|
5
7
|
import { loadRuntimeConfig } from '../runtimeConfig.js';
|
|
@@ -260,6 +262,55 @@ function concatMerge(lanes) {
|
|
|
260
262
|
return merged;
|
|
261
263
|
}
|
|
262
264
|
|
|
265
|
+
// FR-202b (TASK-228): fire-and-forget lane telemetry. One JSONL line per
|
|
266
|
+
// retrieval at .ukit/storage/cache/retriever-lanes.jsonl; query persisted as
|
|
267
|
+
// sha256 only (no raw prompts on disk). Failures are swallowed — telemetry
|
|
268
|
+
// must never break retrieve(). Sync writes keep the fire-and-forget contract
|
|
269
|
+
// deterministic for callers/tests.
|
|
270
|
+
const RETRIEVER_LANES_MAX_BYTES = 256 * 1024;
|
|
271
|
+
const RETRIEVER_LANES_KEEP_LINES = 2000;
|
|
272
|
+
|
|
273
|
+
function emitRetrieverLaneTelemetry(rootDir, { query, mode, lanes, omitted, mergedPaths, merge, weights }) {
|
|
274
|
+
try {
|
|
275
|
+
const laneStats = {};
|
|
276
|
+
for (const laneName of LANE_ORDER) {
|
|
277
|
+
const lane = lanes.get(laneName);
|
|
278
|
+
const stat = {
|
|
279
|
+
hits: lane ? lane.length : 0,
|
|
280
|
+
weightUsed: weights?.[laneName] ?? 0,
|
|
281
|
+
};
|
|
282
|
+
const why = omitted?.find((o) => o?.what === `${laneName}-lane`)?.why;
|
|
283
|
+
if (!lane && why) stat.omitted = why;
|
|
284
|
+
if (lane) stat.paths = lane.map((entry) => entry.path).slice(0, 100);
|
|
285
|
+
laneStats[laneName] = stat;
|
|
286
|
+
}
|
|
287
|
+
const event = {
|
|
288
|
+
ts: Date.now(),
|
|
289
|
+
query: createHash('sha256').update(String(query ?? '')).digest('hex'),
|
|
290
|
+
mode,
|
|
291
|
+
lanes: laneStats,
|
|
292
|
+
mergedPaths,
|
|
293
|
+
merge,
|
|
294
|
+
};
|
|
295
|
+
const filePath = path.join(rootDir, '.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
|
|
296
|
+
fsSync.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
297
|
+
// Best-effort cap: past 256 KiB keep only the newest ~2000 lines.
|
|
298
|
+
try {
|
|
299
|
+
const st = fsSync.statSync(filePath);
|
|
300
|
+
if (st.size > RETRIEVER_LANES_MAX_BYTES) {
|
|
301
|
+
const kept = fsSync.readFileSync(filePath, 'utf8').split('\n').filter(Boolean)
|
|
302
|
+
.slice(-RETRIEVER_LANES_KEEP_LINES);
|
|
303
|
+
fsSync.writeFileSync(filePath, kept.length ? `${kept.join('\n')}\n` : '');
|
|
304
|
+
}
|
|
305
|
+
} catch {
|
|
306
|
+
// Missing/unreadable file is fine — append below recreates it.
|
|
307
|
+
}
|
|
308
|
+
fsSync.appendFileSync(filePath, `${JSON.stringify(event)}\n`);
|
|
309
|
+
} catch {
|
|
310
|
+
// Telemetry is advisory; never propagate.
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
263
314
|
/**
|
|
264
315
|
* retrieve(projectRoot, query, { mode, limit, snapshot, merge, weights }) →
|
|
265
316
|
* { anchors, evidence, relations, omitted }
|
|
@@ -384,6 +435,20 @@ export async function retrieve(projectRoot, query, { mode = 'search', limit, sna
|
|
|
384
435
|
omitted.push({ what: 'anchors', why: 'no-match' });
|
|
385
436
|
}
|
|
386
437
|
|
|
438
|
+
// FR-202b: emit lane telemetry only when a merge result exists — skipped
|
|
439
|
+
// entirely on the early no-index return and on empty merges.
|
|
440
|
+
if (merged.length > 0) {
|
|
441
|
+
emitRetrieverLaneTelemetry(rootDir, {
|
|
442
|
+
query: normalizedQuery,
|
|
443
|
+
mode,
|
|
444
|
+
lanes,
|
|
445
|
+
omitted,
|
|
446
|
+
mergedPaths: anchors.map((a) => a.path),
|
|
447
|
+
merge: effectiveMerge,
|
|
448
|
+
weights: effectiveWeights,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
|
|
387
452
|
const relations = [];
|
|
388
453
|
|
|
389
454
|
// Impact mode — 1-hop reverse imports: who imports each anchor file.
|
package/src/core/memory/store.js
CHANGED
|
@@ -540,7 +540,8 @@ async function proposePatternCandidateV2(projectRoot, projectId, candidate) {
|
|
|
540
540
|
.filter((record) => record.provenance === 'pattern-candidate'
|
|
541
541
|
|| String(record.provenance ?? '').includes('pattern-candidate'))
|
|
542
542
|
.find((record) => record.meta?.legacyStatus === 'pending'
|
|
543
|
-
&& normalizePatternText(record.text) === normalizedText
|
|
543
|
+
&& (normalizePatternText(record.text) === normalizedText
|
|
544
|
+
|| (candidate?.signature && record.meta?.signature === candidate.signature)));
|
|
544
545
|
if (existing) {
|
|
545
546
|
return patternCandidateFromRecord(existing);
|
|
546
547
|
}
|
|
@@ -553,6 +554,7 @@ async function proposePatternCandidateV2(projectRoot, projectId, candidate) {
|
|
|
553
554
|
detectedAt: Date.now(),
|
|
554
555
|
status: 'pending',
|
|
555
556
|
};
|
|
557
|
+
if (candidate?.signature) entry.signature = candidate.signature;
|
|
556
558
|
|
|
557
559
|
await v2.addRecord(projectRoot, {
|
|
558
560
|
type: 'derived_fact',
|
|
@@ -626,7 +628,9 @@ export async function proposePatternCandidate(projectRoot, projectId, candidate)
|
|
|
626
628
|
|
|
627
629
|
const patternCandidates = Array.isArray(memory.patternCandidates) ? memory.patternCandidates : [];
|
|
628
630
|
const existingPending = patternCandidates.find(
|
|
629
|
-
(entry) => entry.status === 'pending'
|
|
631
|
+
(entry) => entry.status === 'pending'
|
|
632
|
+
&& (normalizePatternText(entry.text) === normalizedText
|
|
633
|
+
|| (candidate?.signature && entry.signature === candidate.signature)),
|
|
630
634
|
);
|
|
631
635
|
if (existingPending) {
|
|
632
636
|
return existingPending;
|
|
@@ -640,6 +644,7 @@ export async function proposePatternCandidate(projectRoot, projectId, candidate)
|
|
|
640
644
|
detectedAt: Date.now(),
|
|
641
645
|
status: 'pending',
|
|
642
646
|
};
|
|
647
|
+
if (candidate?.signature) entry.signature = candidate.signature;
|
|
643
648
|
|
|
644
649
|
memory.patternCandidates = [...patternCandidates, entry];
|
|
645
650
|
await persistProjectMemoryWithHygiene(projectRoot, runtimePaths, projectId, filePath, memory);
|
|
@@ -209,6 +209,14 @@ export function buildDefaultRuntimeConfig(overrides = {}) {
|
|
|
209
209
|
maxRetries: 1,
|
|
210
210
|
confidenceThreshold: 50,
|
|
211
211
|
},
|
|
212
|
+
// Phase-4 learning loop (SPEC §8a). Advisory only — applyMode 'manual'
|
|
213
|
+
// means suggestions are computed + persisted, never auto-applied.
|
|
214
|
+
learning: {
|
|
215
|
+
feedback: { enabled: true, maxEvents: 200 },
|
|
216
|
+
proposals: { minCount: 3, minSessions: 2 },
|
|
217
|
+
episodes: { autoWrite: false },
|
|
218
|
+
tuning: { enabled: true, applyMode: 'manual' },
|
|
219
|
+
},
|
|
212
220
|
safePatch: {
|
|
213
221
|
enabled: true,
|
|
214
222
|
strictSharedRisk: true,
|
|
@@ -564,6 +572,51 @@ export function validateRuntimeConfig(config) {
|
|
|
564
572
|
pushPositiveNumberError(errors, config.safePatch.deltaMaxDiffCells, 'safePatch.deltaMaxDiffCells');
|
|
565
573
|
}
|
|
566
574
|
|
|
575
|
+
// learning is optional-present: absent → valid (old configs); present but
|
|
576
|
+
// not an object → error. Per-key checks apply only when the sub-object
|
|
577
|
+
// exists, so partial learning blocks stay valid via defaults merge.
|
|
578
|
+
if (config.learning !== undefined) {
|
|
579
|
+
if (!isPlainObject(config.learning)) {
|
|
580
|
+
errors.push('learning must be an object.');
|
|
581
|
+
} else {
|
|
582
|
+
const learning = config.learning;
|
|
583
|
+
if (learning.feedback !== undefined) {
|
|
584
|
+
if (!isPlainObject(learning.feedback)) {
|
|
585
|
+
errors.push('learning.feedback must be an object.');
|
|
586
|
+
} else {
|
|
587
|
+
pushBooleanError(errors, learning.feedback.enabled, 'learning.feedback.enabled');
|
|
588
|
+
pushPositiveNumberError(errors, learning.feedback.maxEvents, 'learning.feedback.maxEvents');
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
if (learning.proposals !== undefined) {
|
|
592
|
+
if (!isPlainObject(learning.proposals)) {
|
|
593
|
+
errors.push('learning.proposals must be an object.');
|
|
594
|
+
} else {
|
|
595
|
+
pushPositiveNumberError(errors, learning.proposals.minCount, 'learning.proposals.minCount');
|
|
596
|
+
pushPositiveNumberError(errors, learning.proposals.minSessions, 'learning.proposals.minSessions');
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
if (learning.episodes !== undefined) {
|
|
600
|
+
if (!isPlainObject(learning.episodes)) {
|
|
601
|
+
errors.push('learning.episodes must be an object.');
|
|
602
|
+
} else {
|
|
603
|
+
pushBooleanError(errors, learning.episodes.autoWrite, 'learning.episodes.autoWrite');
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
if (learning.tuning !== undefined) {
|
|
607
|
+
if (!isPlainObject(learning.tuning)) {
|
|
608
|
+
errors.push('learning.tuning must be an object.');
|
|
609
|
+
} else {
|
|
610
|
+
pushBooleanError(errors, learning.tuning.enabled, 'learning.tuning.enabled');
|
|
611
|
+
const VALID_APPLY_MODES = new Set(['manual', 'off']);
|
|
612
|
+
if (!VALID_APPLY_MODES.has(learning.tuning.applyMode)) {
|
|
613
|
+
errors.push(`learning.tuning.applyMode must be one of: ${[...VALID_APPLY_MODES].join(', ')}.`);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
|
|
567
620
|
if (!isPlainObject(config.subagents)) {
|
|
568
621
|
errors.push('subagents must be an object.');
|
|
569
622
|
} else {
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// failurePatterns.js — advisory failure-pattern mining over exec-ledger receipts (SI-103).
|
|
2
|
+
//
|
|
3
|
+
// What this is for: ledgers already record every verification receipt, but nothing
|
|
4
|
+
// distilled "which commands keep failing". This module scans
|
|
5
|
+
// `.ukit/storage/cache/exec-ledger/*.json`, extracts failed verification receipts,
|
|
6
|
+
// normalizes each command into a deterministic two-token signature, and persists the
|
|
7
|
+
// top patterns to `.ukit/storage/cache/failure-patterns.json` for `ukit metrics`.
|
|
8
|
+
//
|
|
9
|
+
// Contracts (asserted by tests/core/failurePatterns.test.js):
|
|
10
|
+
// * NEVER THROWS. Missing dir, malformed JSON, unreadable files → zeroed/partial
|
|
11
|
+
// result, no exception escapes. Artifact write failure is non-fatal: the computed
|
|
12
|
+
// object is still returned.
|
|
13
|
+
// * EVIDENCE-ONLY / ADVISORY. Zero writes to memory store, MEMORY.md, or routing
|
|
14
|
+
// inputs — the only side effect is the last-result cache artifact.
|
|
15
|
+
// * Deterministic normalization: first two whitespace tokens, lowercased;
|
|
16
|
+
// path-like and numeric tokens collapse to `<arg>`.
|
|
17
|
+
// * Same ledger-dir discipline as SPEC §4: glob `*.json`, skip
|
|
18
|
+
// `gate-crash-counter.json`, `*.journal*`, `*.quarantine*`, bound by mtime.
|
|
19
|
+
|
|
20
|
+
import fs from 'node:fs/promises';
|
|
21
|
+
import path from 'node:path';
|
|
22
|
+
import { listLedgerFiles, LEDGER_DIR_REL } from './ledgerFiles.js';
|
|
23
|
+
|
|
24
|
+
const ARTIFACT_REL = path.join('.ukit', 'storage', 'cache', 'failure-patterns.json');
|
|
25
|
+
const MAX_PATTERNS = 10;
|
|
26
|
+
const MAX_EXAMPLES = 3;
|
|
27
|
+
|
|
28
|
+
// Receipt kinds that count as verification-ish evidence.
|
|
29
|
+
const VERIFY_KINDS = new Set(['verify', 'verification']);
|
|
30
|
+
|
|
31
|
+
function isObject(value) {
|
|
32
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// First two whitespace tokens, lowercased; path-like or numeric tokens become `<arg>`.
|
|
36
|
+
function normalizeSignature(command) {
|
|
37
|
+
if (typeof command !== 'string') return null;
|
|
38
|
+
const tokens = command.trim().toLowerCase().split(/\s+/).filter(Boolean);
|
|
39
|
+
if (tokens.length === 0) return null;
|
|
40
|
+
const head = tokens.slice(0, 2).map((token) => {
|
|
41
|
+
if (/^\d+(\.\d+)?$/.test(token)) return '<arg>';
|
|
42
|
+
if (token.includes('/') || token.includes('\\') || token.endsWith('.js')
|
|
43
|
+
|| token.endsWith('.ts') || token.endsWith('.json')) {
|
|
44
|
+
return '<arg>';
|
|
45
|
+
}
|
|
46
|
+
return token;
|
|
47
|
+
});
|
|
48
|
+
return head.join(' ');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Extract failure events from one parsed ledger. A failure event is a receipt with
|
|
52
|
+
// `success === false` on a verification-ish kind carrying a `command`, OR (when the
|
|
53
|
+
// ledger declares `verificationFailed === true`) any receipt carrying a `command`.
|
|
54
|
+
function failureCommands(ledger) {
|
|
55
|
+
if (!isObject(ledger) || !Array.isArray(ledger.receipts)) return [];
|
|
56
|
+
const out = [];
|
|
57
|
+
for (const receipt of ledger.receipts) {
|
|
58
|
+
if (!isObject(receipt) || typeof receipt.command !== 'string') continue;
|
|
59
|
+
const verifyFail = receipt.success === false && VERIFY_KINDS.has(receipt.kind);
|
|
60
|
+
const ledgerFlagged = ledger.verificationFailed === true && receipt.success === false;
|
|
61
|
+
if (verifyFail || ledgerFlagged) {
|
|
62
|
+
out.push({ command: receipt.command, file: receipt.file });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function writeArtifact(filePath, result) {
|
|
69
|
+
const dir = path.dirname(filePath);
|
|
70
|
+
const tmp = path.join(dir, `.failure-patterns-${process.pid}.tmp`);
|
|
71
|
+
try {
|
|
72
|
+
await fs.mkdir(dir, { recursive: true });
|
|
73
|
+
await fs.writeFile(tmp, JSON.stringify(result, null, 2));
|
|
74
|
+
await fs.rename(tmp, filePath);
|
|
75
|
+
} catch {
|
|
76
|
+
// Non-fatal: artifact is advisory; the computed result is still returned.
|
|
77
|
+
await fs.rm(tmp, { force: true }).catch(() => {});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Mine verification-failure patterns from exec-ledger receipts.
|
|
83
|
+
*
|
|
84
|
+
* @param {string} projectRoot repository root containing `.ukit/storage/cache/exec-ledger/`.
|
|
85
|
+
* @param {{ limitLedgers?: number, minCount?: number }} [options]
|
|
86
|
+
* @returns {Promise<{generatedAt: string, ledgersScanned: number,
|
|
87
|
+
* ledgersWithFailures: number,
|
|
88
|
+
* patterns: Array<{signature: string, count: number, sessions: number,
|
|
89
|
+
* commands: string[], files: string[]}>}>} never throws.
|
|
90
|
+
*/
|
|
91
|
+
export async function mineFailurePatterns(projectRoot, { limitLedgers = 500, minCount = 2 } = {}) {
|
|
92
|
+
const result = {
|
|
93
|
+
generatedAt: new Date().toISOString(),
|
|
94
|
+
ledgersScanned: 0,
|
|
95
|
+
ledgersWithFailures: 0,
|
|
96
|
+
patterns: [],
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
try {
|
|
100
|
+
const dir = path.join(projectRoot, LEDGER_DIR_REL);
|
|
101
|
+
const names = await listLedgerFiles(dir, limitLedgers);
|
|
102
|
+
const groups = new Map();
|
|
103
|
+
|
|
104
|
+
for (const name of names) {
|
|
105
|
+
let ledger;
|
|
106
|
+
try {
|
|
107
|
+
ledger = JSON.parse(await fs.readFile(path.join(dir, name), 'utf8'));
|
|
108
|
+
} catch {
|
|
109
|
+
continue; // malformed/unreadable ledger — skip, never throw
|
|
110
|
+
}
|
|
111
|
+
result.ledgersScanned += 1;
|
|
112
|
+
|
|
113
|
+
const failures = failureCommands(ledger);
|
|
114
|
+
if (failures.length === 0) continue;
|
|
115
|
+
result.ledgersWithFailures += 1;
|
|
116
|
+
const sessionId = typeof ledger.sessionId === 'string' ? ledger.sessionId : null;
|
|
117
|
+
|
|
118
|
+
for (const { command, file } of failures) {
|
|
119
|
+
const signature = normalizeSignature(command);
|
|
120
|
+
if (!signature) continue;
|
|
121
|
+
let group = groups.get(signature);
|
|
122
|
+
if (!group) {
|
|
123
|
+
group = { signature, count: 0, sessions: new Set(), commands: [], files: [] };
|
|
124
|
+
groups.set(signature, group);
|
|
125
|
+
}
|
|
126
|
+
group.count += 1;
|
|
127
|
+
if (sessionId) group.sessions.add(sessionId);
|
|
128
|
+
if (group.commands.length < MAX_EXAMPLES && !group.commands.includes(command)) {
|
|
129
|
+
group.commands.push(command);
|
|
130
|
+
}
|
|
131
|
+
if (typeof file === 'string' && group.files.length < MAX_EXAMPLES && !group.files.includes(file)) {
|
|
132
|
+
group.files.push(file);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
result.patterns = [...groups.values()]
|
|
138
|
+
.filter((g) => g.count >= minCount)
|
|
139
|
+
.sort((a, b) => b.count - a.count || a.signature.localeCompare(b.signature))
|
|
140
|
+
.slice(0, MAX_PATTERNS)
|
|
141
|
+
.map((g) => ({
|
|
142
|
+
signature: g.signature,
|
|
143
|
+
count: g.count,
|
|
144
|
+
sessions: g.sessions.size,
|
|
145
|
+
commands: g.commands,
|
|
146
|
+
files: g.files,
|
|
147
|
+
}));
|
|
148
|
+
} catch {
|
|
149
|
+
// Any unexpected failure → return whatever was computed so far.
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
await writeArtifact(path.join(projectRoot, ARTIFACT_REL), result);
|
|
153
|
+
return result;
|
|
154
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// feedbackEvents.js — labeled wrong-route feedback events collector (SPEC C32 §3b).
|
|
2
|
+
//
|
|
3
|
+
// Derives labeled events from telemetry already on disk:
|
|
4
|
+
// * rescue — route-audit entry with `rescueMode != null` → detail = rescueMode
|
|
5
|
+
// * re-route — consecutive same-fingerprint audit entries where the later
|
|
6
|
+
// entry changed executionMode (detail "<prev>→<new>")
|
|
7
|
+
// * repeat-stall — audit `repeatCount >= minRepeat` with no matching
|
|
8
|
+
// ledger `writeSucceeded` (join on requestKey)
|
|
9
|
+
// * user-correction — manual labels appended by `ukit feedback "<text>"` to
|
|
10
|
+
// `.ukit/storage/cache/feedback-manual.jsonl`
|
|
11
|
+
//
|
|
12
|
+
// Contracts:
|
|
13
|
+
// * NEVER THROWS. Missing/malformed telemetry → zeroed result, `error` field
|
|
14
|
+
// on failure. Artifact write failure is non-fatal.
|
|
15
|
+
// * EVIDENCE-ONLY. The only side effect is the advisory artifact
|
|
16
|
+
// `.ukit/storage/learning/feedback-events.json` (tmp+rename).
|
|
17
|
+
// * Gated by config `learning?.feedback?.enabled !== false` (default true;
|
|
18
|
+
// the `learning` namespace lands in TASK-232 so the read is defensive).
|
|
19
|
+
|
|
20
|
+
import fs from 'node:fs/promises';
|
|
21
|
+
import path from 'node:path';
|
|
22
|
+
import { listLedgerFiles, LEDGER_DIR_REL } from './ledgerFiles.js';
|
|
23
|
+
|
|
24
|
+
const CACHE_DIR_REL = path.join('.ukit', 'storage', 'cache');
|
|
25
|
+
const AUDIT_REL = path.join(CACHE_DIR_REL, 'route-audit.json');
|
|
26
|
+
const MANUAL_REL = path.join(CACHE_DIR_REL, 'feedback-manual.jsonl');
|
|
27
|
+
const ARTIFACT_REL = path.join('.ukit', 'storage', 'learning', 'feedback-events.json');
|
|
28
|
+
const CONFIG_REL = path.join('.ukit', 'storage', 'config.json');
|
|
29
|
+
const MAX_EVENTS = 200;
|
|
30
|
+
|
|
31
|
+
function isObject(value) {
|
|
32
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function zeroedResult(extra = {}) {
|
|
36
|
+
return {
|
|
37
|
+
generatedAt: new Date().toISOString(),
|
|
38
|
+
auditRowsScanned: 0,
|
|
39
|
+
ledgersScanned: 0,
|
|
40
|
+
byKind: { rescue: 0, 're-route': 0, 'repeat-stall': 0, 'user-correction': 0 },
|
|
41
|
+
events: [],
|
|
42
|
+
...extra,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function readJson(filePath) {
|
|
47
|
+
try {
|
|
48
|
+
return JSON.parse(await fs.readFile(filePath, 'utf8'));
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Default true when `learning`/`feedback` is absent (namespace lands in TASK-232).
|
|
55
|
+
async function feedbackEnabled(projectRoot) {
|
|
56
|
+
const config = await readJson(path.join(projectRoot, CONFIG_REL));
|
|
57
|
+
return config?.learning?.feedback?.enabled !== false;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Manual jsonl is field-tolerant: unknown keys ignored, a legacy `context`
|
|
61
|
+
// key is treated as an optional extra. Bad lines are skipped, not fatal.
|
|
62
|
+
async function readManualEvents(projectRoot, byKind) {
|
|
63
|
+
const events = [];
|
|
64
|
+
let raw;
|
|
65
|
+
try {
|
|
66
|
+
raw = await fs.readFile(path.join(projectRoot, MANUAL_REL), 'utf8');
|
|
67
|
+
} catch {
|
|
68
|
+
return events;
|
|
69
|
+
}
|
|
70
|
+
for (const line of raw.split('\n')) {
|
|
71
|
+
const trimmed = line.trim();
|
|
72
|
+
if (!trimmed) continue;
|
|
73
|
+
let record;
|
|
74
|
+
try {
|
|
75
|
+
record = JSON.parse(trimmed);
|
|
76
|
+
} catch {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
if (!isObject(record) || typeof record.text !== 'string') continue;
|
|
80
|
+
const event = {
|
|
81
|
+
ts: typeof record.ts === 'number' ? record.ts : 0,
|
|
82
|
+
kind: 'user-correction',
|
|
83
|
+
detail: record.text,
|
|
84
|
+
source: 'manual',
|
|
85
|
+
};
|
|
86
|
+
if (typeof record.targetFile === 'string') event.targetFile = record.targetFile;
|
|
87
|
+
events.push(event);
|
|
88
|
+
byKind['user-correction'] += 1;
|
|
89
|
+
}
|
|
90
|
+
return events;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function writeArtifact(filePath, result) {
|
|
94
|
+
const dir = path.dirname(filePath);
|
|
95
|
+
const tmp = path.join(dir, `.feedback-events-${process.pid}.tmp`);
|
|
96
|
+
try {
|
|
97
|
+
await fs.mkdir(dir, { recursive: true });
|
|
98
|
+
await fs.writeFile(tmp, JSON.stringify(result, null, 2));
|
|
99
|
+
await fs.rename(tmp, filePath);
|
|
100
|
+
} catch {
|
|
101
|
+
await fs.rm(tmp, { force: true }).catch(() => {});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Collect labeled wrong-route feedback events.
|
|
107
|
+
*
|
|
108
|
+
* @param {string} projectRoot repository root containing `.ukit/storage/`.
|
|
109
|
+
* @param {{ limitLedgers?: number, minRepeat?: number }} [options]
|
|
110
|
+
* @returns {Promise<{generatedAt: string, auditRowsScanned: number,
|
|
111
|
+
* ledgersScanned: number, byKind: Record<string, number>,
|
|
112
|
+
* events: Array<object>, error?: string}>} never throws.
|
|
113
|
+
*/
|
|
114
|
+
export async function collectFeedbackEvents(projectRoot, { limitLedgers = 500, minRepeat = 2 } = {}) {
|
|
115
|
+
const result = zeroedResult();
|
|
116
|
+
try {
|
|
117
|
+
if (!(await feedbackEnabled(projectRoot))) {
|
|
118
|
+
return result;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const auditDoc = await readJson(path.join(projectRoot, AUDIT_REL));
|
|
122
|
+
const auditEntries = Array.isArray(auditDoc?.entries)
|
|
123
|
+
? auditDoc.entries.filter(isObject)
|
|
124
|
+
: [];
|
|
125
|
+
result.auditRowsScanned = auditEntries.length;
|
|
126
|
+
|
|
127
|
+
const ledgerDir = path.join(projectRoot, LEDGER_DIR_REL);
|
|
128
|
+
const ledgerNames = await listLedgerFiles(ledgerDir, limitLedgers);
|
|
129
|
+
const writeOkKeys = new Set();
|
|
130
|
+
for (const name of ledgerNames) {
|
|
131
|
+
const ledger = await readJson(path.join(ledgerDir, name));
|
|
132
|
+
if (!isObject(ledger)) continue;
|
|
133
|
+
result.ledgersScanned += 1;
|
|
134
|
+
if (ledger.writeSucceeded === true && typeof ledger.requestKey === 'string') {
|
|
135
|
+
writeOkKeys.add(ledger.requestKey);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const events = [];
|
|
140
|
+
for (const entry of auditEntries) {
|
|
141
|
+
const base = {
|
|
142
|
+
ts: typeof entry.ts === 'number' ? entry.ts : 0,
|
|
143
|
+
requestKey: typeof entry.requestKey === 'string' ? entry.requestKey : undefined,
|
|
144
|
+
targetFile: typeof entry.targetFile === 'string' ? entry.targetFile : undefined,
|
|
145
|
+
taskType: typeof entry.taskType === 'string' ? entry.taskType : undefined,
|
|
146
|
+
executionMode: typeof entry.executionMode === 'string' ? entry.executionMode : undefined,
|
|
147
|
+
source: 'derived',
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
if (entry.rescueMode != null) {
|
|
151
|
+
events.push({ ...base, kind: 'rescue', detail: String(entry.rescueMode) });
|
|
152
|
+
result.byKind.rescue += 1;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if ((entry.repeatCount ?? 0) >= minRepeat && !writeOkKeys.has(entry.requestKey)) {
|
|
156
|
+
events.push({ ...base, kind: 'repeat-stall', detail: `repeatCount=${entry.repeatCount}` });
|
|
157
|
+
result.byKind['repeat-stall'] += 1;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// re-route: audit is newest-first — entry i is the later entry, i+1 the
|
|
162
|
+
// previous one for the same promptFingerprint + targetFile.
|
|
163
|
+
for (let i = 0; i < auditEntries.length - 1; i += 1) {
|
|
164
|
+
const later = auditEntries[i];
|
|
165
|
+
const prev = auditEntries[i + 1];
|
|
166
|
+
if (typeof later.promptFingerprint !== 'string' || !later.promptFingerprint) continue;
|
|
167
|
+
if (later.promptFingerprint !== prev.promptFingerprint) continue;
|
|
168
|
+
if (later.targetFile !== prev.targetFile) continue;
|
|
169
|
+
if (later.wideningBlocked === true) continue;
|
|
170
|
+
const prevMode = prev.executionMode;
|
|
171
|
+
const newMode = later.executionMode;
|
|
172
|
+
if (prevMode == null || newMode == null || prevMode === newMode) continue;
|
|
173
|
+
events.push({
|
|
174
|
+
ts: typeof later.ts === 'number' ? later.ts : 0,
|
|
175
|
+
kind: 're-route',
|
|
176
|
+
requestKey: typeof later.requestKey === 'string' ? later.requestKey : undefined,
|
|
177
|
+
targetFile: typeof later.targetFile === 'string' ? later.targetFile : undefined,
|
|
178
|
+
taskType: typeof later.taskType === 'string' ? later.taskType : undefined,
|
|
179
|
+
executionMode: typeof later.executionMode === 'string' ? later.executionMode : undefined,
|
|
180
|
+
detail: `${prevMode}→${newMode}`,
|
|
181
|
+
source: 'derived',
|
|
182
|
+
});
|
|
183
|
+
result.byKind['re-route'] += 1;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
events.push(...await readManualEvents(projectRoot, result.byKind));
|
|
187
|
+
|
|
188
|
+
events.sort((a, b) => b.ts - a.ts);
|
|
189
|
+
result.events = events.slice(0, MAX_EVENTS);
|
|
190
|
+
} catch (error) {
|
|
191
|
+
result.error = error?.message ?? String(error);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
await writeArtifact(path.join(projectRoot, ARTIFACT_REL), result);
|
|
195
|
+
return result;
|
|
196
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// laneStats.js — retriever lane contribution stats (SPEC C32 §5b).
|
|
2
|
+
//
|
|
3
|
+
// Reads `.ukit/storage/cache/retriever-lanes.jsonl` emitted by the retriever
|
|
4
|
+
// (FR-202b) and rolls up per-lane counters:
|
|
5
|
+
// retrievals = events where the lane had hits > 0
|
|
6
|
+
// totalHits = sum of lane hits
|
|
7
|
+
// exclusive = events where this lane was the SOLE lane with hits > 0
|
|
8
|
+
// (emitters record per-lane hit counts only, so exclusivity
|
|
9
|
+
// is approximated rather than per-path recomputed)
|
|
10
|
+
// avgWeight = mean weightUsed over events that recorded one
|
|
11
|
+
// contribution = retrievals / max(1, events)
|
|
12
|
+
//
|
|
13
|
+
// Contracts: NEVER THROWS — missing/empty file → `empty: true` + zeroed lanes.
|
|
14
|
+
|
|
15
|
+
import fs from 'node:fs/promises';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
|
|
18
|
+
const LANES_REL = path.join('.ukit', 'storage', 'cache', 'retriever-lanes.jsonl');
|
|
19
|
+
|
|
20
|
+
function isObject(value) {
|
|
21
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function zeroedResult(extra = {}) {
|
|
25
|
+
return {
|
|
26
|
+
generatedAt: new Date().toISOString(),
|
|
27
|
+
events: 0,
|
|
28
|
+
lanes: {},
|
|
29
|
+
empty: false,
|
|
30
|
+
...extra,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function emptyLaneBucket() {
|
|
35
|
+
return {
|
|
36
|
+
retrievals: 0,
|
|
37
|
+
totalHits: 0,
|
|
38
|
+
exclusive: 0,
|
|
39
|
+
avgWeight: 0,
|
|
40
|
+
contribution: 0,
|
|
41
|
+
_weightSum: 0,
|
|
42
|
+
_weightCount: 0,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Collect retriever lane contribution stats.
|
|
48
|
+
*
|
|
49
|
+
* @param {string} projectRoot repository root containing `.ukit/storage/`.
|
|
50
|
+
* @param {{ maxEvents?: number }} [options]
|
|
51
|
+
* @returns {Promise<object>} never throws.
|
|
52
|
+
*/
|
|
53
|
+
export async function collectLaneStats(projectRoot, { maxEvents = 2000 } = {}) {
|
|
54
|
+
const result = zeroedResult();
|
|
55
|
+
try {
|
|
56
|
+
let raw;
|
|
57
|
+
try {
|
|
58
|
+
raw = await fs.readFile(path.join(projectRoot, LANES_REL), 'utf8');
|
|
59
|
+
} catch {
|
|
60
|
+
return { ...result, empty: true };
|
|
61
|
+
}
|
|
62
|
+
const lines = raw.split('\n');
|
|
63
|
+
const events = [];
|
|
64
|
+
for (const line of lines) {
|
|
65
|
+
const trimmed = line.trim();
|
|
66
|
+
if (!trimmed) continue;
|
|
67
|
+
try {
|
|
68
|
+
const record = JSON.parse(trimmed);
|
|
69
|
+
if (isObject(record) && isObject(record.lanes)) events.push(record);
|
|
70
|
+
} catch {
|
|
71
|
+
// skip malformed line
|
|
72
|
+
}
|
|
73
|
+
if (events.length >= maxEvents) break;
|
|
74
|
+
}
|
|
75
|
+
if (events.length === 0) {
|
|
76
|
+
return { ...result, empty: true };
|
|
77
|
+
}
|
|
78
|
+
result.events = events.length;
|
|
79
|
+
|
|
80
|
+
for (const event of events) {
|
|
81
|
+
const positive = [];
|
|
82
|
+
for (const [lane, stats] of Object.entries(event.lanes)) {
|
|
83
|
+
if (!isObject(stats)) continue;
|
|
84
|
+
const bucket = result.lanes[lane] ??= emptyLaneBucket();
|
|
85
|
+
const hits = typeof stats.hits === 'number' ? stats.hits : 0;
|
|
86
|
+
if (hits > 0) {
|
|
87
|
+
bucket.retrievals += 1;
|
|
88
|
+
bucket.totalHits += hits;
|
|
89
|
+
positive.push(lane);
|
|
90
|
+
}
|
|
91
|
+
if (typeof stats.weightUsed === 'number') {
|
|
92
|
+
bucket._weightSum += stats.weightUsed;
|
|
93
|
+
bucket._weightCount += 1;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (positive.length === 1) {
|
|
97
|
+
result.lanes[positive[0]].exclusive += 1;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
for (const bucket of Object.values(result.lanes)) {
|
|
102
|
+
bucket.avgWeight = bucket._weightCount === 0 ? 0 : bucket._weightSum / bucket._weightCount;
|
|
103
|
+
bucket.contribution = bucket.retrievals / Math.max(1, result.events);
|
|
104
|
+
delete bucket._weightSum;
|
|
105
|
+
delete bucket._weightCount;
|
|
106
|
+
}
|
|
107
|
+
} catch (error) {
|
|
108
|
+
result.error = error?.message ?? String(error);
|
|
109
|
+
}
|
|
110
|
+
return result;
|
|
111
|
+
}
|