@ngockhoale/ukit 3.5.5 → 3.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/manifests/engineConformance.yaml +0 -13
  3. package/manifests/platform.full.yaml +25 -0
  4. package/package.json +1 -8
  5. package/src/cli/commands/indexTools.js +1 -0
  6. package/src/cli/commands/memory.js +33 -4
  7. package/src/core/applyPlan.js +25 -72
  8. package/src/core/diffPlan.js +38 -9
  9. package/src/core/managedTemplateMerge.js +146 -0
  10. package/src/core/memory/memoryFlags.js +5 -6
  11. package/src/core/memory/policy.js +5 -0
  12. package/src/core/memory/store.js +17 -0
  13. package/src/core/metadata.js +12 -8
  14. package/src/core/runInstallPipeline.js +16 -10
  15. package/src/core/runtimeConfig.js +14 -83
  16. package/src/core/uninstall.js +8 -1
  17. package/src/decision/registry.js +17 -17
  18. package/src/decision/runtimeDecide.js +5 -5
  19. package/src/index/queryIndex.js +41 -4
  20. package/src/index/routeResolver.js +2 -2
  21. package/src/index/semanticLane.js +269 -0
  22. package/src/learning/lessonClass.js +110 -0
  23. package/src/learning/patternProposals.js +111 -7
  24. package/src/learning/reeditMiner.js +104 -0
  25. package/src/learning/selfImprove.js +9 -1
  26. package/template_project/.claude/skills/canvas-design/SKILL.md +2 -2
  27. package/template_project/.claude/skills/canvas-design/canvas-fonts/{InstrumentSans-OFL.txt → InstrumentSerif-OFL.txt} +3 -3
  28. package/template_project/.claude/skills/canvas-design/scripts/fetch-fonts.mjs +148 -0
  29. package/template_project/.claude/ukit/index/embed-index.mjs +26 -0
  30. package/template_project/.claude/ukit/index/lib/index-core.mjs +43 -4
  31. package/template_project/.claude/ukit/index/lib/semantic-lane.mjs +269 -0
  32. package/template_project/.claude/ukit/index/query-index.mjs +9 -1
  33. package/template_project/.claude/ukit/index/route-resolver.mjs +2 -2
  34. package/template_project/.claude/ukit/runtime/memory-policy.mjs +5 -0
  35. package/template_project/ukit/storage/config.json +0 -32
  36. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Bold.ttf +0 -0
  37. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-OFL.txt +0 -93
  38. package/template_project/.claude/skills/canvas-design/canvas-fonts/BricolageGrotesque-Regular.ttf +0 -0
  39. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Bold.ttf +0 -0
  40. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-BoldItalic.ttf +0 -0
  41. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Italic.ttf +0 -0
  42. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSans-Regular.ttf +0 -0
  43. package/template_project/.claude/skills/canvas-design/canvas-fonts/InstrumentSerif-Italic.ttf +0 -0
  44. package/template_project/.claude/skills/canvas-design/canvas-fonts/JetBrainsMono-Bold.ttf +0 -0
  45. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Bold.ttf +0 -0
  46. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-BoldItalic.ttf +0 -0
  47. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Italic.ttf +0 -0
  48. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-OFL.txt +0 -93
  49. package/template_project/.claude/skills/canvas-design/canvas-fonts/Lora-Regular.ttf +0 -0
  50. package/template_project/.claude/skills/canvas-design/canvas-fonts/Outfit-Bold.ttf +0 -0
  51. package/template_project/.claude/skills/canvas-design/canvas-fonts/Tektur-Medium.ttf +0 -0
  52. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-OFL.txt +0 -93
  53. package/template_project/.claude/skills/canvas-design/canvas-fonts/YoungSerif-Regular.ttf +0 -0
@@ -0,0 +1,269 @@
1
+ // Real-embedding lane for `ukit index query` (opt-in; no dependency; global fetch only).
2
+ //
3
+ // PRIVACY: building vectors SENDS FILE TEXT (path, symbol names, leading comment and,
4
+ // at most ~240 characters per file) to the endpoint in
5
+ // UKIT_INDEX_EMBED_URL, and queries send the question text. With no URL configured nothing
6
+ // here runs and nothing leaves the machine. Sensitive paths (.env, keys, credentials ...)
7
+ // are never embedded; only files already present in the index are considered.
8
+ //
9
+ // Byte-identical mirror: template_project/.claude/ukit/index/lib/semantic-lane.mjs
10
+ // (tests/index/semanticLane.test.js enforces it). Keep this file dependency-free.
11
+ //
12
+ // Env: UKIT_INDEX_EMBED_URL (base URL ending in /v1), UKIT_INDEX_EMBED_MODEL,
13
+ // UKIT_INDEX_EMBED_KEY (optional Bearer).
14
+ // Build is an explicit command (embed-index.mjs), never part of index build/refresh, never in
15
+ // a hook. Query use is reachable only through queryCodeIndex({ semantic: true }), which only
16
+ // the CLI query path passes.
17
+ import fs from 'node:fs/promises';
18
+ import path from 'node:path';
19
+ import crypto from 'node:crypto';
20
+
21
+ export const SEMANTIC_MANIFEST = 'semantic.json';
22
+ export const SEMANTIC_BIN = 'semantic.bin';
23
+ // A CLI query waits at most this long for the embedding of the question. Measured on the
24
+ // reference endpoint: warm question embedding ~0.45 s, 64 descriptors ~24 s (serial service).
25
+ export const QUERY_TIMEOUT_MS = 1500;
26
+ // One descriptor per file, ~60 tokens. Endpoint throughput is ~170-200 tokens/s and serial, so
27
+ // file bodies (>1 h for 2.4k files) are not embedded; descriptors build in minutes.
28
+ export const DOC_CHARS = 240;
29
+ const NAMES_CHARS = 90;
30
+ const LEAD_CHARS = 100;
31
+ const QUERY_INSTRUCTION = 'Instruct: Given a question about a code repository, retrieve the source file that answers it\nQuery: ';
32
+
33
+ export function embedConfig(env = process.env) {
34
+ // Accepts the base (.../v1) or the full endpoint (.../v1/embeddings).
35
+ const url = String(env.UKIT_INDEX_EMBED_URL ?? '').trim().replace(/\/+$/, '').replace(/\/embeddings$/i, '');
36
+ if (!url) return null;
37
+ return { url, model: String(env.UKIT_INDEX_EMBED_MODEL ?? '').trim() || 'unic-embedding', key: String(env.UKIT_INDEX_EMBED_KEY ?? '').trim() };
38
+ }
39
+
40
+ function isSensitive(value) {
41
+ const segments = String(value ?? '').replace(/\\/g, '/').split('/');
42
+ const base = segments[segments.length - 1] ?? '';
43
+ return base.toLowerCase().startsWith('.env')
44
+ || /\.(?:pem|key|p12|pfx|jks|keystore)$/i.test(base)
45
+ || /^(?:credentials?|\.npmrc|\.netrc|\.pypirc)(?:\.[^/]*)?$/i.test(base)
46
+ || /^id_(?:rsa|dsa|ecdsa|ed25519)(?:\.[^/]*)?$/i.test(base)
47
+ || segments.includes('.ssh') || segments.includes('.aws') || segments.includes('.gnupg');
48
+ }
49
+
50
+ const sha = (text) => crypto.createHash('sha256').update(text).digest('hex').slice(0, 24);
51
+
52
+ // POST /embeddings; returns arrays in input order or throws. Validates shape.
53
+ export async function embedTexts(texts, cfg, { timeoutMs = 60000 } = {}) {
54
+ const headers = { 'content-type': 'application/json' };
55
+ if (cfg.key) headers.authorization = `Bearer ${cfg.key}`;
56
+ const response = await fetch(`${cfg.url}/embeddings`, {
57
+ method: 'POST',
58
+ headers,
59
+ body: JSON.stringify({ model: cfg.model, input: texts }),
60
+ signal: AbortSignal.timeout(timeoutMs),
61
+ });
62
+ if (!response.ok) throw new Error(`embedding endpoint HTTP ${response.status}`);
63
+ const data = (await response.json())?.data;
64
+ if (!Array.isArray(data) || data.length !== texts.length) throw new Error('embedding response shape mismatch');
65
+ const rows = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0)).map((d) => d.embedding);
66
+ const dims = rows[0]?.length;
67
+ if (!dims || rows.some((r) => !Array.isArray(r) || r.length !== dims || r.some((x) => !Number.isFinite(x)))) {
68
+ throw new Error('embedding response vectors invalid');
69
+ }
70
+ return rows;
71
+ }
72
+
73
+ function normalizeTo(vec, dims) {
74
+ const out = vec.slice(0, dims);
75
+ let n = 0;
76
+ for (const x of out) n += x * x;
77
+ n = Math.sqrt(n) || 1;
78
+ return out.map((x) => x / n);
79
+ }
80
+
81
+ // Descriptor for one file: path + exported/declared symbol names + leading doc comment,
82
+ // capped at DOC_CHARS. The path is never truncated; names and comment share the remainder.
83
+ export function buildDocumentText({ filePath, symbols = [], content = '' }) {
84
+ const names = [...new Set(symbols)].join(' ').slice(0, NAMES_CHARS);
85
+ const lines = String(content).split('\n');
86
+ const lead = [];
87
+ for (const line of lines.slice(0, 80)) {
88
+ const t = line.trim();
89
+ if (!t || t.startsWith('#!') || t === '---' || t.startsWith("'use ")) { if (lead.length) break; continue; }
90
+ if (/^(\/\/|\/\*|\*|#|<!--|"""|--)/.test(t)) lead.push(t.replace(/^(\/\/+|\/\*+|\*+\/?|#+|<!--|-->|"""|--)\s?/, ''));
91
+ else break;
92
+ }
93
+ const leading = (lead.length ? lead.join(' ') : '').slice(0, LEAD_CHARS);
94
+ const text = `${filePath}\n${names}\n${leading}`;
95
+ return text.length > DOC_CHARS ? text.slice(0, DOC_CHARS) : text;
96
+ }
97
+
98
+ async function readJson(file) {
99
+ try { return JSON.parse(await fs.readFile(file, 'utf8')); } catch { return null; }
100
+ }
101
+
102
+ async function loadStore(indexDir) {
103
+ const manifest = await readJson(path.join(indexDir, SEMANTIC_MANIFEST));
104
+ if (!manifest || manifest.v !== 1 || !Array.isArray(manifest.order)) return null;
105
+ let bin;
106
+ try { bin = await fs.readFile(path.join(indexDir, SEMANTIC_BIN)); } catch { return null; }
107
+ const rec = 4 + manifest.dims;
108
+ if (bin.length !== rec * manifest.order.length) return null;
109
+ return { manifest, bin, rec };
110
+ }
111
+
112
+ // Explicit build step. Embeds only documents whose hash (document text + model) is not in the
113
+ // existing store. Returns stats. Never called from index build/refresh or hooks.
114
+ export async function buildSemanticVectors({
115
+ rootDir = process.cwd(), env = process.env, dims = 0,
116
+ batchSize = 64, onProgress = null,
117
+ } = {}) {
118
+ const cfg = embedConfig(env);
119
+ if (!cfg) throw new Error('UKIT_INDEX_EMBED_URL is not set; semantic vectors are opt-in');
120
+ const root = path.resolve(rootDir);
121
+ const indexDir = path.join(root, '.cache', 'index');
122
+ const filesArtifact = await readJson(path.join(indexDir, 'files.json'));
123
+ const symbolsArtifact = await readJson(path.join(indexDir, 'symbols.json'));
124
+ if (!filesArtifact?.items) throw new Error('index missing; run the index build first');
125
+ const symbolsByFile = new Map();
126
+ for (const s of symbolsArtifact?.items ?? []) {
127
+ if (!symbolsByFile.has(s.filePath)) symbolsByFile.set(s.filePath, []);
128
+ symbolsByFile.get(s.filePath).push(s.name);
129
+ }
130
+ // Reuse vectors from a store built with the same model and truncation.
131
+ const old = await loadStore(indexDir);
132
+ const reusable = old && old.manifest.model === cfg.model
133
+ && (!dims || old.manifest.dims === dims) ? old : null;
134
+ const oldByHash = new Map();
135
+ if (reusable) reusable.manifest.order.forEach(([, h], i) => oldByHash.set(h, reusable.bin.subarray(i * reusable.rec, (i + 1) * reusable.rec)));
136
+
137
+ const docs = [];
138
+ for (const item of filesArtifact.items) {
139
+ const filePath = item.filePath;
140
+ if (typeof filePath !== 'string' || isSensitive(filePath)) continue;
141
+ let content = '';
142
+ try { content = await fs.readFile(path.join(root, filePath), 'utf8'); } catch { continue; }
143
+ const text = buildDocumentText({ filePath, symbols: symbolsByFile.get(filePath) ?? [], content });
144
+ docs.push({ filePath, text, hash: sha(`${cfg.model}\0${text}`) });
145
+ }
146
+ const missing = [...new Map(docs.filter((d) => !oldByHash.has(d.hash)).map((d) => [d.hash, d])).values()];
147
+ const fresh = new Map();
148
+ let outDims = dims || reusable?.manifest.dims || 0;
149
+ // Requests are strictly sequential: the embedding service is serial, so parallel requests
150
+ // only queue. Partial progress is saved: a failed batch leaves its files without vectors and
151
+ // the error is rethrown after the store is written, so a rerun only embeds what is missing.
152
+ const failures = [];
153
+ try {
154
+ for (let i = 0; i < missing.length; i += batchSize) {
155
+ const batch = missing.slice(i, i + batchSize);
156
+ const rows = await embedTexts(batch.map((d) => d.text), cfg, { timeoutMs: 180000 });
157
+ if (!outDims) outDims = rows[0].length;
158
+ if (rows[0].length < outDims) throw new Error('endpoint returned fewer dimensions than requested');
159
+ batch.forEach((d, k) => fresh.set(d.hash, rows[k]));
160
+ onProgress?.({ done: fresh.size, total: missing.length });
161
+ }
162
+ } catch (e) { failures.push(e); }
163
+ if (!outDims) outDims = old?.manifest.dims ?? 0;
164
+ if (!outDims && docs.length) throw new Error('no vectors produced');
165
+
166
+ const rec = 4 + outDims;
167
+ const order = docs.filter((d) => fresh.has(d.hash) || oldByHash.has(d.hash)).map((d) => [d.filePath, d.hash]);
168
+ const bin = Buffer.alloc(rec * order.length);
169
+ order.forEach(([, hash], i) => {
170
+ const off = i * rec;
171
+ const prior = oldByHash.get(hash);
172
+ if (prior) { prior.copy(bin, off); return; }
173
+ const v = normalizeTo(fresh.get(hash), outDims);
174
+ const maxAbs = Math.max(...v.map(Math.abs)) || 1;
175
+ bin.writeFloatLE(maxAbs / 127, off);
176
+ for (let k = 0; k < outDims; k += 1) bin.writeInt8(Math.max(-127, Math.min(127, Math.round(v[k] / maxAbs * 127))), off + 4 + k);
177
+ });
178
+ const manifest = { v: 1, model: cfg.model, dims: outDims, createdAt: new Date().toISOString(), order };
179
+ const tmp = `.tmp-${process.pid}`;
180
+ await fs.writeFile(path.join(indexDir, SEMANTIC_BIN + tmp), bin);
181
+ await fs.writeFile(path.join(indexDir, SEMANTIC_MANIFEST + tmp), JSON.stringify(manifest));
182
+ await fs.rename(path.join(indexDir, SEMANTIC_BIN + tmp), path.join(indexDir, SEMANTIC_BIN));
183
+ await fs.rename(path.join(indexDir, SEMANTIC_MANIFEST + tmp), path.join(indexDir, SEMANTIC_MANIFEST));
184
+ if (failures.length) throw new Error(`partial store saved (${order.length}/${docs.length} files); rerun to resume: ${failures[0]?.message ?? failures[0]}`);
185
+ return { files: docs.length, embedded: fresh.size, reused: docs.length - missing.length, dims: outDims, bytes: bin.length, model: cfg.model };
186
+ }
187
+
188
+ // Cache-key part: changes when the vector store is built, rebuilt or removed, so a result cached
189
+ // before `embed-index` ran never outlives the vectors. Never throws.
190
+ export async function semanticStoreSignature(rootDir) {
191
+ try {
192
+ const stat = await fs.stat(path.join(path.resolve(rootDir), '.cache', 'index', SEMANTIC_MANIFEST));
193
+ return `${stat.mtimeMs}:${stat.size}`;
194
+ } catch {
195
+ return 'none';
196
+ }
197
+ }
198
+
199
+ const STORE_CACHE = new Map();
200
+
201
+ // Query-side ranking. Returns [{ filePath, similarity }] (best first) or null on ANY problem
202
+ // (lane off, no vectors, model/dimension mismatch, timeout, HTTP error). Never throws.
203
+ export async function semanticCandidates({
204
+ rootDir, query, env = process.env, timeoutMs = QUERY_TIMEOUT_MS, limit = 100,
205
+ queryInstruction = QUERY_INSTRUCTION, accept = () => true, weight = () => 1,
206
+ } = {}) {
207
+ try {
208
+ const cfg = embedConfig(env);
209
+ if (!cfg) return null;
210
+ const indexDir = path.join(path.resolve(rootDir), '.cache', 'index');
211
+ const stat = await fs.stat(path.join(indexDir, SEMANTIC_MANIFEST));
212
+ const key = `${indexDir}:${stat.mtimeMs}:${stat.size}`;
213
+ let store = STORE_CACHE.get(key);
214
+ if (!store) { store = await loadStore(indexDir); if (!store) return null; STORE_CACHE.clear(); STORE_CACHE.set(key, store); }
215
+ const { manifest, bin, rec } = store;
216
+ if (manifest.model !== cfg.model) return null;
217
+ const [raw] = await embedTexts([`${queryInstruction}${query}`], cfg, { timeoutMs });
218
+ if (raw.length < manifest.dims) return null;
219
+ const q = normalizeTo(raw, manifest.dims);
220
+ const scored = [];
221
+ for (let i = 0; i < manifest.order.length; i += 1) {
222
+ const filePath = manifest.order[i][0];
223
+ if (!accept(filePath)) continue;
224
+ const off = i * rec;
225
+ let dot = 0;
226
+ for (let k = 0; k < manifest.dims; k += 1) dot += q[k] * bin.readInt8(off + 4 + k);
227
+ scored.push({ filePath, similarity: dot * bin.readFloatLE(off) * weight(filePath) });
228
+ }
229
+ scored.sort((a, b) => b.similarity - a.similarity || a.filePath.localeCompare(b.filePath));
230
+ return scored.slice(0, limit);
231
+ } catch {
232
+ return null;
233
+ }
234
+ }
235
+
236
+ // Plain-language gate: >= 4 words, no path separators or code punctuation, no quotes/colons,
237
+ // no trailing period (a finished statement is error-message shaped), no error-leading word, and
238
+ // no identifier-shaped token (camelCase, snake_case, kebab with digits, dotted name, file
239
+ // extension, token containing a digit, ALL-CAPS code such as ENOENT). Non-ASCII text
240
+ // (Vietnamese) and ordinary lowercase sentences pass; everything else keeps the lexical result.
241
+ const ERROR_LEADS = /^(?:error|errors|failed|failure|cannot|can't|unable|exception|invalid|unexpected|uncaught|warning|fatal|missing|undefined|null|typeerror|referenceerror)\b/i;
242
+ function identifierShaped(token) {
243
+ const t = token.replace(/^[^\w]+|[^\w]+$/g, '');
244
+ if (!t) return false;
245
+ return /[a-z][A-Z]/.test(t) || /_/.test(t) || /\d/.test(t) || /\w\.\w/.test(t) || /\w-\w*\d/.test(t)
246
+ || /^[A-Z]{3,}$/.test(t) || /^[A-Z][a-z]+[A-Z]/.test(t) || /\.(?:js|mjs|cjs|ts|tsx|jsx|json|md|py|go|rs|java|rb|yml|yaml|sh)\b/i.test(token);
247
+ }
248
+ export function looksLikeQuestion(query) {
249
+ const q = String(query ?? '').trim();
250
+ const words = q.split(/\s+/).filter(Boolean);
251
+ if (words.length < 4 || /[\\/]|[(){}\[\]=;<>@#$%^&*|~`'"]|::|=>|:/.test(q.replace(/[^\x00-\x7f]/g, ''))) return false;
252
+ if (/[^\x20-\x7e]/.test(q)) return true;
253
+ if (/\.\s*$/.test(q) || ERROR_LEADS.test(q) || words.some(identifierShaped)) return false;
254
+ return true;
255
+ }
256
+
257
+ // Combine: weighted RRF over the lexical ranking and the semantic list. Semantic-only files
258
+ // are injected with a floor score. Returns null (caller keeps lexical) when no candidates.
259
+ export function fuseSemantic({ ranked, candidates, rrfK = 5, weight = 1, depth = 100 }) {
260
+ if (!candidates?.length) return null;
261
+ const rrf = new Map();
262
+ ranked.slice(0, depth).forEach((e, i) => rrf.set(e.filePath, 1 / (rrfK + i + 1)));
263
+ candidates.slice(0, depth).forEach((c, i) => rrf.set(c.filePath, (rrf.get(c.filePath) ?? 0) + weight / (rrfK + i + 1)));
264
+ const byPath = new Map(ranked.map((e) => [e.filePath, e]));
265
+ const floor = ranked.length ? ranked[ranked.length - 1].score : 0.01;
266
+ return [...rrf.entries()]
267
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
268
+ .map(([filePath]) => byPath.get(filePath) ?? { filePath, score: Math.min(0.01, floor), reasons: ['semantic:vector'], tests: [] });
269
+ }
@@ -0,0 +1,110 @@
1
+ // lessonClass.js — is a mined pattern a real recurring lesson, or noise?
2
+ //
3
+ // Deterministic rules are the floor and the fallback. For the candidates the
4
+ // rules leave 'ambiguous', the decision plane (`unic-decision`, the owner's
5
+ // local decision model, reached through src/decision/client.js) is asked one
6
+ // yes/no question each. Its answer can only DOWNGRADE a candidate to 'noise';
7
+ // it never approves or promotes anything. Stage off, provider unreachable,
8
+ // malformed answer, timeout or throw -> the floor result stands.
9
+ //
10
+ // Call sites: `ukit memory learn` and the session-end self-improve pass only.
11
+ // Never from a deadline-armed hook.
12
+
13
+ import { resolveConfigStage } from '../core/runtimeConfig.js';
14
+ import { createDecisionClient } from '../decision/client.js';
15
+
16
+ export const LESSON_DECISION_KEY = 'learn.candidate-class.v1';
17
+ export const LESSON_TIMEOUT_MS = 2000;
18
+ const MAX_ASKED = 8;
19
+ const TEXT_CHARS = 200;
20
+
21
+ export function resolveLessonStage(config) {
22
+ return resolveConfigStage(config, 'decisionPlane.families.learn.stage');
23
+ }
24
+
25
+ // Failure patterns: runners that verify work are lessons; shell navigation is noise.
26
+ const RUNNER_RE = /^(yarn|npm|pnpm|npx|vitest|jest|pytest|cargo|go|make|tsc|eslint)\b/;
27
+ const TRIVIA_RE = /^(cd|ls|pwd|echo|cat|head|tail|grep|find|which|true|false)\b/;
28
+
29
+ export function classifyPattern(pattern) {
30
+ const sig = String(pattern?.signature ?? '').trim().toLowerCase();
31
+ if (!sig || TRIVIA_RE.test(sig)) return 'noise';
32
+ return RUNNER_RE.test(sig) ? 'lesson' : 'ambiguous';
33
+ }
34
+
35
+ // Re-edited files: product source is a lesson, data files are noise, the rest is unclear.
36
+ // Current layout: src/, bin/, scripts/ and the shipped template runtime. (The old
37
+ // templates/.claude/** layout is gone and is dropped by dropMissingFiles first.)
38
+ const SOURCE_RE = /^(src|lib|bin|scripts|template_project\/\.claude\/(hooks|ukit))\/.*\.(js|mjs|cjs|ts|tsx|sh|py)$/;
39
+ const DATA_RE = /\.(json|ya?ml|lock|txt|csv|svg|png)$/;
40
+
41
+ export function classifyReedit(reedit) {
42
+ const file = String(reedit?.file ?? '');
43
+ if (!file || DATA_RE.test(file)) return 'noise';
44
+ return SOURCE_RE.test(file) ? 'lesson' : 'ambiguous';
45
+ }
46
+
47
+ const oneLine = (text) => String(text ?? '').replace(/\s+/g, ' ').trim().slice(0, TEXT_CHARS);
48
+
49
+ function withTimeout(promise, ms) {
50
+ let timer;
51
+ const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve({ status: 'timeout', answers: [] }), ms); });
52
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
53
+ }
54
+
55
+ /**
56
+ * @param {Array<{id:string, text:string, cls:'lesson'|'noise'|'ambiguous'}>} items
57
+ * @returns {Promise<{items: Array, meta: {stage, asked, answered, changed, applied, status}}>}
58
+ * `answered` counts real valid provider responses, so callers can tell a run
59
+ * that used the decision plane from one that fell back.
60
+ */
61
+ export async function refineWithDecision(items, {
62
+ client, stage = 'off', timeoutMs = LESSON_TIMEOUT_MS,
63
+ } = {}) {
64
+ const meta = { stage, asked: 0, answered: 0, changed: 0, applied: false, status: 'skipped' };
65
+ const list = Array.isArray(items) ? items : [];
66
+ try {
67
+ if (stage === 'off' || !client) return { items: list, meta };
68
+ const ambiguous = list.filter((item) => item?.cls === 'ambiguous').slice(0, MAX_ASKED);
69
+ if (ambiguous.length === 0) return { items: list, meta };
70
+ meta.asked = ambiguous.length;
71
+ const result = await withTimeout(client.requestBatch({
72
+ statePacket: { task: 'learn-candidate-class', candidateCount: ambiguous.length },
73
+ questions: ambiguous.map((item, i) => ({
74
+ decisionKey: `${LESSON_DECISION_KEY}#c${i + 1}`,
75
+ schemaVersion: 1,
76
+ family: 'learn',
77
+ owner: 'lessonClass',
78
+ kind: 'noul',
79
+ instruction: `Is this a real recurring problem worth remembering for future edits? ${oneLine(item.text)}`,
80
+ candidates: [],
81
+ })),
82
+ deadlineMs: timeoutMs,
83
+ }), timeoutMs);
84
+ meta.status = result?.status ?? 'invalid';
85
+ const downgrade = new Set();
86
+ for (const answer of result?.answers ?? []) {
87
+ if (answer?.validationStatus !== 'valid' || !Number.isFinite(answer.value)) continue;
88
+ const idx = Number(String(answer.decisionKey).split('#c')[1]);
89
+ if (!Number.isInteger(idx) || idx < 1 || idx > ambiguous.length) continue;
90
+ meta.answered += 1;
91
+ if (answer.value < 0.5) downgrade.add(ambiguous[idx - 1]);
92
+ }
93
+ if (stage === 'shadow' || downgrade.size === 0) return { items: list, meta };
94
+ meta.changed = downgrade.size;
95
+ meta.applied = true;
96
+ return { items: list.map((item) => (downgrade.has(item) ? { ...item, cls: 'noise' } : item)), meta };
97
+ } catch {
98
+ meta.status = 'error';
99
+ return { items: list, meta };
100
+ }
101
+ }
102
+
103
+ export function createLessonClient({ config, projectRoot, homeDir, transport } = {}) {
104
+ if (resolveLessonStage(config) === 'off') return null;
105
+ try {
106
+ return createDecisionClient({ config, projectRoot, homeDir, transport });
107
+ } catch {
108
+ return null;
109
+ }
110
+ }
@@ -15,6 +15,7 @@
15
15
 
16
16
  import fs from 'node:fs/promises';
17
17
  import path from 'node:path';
18
+ import { classifyPattern, classifyReedit, refineWithDecision } from './lessonClass.js';
18
19
  import {
19
20
  listPendingPatternCandidates,
20
21
  proposePatternCandidate,
@@ -22,16 +23,49 @@ import {
22
23
 
23
24
  const ARTIFACT_REL = path.join('.ukit', 'storage', 'cache', 'failure-patterns.json');
24
25
  const MIN_SESSIONS = 2;
26
+ // Unapproved mined lessons expire; approval clears it (store.resolvePatternCandidate).
27
+ const LESSON_TTL_DAYS = 90;
25
28
 
26
29
  function normalizeText(text) {
27
30
  return String(text ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
28
31
  }
29
32
 
33
+ // Keys the rules (and, when staged on, the decision plane) call noise. Downgrade
34
+ // only: nothing here approves a candidate. `result.decision` reports how many
35
+ // real provider answers there were, so a fallback run is distinguishable.
36
+ async function noiseSet(items, { client, stage }, result) {
37
+ const refined = await refineWithDecision(items, { client, stage });
38
+ result.decision = refined.meta;
39
+ return new Set(refined.items.filter((i) => i.cls === 'noise').map((i) => i.key));
40
+ }
41
+
42
+ function errorClass(pattern) {
43
+ if (pattern.timedOut) return 'timed out';
44
+ const code = Array.isArray(pattern.exitCodes) ? pattern.exitCodes.find((c) => c !== 0) : undefined;
45
+ return Number.isInteger(code) ? `exit ${code}` : 'exit unproven';
46
+ }
47
+
48
+ // A lesson must name something concrete: the command it failed on and/or a file.
49
+ // Returns null when there is nothing to say (no candidate is created then).
30
50
  function candidateText(pattern) {
31
- const firstFile = Array.isArray(pattern.files) && pattern.files.length > 0
32
- ? pattern.files[0]
33
- : 'the affected files';
34
- return `Verification pattern "${pattern.signature}" failed ${pattern.count}× across ${pattern.sessions} sessions — check ${firstFile} before retrying.`;
51
+ const files = Array.isArray(pattern.files) ? pattern.files : [];
52
+ const commands = Array.isArray(pattern.commands) ? pattern.commands : [];
53
+ if (files.length === 0 && commands.length === 0) return null;
54
+ const seen = files.length > 0 ? `; seen on ${files.slice(0, 2).join(', ')}` : '';
55
+ return `Command "${pattern.signature}" failed ${pattern.count}× across ${pattern.sessions} sessions (${errorClass(pattern)})${seen}.`;
56
+ }
57
+
58
+ async function fileEvidence(projectRoot, files) {
59
+ const out = [];
60
+ for (const file of (Array.isArray(files) ? files : []).slice(0, 2)) {
61
+ try {
62
+ const abs = path.resolve(projectRoot, file);
63
+ if (abs.startsWith(path.resolve(projectRoot) + path.sep) && (await fs.stat(abs)).isFile()) {
64
+ out.push({ kind: 'file', locator: file });
65
+ }
66
+ } catch { /* missing file: no evidence entry */ }
67
+ }
68
+ return out;
35
69
  }
36
70
 
37
71
  async function loadPatterns(projectRoot) {
@@ -86,6 +120,63 @@ async function knownSignaturesAndTexts(projectRoot, projectId) {
86
120
  return { signatures, texts };
87
121
  }
88
122
 
123
+ /**
124
+ * Propose pending candidates from git re-edit mining (reeditMiner.js).
125
+ * Each lesson names the file and carries `commit:<sha>` evidence. Same
126
+ * contracts as proposeFromPatterns: never throws, dry-run by default, nothing
127
+ * auto-promotes.
128
+ *
129
+ * @param {{ reedits: Array<{file:string, fixes:number, sha:string}>, dryRun?: boolean }} options
130
+ */
131
+ export async function proposeFromReedits(projectRoot, projectId, {
132
+ reedits = [], dryRun = true, client = null, stage = 'off',
133
+ } = {}) {
134
+ const result = { generatedAt: new Date().toISOString(), proposed: [], dryRun };
135
+ try {
136
+ const known = await knownSignaturesAndTexts(projectRoot, projectId);
137
+ const textFor = (r) => `${r.file} needed ${r.fixes} fix commits in the last 90 days (latest ${String(r.sha).slice(0, 7)}); re-read its tests and mirrors before editing.`;
138
+ const valid = (Array.isArray(reedits) ? reedits : []).filter((r) => r?.file && r?.sha);
139
+ const noise = await noiseSet(
140
+ valid.map((r) => ({ key: r.file, text: textFor(r), cls: classifyReedit(r) })),
141
+ { client, stage }, result,
142
+ );
143
+ for (const r of valid) {
144
+ const signature = `reedit:${r.file}`;
145
+ const text = textFor(r);
146
+ if (noise.has(r.file)) {
147
+ result.proposed.push({ signature, text, status: 'skipped' });
148
+ continue;
149
+ }
150
+ if (known.signatures.has(signature)) {
151
+ result.proposed.push({ signature, text, status: 'duplicate' });
152
+ continue;
153
+ }
154
+ if (dryRun) {
155
+ result.proposed.push({ signature, text, status: 'proposed' });
156
+ continue;
157
+ }
158
+ try {
159
+ await proposePatternCandidate(projectRoot, projectId, {
160
+ text,
161
+ category: 'reedit',
162
+ signature,
163
+ detectedFrom: 'git-reedit',
164
+ ttlDays: LESSON_TTL_DAYS,
165
+ evidence: [{ kind: 'command', locator: `commit:${r.sha}` }, ...(await fileEvidence(projectRoot, [r.file]))],
166
+ });
167
+ known.signatures.add(signature);
168
+ result.proposed.push({ signature, text, status: 'proposed' });
169
+ } catch (err) {
170
+ result.proposed.push({ signature, text, status: 'skipped' });
171
+ result.error = err?.message ?? String(err);
172
+ }
173
+ }
174
+ } catch (err) {
175
+ result.error = err?.message ?? String(err);
176
+ }
177
+ return result;
178
+ }
179
+
89
180
  /**
90
181
  * Propose pending pattern-candidates from mined failure patterns.
91
182
  *
@@ -96,7 +187,9 @@ async function knownSignaturesAndTexts(projectRoot, projectId) {
96
187
  * proposed: Array<{signature: string, text: string,
97
188
  * status: 'proposed'|'duplicate'|'skipped'}>, dryRun: boolean, error?: string}>}
98
189
  */
99
- export async function proposeFromPatterns(projectRoot, projectId, { minCount = 3, dryRun = true } = {}) {
190
+ export async function proposeFromPatterns(projectRoot, projectId, {
191
+ minCount = 3, dryRun = true, client = null, stage = 'off',
192
+ } = {}) {
100
193
  const result = {
101
194
  generatedAt: new Date().toISOString(),
102
195
  patternsScanned: 0,
@@ -109,6 +202,12 @@ export async function proposeFromPatterns(projectRoot, projectId, { minCount = 3
109
202
  const patterns = await loadPatterns(projectRoot);
110
203
  result.patternsScanned = patterns.length;
111
204
  const known = await knownSignaturesAndTexts(projectRoot, projectId);
205
+ const noise = await noiseSet(
206
+ patterns.filter((p) => typeof p?.signature === 'string'
207
+ && (Number(p.count) || 0) >= minCount && (Number(p.sessions) || 0) >= MIN_SESSIONS)
208
+ .map((p) => ({ key: p.signature, text: candidateText({ ...p }) ?? '', cls: classifyPattern(p) })),
209
+ { client, stage }, result,
210
+ );
112
211
 
113
212
  for (const pattern of patterns) {
114
213
  const signature = typeof pattern?.signature === 'string' ? pattern.signature : null;
@@ -116,9 +215,12 @@ export async function proposeFromPatterns(projectRoot, projectId, { minCount = 3
116
215
  const sessions = Number(pattern?.sessions) || 0;
117
216
  if (!signature) continue;
118
217
 
119
- const text = candidateText({ signature, count, sessions, files: pattern.files });
218
+ const text = candidateText({
219
+ signature, count, sessions, files: pattern.files, commands: pattern.commands,
220
+ exitCodes: pattern.exitCodes, timedOut: pattern.timedOut,
221
+ });
120
222
  const eligible = count >= minCount && sessions >= MIN_SESSIONS;
121
- if (!eligible) {
223
+ if (!eligible || text === null || noise.has(signature)) {
122
224
  result.proposed.push({ signature, text, status: 'skipped' });
123
225
  continue;
124
226
  }
@@ -140,6 +242,8 @@ export async function proposeFromPatterns(projectRoot, projectId, { minCount = 3
140
242
  category: 'failure-pattern',
141
243
  signature,
142
244
  detectedFrom: 'memory-learn',
245
+ ttlDays: LESSON_TTL_DAYS,
246
+ evidence: await fileEvidence(projectRoot, pattern.files),
143
247
  });
144
248
  known.signatures.add(signature);
145
249
  known.texts.add(normalizeText(text));
@@ -0,0 +1,104 @@
1
+ // reeditMiner.js — git-history miner: files that keep needing fixes.
2
+ //
3
+ // A file touched by >= minFixes "fix" commits inside the window is a repeated
4
+ // correction. The latest fixing sha is the evidence the memory write gate
5
+ // accepts (`commit:<sha>`). Session-end / explicit command only — never call
6
+ // this from a hook path (it spawns git). Degrades to [] when git is absent,
7
+ // the repo is shallow, or git times out. Never throws.
8
+
9
+ import { execFile } from 'node:child_process';
10
+ import fs from 'node:fs/promises';
11
+ import path from 'node:path';
12
+
13
+ const FIX_RE = /^(fix|hotfix)(\(|:|!|\s)/i;
14
+ // Release churn (version files, changelog, docs), generated/live mirrors, manifests,
15
+ // and tests (a test co-changes with the source it covers; the source is the lesson).
16
+ const IGNORE_RE = new RegExp([
17
+ '(^|/)(package\\.json|yarn\\.lock|package-lock\\.json|CHANGELOG\\.md)$',
18
+ '^docs/', '\\.md$', '^\\.ukit/', '^\\.claude/', '^\\.omp/', '^\\.codex/', '^manifests/',
19
+ '^tests?/', '\\.(test|spec)\\.[cm]?[jt]sx?$',
20
+ ].join('|'));
21
+ const MAX_FILES_PER_COMMIT = 25;
22
+ const GIT_TIMEOUT_MS = 5000;
23
+
24
+ export function parseGitLog(raw) {
25
+ const commits = [];
26
+ let cur = null;
27
+ for (const line of String(raw ?? '').split('\n')) {
28
+ if (line.startsWith('C\u0001')) {
29
+ const [, sha, ts, ...subject] = line.split('\u0001');
30
+ cur = { sha, ts: Number(ts), subject: subject.join('\u0001'), files: [] };
31
+ commits.push(cur);
32
+ } else if (cur && line.trim()) {
33
+ cur.files.push(line.trim());
34
+ }
35
+ }
36
+ return commits;
37
+ }
38
+
39
+ /**
40
+ * @param {Array<{sha:string, ts:number, subject:string, files:string[]}>} commits newest first
41
+ * @returns {Array<{file:string, fixes:number, sha:string, shas:string[]}>}
42
+ */
43
+ export function mineReedits(commits, { minFixes = 3 } = {}) {
44
+ if (!Array.isArray(commits)) return [];
45
+ const byFile = new Map();
46
+ for (const c of commits) {
47
+ if (!c || !FIX_RE.test(String(c.subject ?? '')) || !Array.isArray(c.files)) continue;
48
+ if (c.files.length > MAX_FILES_PER_COMMIT) continue;
49
+ for (const file of c.files) {
50
+ if (IGNORE_RE.test(file)) continue;
51
+ const g = byFile.get(file) ?? { file, ts: -Infinity, sha: null, shas: [] };
52
+ g.shas.push(c.sha);
53
+ if (c.ts > g.ts) { g.ts = c.ts; g.sha = c.sha; }
54
+ byFile.set(file, g);
55
+ }
56
+ }
57
+ return [...byFile.values()]
58
+ .filter((g) => g.shas.length >= minFixes)
59
+ .map((g) => ({ file: g.file, fixes: g.shas.length, sha: g.sha, shas: g.shas.slice(0, 5) }))
60
+ .sort((a, b) => b.fixes - a.fixes || a.file.localeCompare(b.file));
61
+ }
62
+
63
+ function git(cwd, args) {
64
+ return new Promise((resolve) => {
65
+ execFile('git', args, { cwd, timeout: GIT_TIMEOUT_MS, maxBuffer: 4 * 1024 * 1024 }, (err, stdout) => {
66
+ resolve(err ? null : String(stdout));
67
+ });
68
+ });
69
+ }
70
+
71
+ export async function readGitCommits(projectRoot, { sinceDays = 90, limit = 400 } = {}) {
72
+ try {
73
+ const shallow = await git(projectRoot, ['rev-parse', '--is-shallow-repository']);
74
+ if (shallow === null || shallow.trim() !== 'false') return [];
75
+ const raw = await git(projectRoot, [
76
+ 'log', `--since=${sinceDays}.days`, '--no-merges', '--name-only', `-n${limit}`,
77
+ '--format=C%x01%H%x01%ct%x01%s',
78
+ ]);
79
+ return raw === null ? [] : parseGitLog(raw);
80
+ } catch {
81
+ return [];
82
+ }
83
+ }
84
+
85
+ /**
86
+ * Deterministic rule: a lesson about a file that is gone is useless, so drop
87
+ * re-edit candidates whose path is not a regular file in the working tree
88
+ * (existence only; renames are not followed). Async, bounded by candidate count.
89
+ */
90
+ export async function dropMissingFiles(projectRoot, reedits) {
91
+ const root = path.resolve(projectRoot);
92
+ const kept = [];
93
+ for (const r of Array.isArray(reedits) ? reedits : []) {
94
+ try {
95
+ const abs = path.resolve(root, String(r?.file ?? ''));
96
+ if (abs.startsWith(root + path.sep) && (await fs.stat(abs)).isFile()) kept.push(r);
97
+ } catch { /* missing or unreadable: dropped */ }
98
+ }
99
+ return kept;
100
+ }
101
+
102
+ export async function mineGitReedits(projectRoot, options = {}) {
103
+ return dropMissingFiles(projectRoot, mineReedits(await readGitCommits(projectRoot, options), options));
104
+ }