@ngockhoale/ukit 3.5.2 → 3.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +38 -90
  2. package/README.md +1 -0
  3. package/manifests/documentation.yaml +12 -1
  4. package/manifests/platform.full.yaml +12 -0
  5. package/package.json +34 -1
  6. package/src/cli/commands/doctor.js +55 -2
  7. package/src/cli/commands/indexTools.js +10 -4
  8. package/src/cli/commands/memory.js +107 -8
  9. package/src/core/memory/episodes.js +106 -7
  10. package/src/core/memory/memoryCheck.js +241 -0
  11. package/src/core/memory/memoryCheckRun.js +27 -0
  12. package/src/core/memory/memoryHit.js +2 -0
  13. package/src/core/memory/mutateMemory.js +46 -1
  14. package/src/core/memory/observations.js +192 -0
  15. package/src/core/memory/projectIdentity.js +9 -2
  16. package/src/core/memory/recallEngine.js +204 -9
  17. package/src/core/memory/records.js +23 -0
  18. package/src/core/memory/registryPrune.js +230 -0
  19. package/src/core/memory/retrieval.js +12 -3
  20. package/src/core/memory/writeGate.js +137 -0
  21. package/src/core/runtimeConfig.js +12 -2
  22. package/src/index/buildIndex.js +26 -0
  23. package/src/index/paths.js +3 -1
  24. package/src/index/queryIndex.js +372 -26
  25. package/src/index/resolveContext.js +1 -1
  26. package/template_project/.claude/hooks/edit-hint.mjs +64 -17
  27. package/template_project/.claude/hooks/session-episode-start.mjs +35 -0
  28. package/template_project/.claude/hooks/session-episode.sh +9 -4
  29. package/template_project/.claude/settings.json +2 -2
  30. package/template_project/.claude/ukit/index/edit-graph.mjs +141 -11
  31. package/template_project/.claude/ukit/index/lib/index-core.mjs +400 -27
  32. package/template_project/.claude/ukit/index/query-index.mjs +14 -3
  33. package/template_project/.claude/ukit/index/resolve-context.mjs +1 -1
  34. package/template_project/.claude/ukit/index/route-task.mjs +8 -2
  35. package/template_project/.claude/ukit/runtime/memory-check.mjs +241 -0
  36. package/template_project/.claude/ukit/runtime/memory-lanes.mjs +44 -22
  37. package/template_project/.claude/ukit/runtime/memory-recall.mjs +204 -9
  38. package/template_project/.claude/ukit/runtime/memory-validity.mjs +259 -0
  39. package/template_project/.claude/ukit/runtime/reinject-context.mjs +4 -2
  40. package/template_project/.claude/ukit/runtime/self-improve-trigger.mjs +212 -5
  41. package/template_project/.omp/hooks/pre/ukit-bridge.js +2 -1
  42. package/template_project/ukit/storage/config.json +4 -2
  43. package/src/core/agentRuntime/completionGate.js +0 -179
  44. package/src/core/agentRuntime/context.js +0 -571
  45. package/src/core/agentRuntime/qualityComparison.js +0 -112
  46. package/src/core/agentRuntime/resourcePolicy.js +0 -78
  47. package/src/core/agentRuntime/runtimeSupport.js +0 -255
  48. package/src/core/codeintel/manifest.js +0 -176
  49. package/src/core/experiments/deliberation.js +0 -327
  50. package/src/core/experiments/dynamicWorkflow.js +0 -507
  51. package/src/core/handoffDocValidator.js +0 -154
  52. package/src/core/memory/deltaOverlays.js +0 -493
  53. package/src/core/memory/learningCandidates.js +0 -347
  54. package/src/core/observability/emit/sessionBoot.js +0 -407
  55. package/src/core/observability/evaluation/experimentPlan.js +0 -95
  56. package/src/core/observability/evaluation/perturbation.js +0 -273
  57. package/src/core/observability/evaluation/replay.js +0 -149
  58. package/src/core/observability/evaluation/scorecard.js +0 -465
  59. package/src/core/observability/schema/compatibility.js +0 -180
  60. package/src/core/observability/support/renderer.js +0 -130
  61. package/src/core/reviewPanelAggregate.js +0 -204
  62. package/src/core/taskBudgetValidator.js +0 -251
  63. package/src/core/taskProgressGuard.js +0 -227
  64. package/src/decision/reviewVerdict.js +0 -339
  65. package/src/decision/runtimeFilter.js +0 -150
  66. package/src/decision/runtimeScheduler.js +0 -239
  67. package/src/diagnostics/classifyHang.js +0 -246
  68. package/src/diagnostics/ompStallSnapshot.js +0 -199
  69. package/src/index/contextPack.js +0 -285
  70. package/src/index/impactContext.js +0 -360
  71. package/src/index/reviewPolicy.js +0 -368
  72. package/src/index/sensitivePath.js +0 -21
  73. package/src/render/instructionRenderer.js +0 -290
  74. package/src/skill/auditSkill.js +0 -140
  75. package/template_project/instructions/overlays/repo.md +0 -9
@@ -22,6 +22,7 @@ import { runMigration } from '../../core/memory/migrate.js';
22
22
  import { createBackup, restoreBackup, diagnoseStore } from '../../core/memory/storeBackup.js';
23
23
  import { mutateMemory } from '../../core/memory/mutateMemory.js';
24
24
  import { redactWritePayload } from '../../core/memory/writeGuard.js';
25
+ import { checkProposalText } from '../../core/memory/writeGate.js';
25
26
  import { deletePromptCacheEntries } from '../../core/token/index.js';
26
27
  import { expand, getContextInjection, search } from '../../core/memory/retrieval.js';
27
28
  import { inspectRuntimeConfig } from '../../core/runtimeConfig.js';
@@ -29,8 +30,10 @@ import { buildRuntimePaths } from '../../core/runtimePaths.js';
29
30
  import { buildUserPaths } from '../../core/userPaths.js';
30
31
  import { pathExists } from '../../core/fileOps.js';
31
32
  import { detectProjectContext } from '../../context/detectProjectContext.js';
32
- import { episodeText, readRouteState, resolveLedger, writeEpisode } from '../../core/memory/episodes.js';
33
+ import { backfillEpisodes, episodeText, readRouteState, resolveLedger, writeEpisode } from '../../core/memory/episodes.js';
33
34
  import { archiveLegacyEpisodes, purgeArchivedEpisode, restoreArchivedEpisodes } from '../../core/memory/episodeArchive.js';
35
+ import { loadMarkdownDocs, normalizeMemorySources } from '../../core/memory/recallEngine.js';
36
+ import { captureAnchors } from '../../../template_project/.claude/ukit/runtime/memory-validity.mjs';
34
37
  import fs from 'node:fs/promises';
35
38
  import os from 'node:os';
36
39
  import path from 'node:path';
@@ -97,11 +100,25 @@ async function listAllPendingPatternCandidates(projectRoot, runtimePaths, { home
97
100
  }
98
101
  // FR-025: guard rejections surface labels only — never the matched value or
99
102
  // record body. Returns true when the result was a handled rejection.
103
+ const GATE_FIXES = {
104
+ 'too-large': 'shorten the text (one line); put detail in docs/memory/',
105
+ 'no-evidence': 'name an existing file, a commit sha or an episode id as evidence',
106
+ 'near-duplicate': 'update the existing record instead, or replace it with supersedes',
107
+ };
100
108
  function printGuardRejection(res) {
109
+ // Write-gate rejections (3.5.4): one line with the typed reason and what makes
110
+ // it pass, and a non-zero exit so an agent cannot mistake it for success.
111
+ if (res.code && GATE_FIXES[res.code]) {
112
+ console.log(`[UKit] rejected: ${res.reason} (to pass: ${GATE_FIXES[res.code]})`);
113
+ process.exitCode = 1;
114
+ return;
115
+ }
101
116
  const labels = Array.isArray(res.labels) && res.labels.length > 0
102
117
  ? res.labels.join(', ')
103
118
  : 'unknown';
104
119
  console.log(`[UKit] rejected: ${res.reason ?? 'secret-detected'} (${labels})`);
120
+ // Secret rejections are failures too: a non-zero exit on every CLI write path.
121
+ if ((res.reason ?? 'secret-detected') === 'secret-detected') process.exitCode = 1;
105
122
  }
106
123
 
107
124
  // Shared mutateMemory error mapping for CLI write ops: conflict/not-found/
@@ -167,6 +184,32 @@ async function runMemoryV2(projectRoot, homeDir, args) {
167
184
  return;
168
185
  }
169
186
 
187
+ if (op === 'confirm') {
188
+ // Re-anchor a note to its named files' current state (clears the
189
+ // "[source changed since noted — verify]" marker). Record id or markdown path:line.
190
+ const id = rest.join(' ').trim();
191
+ if (!id) {
192
+ throw new Error('Missing id. Usage: ukit memory v2 confirm <record-id | path:line>');
193
+ }
194
+ let doc = null;
195
+ const record = await getRecord(projectRoot, id, { homeDir });
196
+ if (record) {
197
+ doc = { id: `record:${record.id}`, type: record.type, provenance: record.provenance, text: record.text, evidence: record.evidence };
198
+ } else {
199
+ const { config } = await inspectRuntimeConfig(projectRoot, { homeDir });
200
+ const mdDocs = await loadMarkdownDocs(projectRoot, normalizeMemorySources(config)).catch(() => []);
201
+ doc = mdDocs.find((d) => d.provenance === id || d.id === id) ?? null;
202
+ }
203
+ if (!doc) {
204
+ throw new Error(`Memory item not found: ${id}`);
205
+ }
206
+ const n = await captureAnchors(projectRoot, [doc]);
207
+ console.log(n > 0
208
+ ? `[UKit] confirmed ${id}: anchored to current source state.`
209
+ : `[UKit] ${id} names no existing file; nothing to anchor.`);
210
+ return;
211
+ }
212
+
170
213
  if (op === 'promote') {
171
214
  const id = rest.join(' ').trim();
172
215
  if (!id) {
@@ -191,6 +234,7 @@ async function runMemoryV2(projectRoot, homeDir, args) {
191
234
  { kind: 'project', projectRoot, homeDir },
192
235
  {
193
236
  op: 'update',
237
+ actor: 'agent',
194
238
  payload: {
195
239
  id,
196
240
  patch: {
@@ -226,6 +270,7 @@ async function runMemoryV2(projectRoot, homeDir, args) {
226
270
  { kind: 'project', projectRoot, homeDir },
227
271
  {
228
272
  op: 'update',
273
+ actor: 'agent',
229
274
  payload: { id, patch: { status: 'stale' } },
230
275
  expectedRevision: record.revision,
231
276
  },
@@ -276,7 +321,7 @@ async function runMemoryV2(projectRoot, homeDir, args) {
276
321
  console.log(`[UKit] v2 store: ${stats.total} record(s) — ${JSON.stringify(stats.byType)}`);
277
322
  return;
278
323
  }
279
- throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|promote|stale|migrate.`);
324
+ throw new Error(`Unknown memory v2 op: ${op}. Expected list|show|confirm|promote|stale|migrate.`);
280
325
  }
281
326
 
282
327
  // ---- `ukit memory --user` — user-level store at ~/.ukit (FR-011) ----
@@ -287,7 +332,17 @@ async function runMemoryV2(projectRoot, homeDir, args) {
287
332
 
288
333
  const USER_MEMORY_OPS = new Set(['list', 'add', 'get', 'update', 'forget', 'purge', 'stats']);
289
334
 
290
- async function runUserMemory(homeDir, args) {
335
+ // The user store has no project config; honour the current project's memory.writeGate.
336
+ async function userWriteGate(projectRoot, homeDir) {
337
+ try {
338
+ const { config } = await inspectRuntimeConfig(projectRoot, { homeDir });
339
+ return config?.memory?.writeGate;
340
+ } catch {
341
+ return undefined;
342
+ }
343
+ }
344
+
345
+ async function runUserMemory(homeDir, args, projectRoot = process.cwd()) {
291
346
  const rest = (args[0] ?? '').toLowerCase() === 'v2' ? args.slice(1) : args;
292
347
  const op = (rest[0] ?? 'list').toLowerCase();
293
348
  const opArgs = rest.slice(1);
@@ -330,6 +385,8 @@ async function runUserMemory(homeDir, args) {
330
385
  { kind: 'user', homeDir },
331
386
  {
332
387
  op: 'add',
388
+ actor: 'agent',
389
+ writeGate: await userWriteGate(projectRoot, homeDir),
333
390
  payload: {
334
391
  type: typeFlag.value ?? 'derived_fact',
335
392
  scope: scopeFlag.value ?? 'user',
@@ -386,7 +443,7 @@ async function runUserMemory(homeDir, args) {
386
443
  }
387
444
  const res = await mutateMemory(
388
445
  { kind: 'user', homeDir },
389
- { op: 'update', payload: { id, patch } },
446
+ { op: 'update', actor: 'agent', writeGate: await userWriteGate(projectRoot, homeDir), payload: { id, patch } },
390
447
  );
391
448
  const handled = handleMutationResult(res, {
392
449
  notFoundMessage: `User memory record not found: ${id}`,
@@ -554,6 +611,14 @@ async function runMemoryEpisode(projectRoot, homeDir, args) {
554
611
  return;
555
612
  }
556
613
 
614
+ // SessionStart trigger: finished tasks of earlier ledgers (idle) and of the open session
615
+ // (spool only), bounded to the newest 25 ledgers. Same dedupe keys as SessionEnd.
616
+ if (args.includes('--start')) {
617
+ const s = await backfillEpisodes(projectRoot, { config, homeDir, limit: 25, spoolsOnly: true });
618
+ console.log(`[UKit] episode --start: recorded=${s.recorded} skipped=${s.skipped} failed=${s.failed}${s.reason ? ` reason=${s.reason}` : ''}`);
619
+ return;
620
+ }
621
+
557
622
  const { ledger, ledgerKey } = await resolveLedger(projectRoot, sessionFlag ?? process.env.UKIT_SESSION_ID);
558
623
  if (!ledger) {
559
624
  console.log('[UKit] episode: nothing to record');
@@ -682,6 +747,17 @@ export async function runMemory({ projectRoot, homeDir = os.homedir(), argv = []
682
747
  return;
683
748
  }
684
749
 
750
+ // 3.5.4: `check` only reads the markdown memory sources (cap, size, secrets,
751
+ // near-duplicates, supersession), so it runs without the runtime gate too.
752
+ // Exit 1 on violations; --json for machines.
753
+ if (subcommand === 'check') {
754
+ const { runMemoryCheck } = await import('../../core/memory/memoryCheckRun.js');
755
+ const res = await runMemoryCheck(projectRoot, rest);
756
+ console.log(res.output);
757
+ if (!res.ok) process.exitCode = 1;
758
+ return;
759
+ }
760
+
685
761
  // --user operates on ~/.ukit and does not require the project runtime.
686
762
  if (!isUser && !(await pathExists(runtimePaths.runtimeRoot))) {
687
763
  throw new Error('Shared UKit runtime not found. Run `ukit install` first.');
@@ -692,7 +768,7 @@ export async function runMemory({ projectRoot, homeDir = os.homedir(), argv = []
692
768
 
693
769
 
694
770
  if (isUser) {
695
- await runUserMemory(homeDir, argvRest);
771
+ await runUserMemory(homeDir, argvRest, projectRoot);
696
772
  return;
697
773
  }
698
774
 
@@ -736,6 +812,25 @@ export async function runMemory({ projectRoot, homeDir = os.homedir(), argv = []
736
812
  }
737
813
 
738
814
  const projectId = projectFlag.value ?? (await detectProjectContext(projectRoot, { homeDir })).project.name;
815
+ // 3.5.4 gate: size, secrets, near-duplicate of pending/approved candidates.
816
+ const { config: gateConfig } = await inspectRuntimeConfig(projectRoot, { homeDir });
817
+ const pending = await listPendingPatternCandidates(projectRoot, projectId, { homeDir });
818
+ const exported = await exportMemory(projectRoot, { homeDir });
819
+ const approved = (exported.projects.find((p) => p.id === projectId)?.conventions ?? [])
820
+ .map((t, i) => ({ id: `approved convention #${i + 1}`, text: t }));
821
+ const gate = checkProposalText({
822
+ text,
823
+ existing: [...pending.map((c) => ({ id: c.id, text: c.text })), ...approved],
824
+ config: gateConfig,
825
+ projectId,
826
+ });
827
+ if (!gate.ok) {
828
+ printGuardRejection(gate.code === 'secret-detected'
829
+ ? { reason: gate.reason, labels: gate.labels }
830
+ : gate);
831
+ process.exitCode = 1;
832
+ return;
833
+ }
739
834
  const candidate = await proposePatternCandidate(projectRoot, projectId, {
740
835
  text,
741
836
  category: categoryFlag.value,
@@ -936,6 +1031,8 @@ export async function runMemory({ projectRoot, homeDir = os.homedir(), argv = []
936
1031
  return;
937
1032
  }
938
1033
 
1034
+ if (subcommand === 'observe') { await (await import('../../core/memory/observations.js')).runObserve(projectRoot, rest); return; }
1035
+
939
1036
  if (subcommand === 'archive-episodes') {
940
1037
  if (rest.includes('--restore')) {
941
1038
  const r = await restoreArchivedEpisodes(projectRoot, { homeDir });
@@ -1007,7 +1104,7 @@ export async function runMemory({ projectRoot, homeDir = os.homedir(), argv = []
1007
1104
 
1008
1105
  export function printMemoryHelp() {
1009
1106
  console.log('UKit Memory Commands');
1010
- console.log('Usage: ukit memory <list|search|expand|recall|forget|purge|export|propose|approve|reject|learn|promote|episode|archive-episodes|hygiene|backup|restore|doctor> [args]');
1107
+ console.log('Usage: ukit memory <list|search|expand|recall|forget|purge|export|propose|approve|reject|learn|promote|episode|observe|archive-episodes|check|hygiene|backup|restore|doctor> [args]');
1011
1108
  console.log('');
1012
1109
  console.log('Subcommands:');
1013
1110
  console.log(' list List memory items in shared .ukit/storage/memory');
@@ -1023,12 +1120,14 @@ export function printMemoryHelp() {
1023
1120
  console.log(' reject <id> [--project <id>] Reject a pending pattern candidate');
1024
1121
  console.log(' learn [--apply] [--min <n>] [--project <id>] Propose pending candidates from mined failure patterns (dry-run by default)');
1025
1122
  console.log(' promote [--dry-run] Render approved rules/procedures into the ## ukit-learned block in docs/MEMORY.md');
1026
- console.log(' episode [--dry-run] [--session <id>] Write a session episode record from the exec-ledger');
1123
+ console.log(' episode [--dry-run] [--session <id>|--start] Write a session episode record from the exec-ledger');
1124
+ console.log(' observe [--min <n>] [--json] Propose observations from repeated content-bearing episodes (proposals only; nothing is written)');
1125
+ console.log(' check [--json] [--file <rel>] Check docs/MEMORY.md + memory.sources against the memory rules (core cap, size, secrets, near-duplicates, supersession); exit 1 on violations');
1027
1126
  console.log(' archive-episodes [--dry-run|--restore] Move content-free legacy episodes to episodes-archive.json (backup first; --restore re-adds them)');
1028
1127
  console.log(' hygiene [--project <id>] Run decision-conflict resolution + session archiving now');
1029
1128
  console.log(' backup [--out <dir>] [--user] Snapshot v2 store(s) + sha256 manifest into a backup dir');
1030
1129
  console.log(' restore <dir> [--dry-run] [--user] Restore store(s) from a backup dir (plan only with --dry-run)');
1031
1130
  console.log(' doctor [--user] Diagnose v2 store state + counts + quarantine (read-only)');
1032
- console.log(' v2 <list|show|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
1131
+ console.log(' v2 <list|show|confirm|promote|stale|migrate> Operate on the project v2 record store (migrate supports --dry-run plan output)');
1033
1132
  console.log(' --user [v2] <list|add|get|update|forget|purge|stats> Operate on the user-level store at ~/.ukit/storage/memory/v2/records.json (purge requires --confirm)');
1034
1133
  }
@@ -10,6 +10,7 @@
10
10
  // differ, and Codex has no stop event at all.
11
11
 
12
12
  import fs from 'node:fs/promises';
13
+ import { realpathSync } from 'node:fs';
13
14
  import path from 'node:path';
14
15
 
15
16
  import { loadRecords } from './storeV2.js';
@@ -60,6 +61,24 @@ const LINE_MAX = 160;
60
61
 
61
62
  const oneLine = (value, max) => String(value ?? '').replace(/\s+/g, ' ').trim().slice(0, max);
62
63
 
64
+ // Orchestrator envelopes that precede the real task: `[ukit-task <id>]`, a bare leading
65
+ // `TASK-xxx:` tag, "Complete assignment thoroughly:". Stripped (repeatedly) BEFORE the length
66
+ // cut so the stored summary carries the actual task. A prompt that is only boilerplate is kept.
67
+ const BOILERPLATE_PREFIX_RES = [
68
+ /^\[ukit-task[^\]]*\]\s*/i,
69
+ /^(?:TASK|C\d+)-[A-Za-z0-9-]*\s*[:\-–]\s+/,
70
+ /^complete assignment thoroughly\s*[:.\-–]?\s*/i,
71
+ ];
72
+ export function stripPromptBoilerplate(value) {
73
+ let text = String(value ?? '').replace(/\s+/g, ' ').trim();
74
+ for (let guard = 0; guard < 8; guard += 1) {
75
+ const next = BOILERPLATE_PREFIX_RES.reduce((acc, re) => acc.replace(re, ''), text);
76
+ if (next === text) break;
77
+ text = next;
78
+ }
79
+ return text || String(value ?? '').replace(/\s+/g, ' ').trim();
80
+ }
81
+
63
82
  export async function readRouteState(projectRoot, sessionKey) {
64
83
  if (!sessionKey) return null;
65
84
  return readJsonIfExists(path.join(
@@ -71,10 +90,50 @@ export async function readRouteState(projectRoot, sessionKey) {
71
90
  // project (keeps home-directory paths out of the store).
72
91
  function relativeWritten(projectRoot, file) {
73
92
  if (typeof file !== 'string' || !file.trim()) return null;
93
+ // Ledger receipts also carry pseudo targets (`agent:/Main`, `bash-workspace:<h>:<h>`):
94
+ // a scheme-like prefix (2+ chars, so a Windows drive letter still passes) is not a file.
95
+ if (/^[A-Za-z][A-Za-z0-9._-]+:/.test(file.trim())) return null;
74
96
  const abs = path.isAbsolute(file) ? file : path.join(projectRoot, file);
75
- const rel = path.relative(projectRoot, abs);
76
- if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null;
77
- return rel.split(path.sep).join('/');
97
+ // A receipt may hold the unresolved spelling of the root (macOS /var vs /private/var,
98
+ // symlinked checkouts) while the CLI runs from the resolved one: accept either.
99
+ const candidates = [abs];
100
+ const resolved = realpathLoose(abs);
101
+ if (resolved !== abs) candidates.push(resolved);
102
+ for (const root of rootSpellings(projectRoot)) {
103
+ for (const candidate of candidates) {
104
+ const rel = path.relative(root, candidate);
105
+ if (rel && !rel.startsWith('..') && !path.isAbsolute(rel)) return rel.split(path.sep).join('/');
106
+ }
107
+ }
108
+ return null;
109
+ }
110
+
111
+ // realpath of the nearest existing ancestor + the unresolved tail (the file may be gone).
112
+ function realpathLoose(file) {
113
+ let head = file;
114
+ let tail = '';
115
+ for (let i = 0; i < 64; i += 1) {
116
+ try { return path.join(realpathSync(head), tail); } catch { /* climb */ }
117
+ const parent = path.dirname(head);
118
+ if (parent === head) return file;
119
+ tail = tail ? path.join(path.basename(head), tail) : path.basename(head);
120
+ head = parent;
121
+ }
122
+ return file;
123
+ }
124
+
125
+ const rootSpellingCache = new Map();
126
+ function rootSpellings(projectRoot) {
127
+ let spellings = rootSpellingCache.get(projectRoot);
128
+ if (!spellings) {
129
+ spellings = [projectRoot];
130
+ try {
131
+ const real = realpathSync(projectRoot);
132
+ if (real !== projectRoot) spellings.push(real);
133
+ } catch { /* root vanished: the literal spelling is all there is */ }
134
+ rootSpellingCache.set(projectRoot, spellings);
135
+ }
136
+ return spellings;
78
137
  }
79
138
 
80
139
  // Strips a leading `cd <root>;|&&` and rewrites remaining occurrences of the
@@ -108,6 +167,9 @@ export function buildEpisode(ledger, { routeState = null, projectRoot = '' } = {
108
167
  const rel = relativeWritten(projectRoot, r.file);
109
168
  if (rel && !files.includes(rel)) files.push(rel);
110
169
  }
170
+ // No real project file (only pseudo targets or paths outside the project): nothing
171
+ // verifiable to remember. A content-free record would only crowd the store.
172
+ if (files.length === 0) return null;
111
173
  const verifications = receipts.filter((r) => r.kind === 'verification');
112
174
  const lastVerification = verifications.length > 0 ? verifications[verifications.length - 1] : null;
113
175
  const command = oneLine(normalizeCommand(lastVerification?.command, projectRoot), LINE_MAX) || null;
@@ -130,8 +192,10 @@ export function buildEpisode(ledger, { routeState = null, projectRoot = '' } = {
130
192
  && routeState.taskInstanceId === ledger.taskInstanceId;
131
193
  const legacyMatch = routeState && !ledger.taskInstanceId && !routeState.taskInstanceId;
132
194
  const task = oneLine(
133
- ledger.taskPrompt
134
- ?? (provenMatch || legacyMatch ? routeState?.routingContext?.lastExplicitUserPromptText : ''),
195
+ stripPromptBoilerplate(
196
+ ledger.taskPrompt
197
+ ?? (provenMatch || legacyMatch ? routeState?.routingContext?.lastExplicitUserPromptText : ''),
198
+ ),
135
199
  PROMPT_MAX,
136
200
  );
137
201
 
@@ -276,23 +340,54 @@ export async function backfillEpisodes(projectRoot, {
276
340
  idleMs = 10 * 60 * 1000,
277
341
  limit = 50,
278
342
  now = Date.now(),
343
+ // Start-trigger mode (3.5.4): a ledger that is not idle belongs to an open session.
344
+ // Its finished tasks (the spool) are recorded; its live task never is.
345
+ spoolsOnly = false,
279
346
  } = {}) {
280
347
  const summary = { scanned: 0, recorded: 0, skipped: 0, failed: 0 };
281
348
  try {
282
349
  if (config?.memoryV2?.enabled === false || config?.learning?.episodes?.autoWrite === false) {
283
350
  return { ...summary, reason: 'disabled' };
284
351
  }
352
+ // A deleted project must stay deleted: every store write below uses recursive mkdir, so
353
+ // refuse up front and re-check between ledgers (a background run outlives its project).
354
+ const storageDir = path.join(projectRoot, '.ukit', 'storage');
355
+ const storageExists = async () => { try { await fs.stat(storageDir); return true; } catch { return false; } };
356
+ if (!(await storageExists())) return { ...summary, reason: 'project gone' };
285
357
  const dir = path.join(projectRoot, LEDGER_DIR_REL);
286
- const names = await listLedgerFiles(dir, limit);
358
+ // Start mode scans deeper than `limit` (already-recorded ledgers are free) and bounds the
359
+ // WRITE work instead, so ledgers beyond the newest `limit` are reached on later runs.
360
+ const names = await listLedgerFiles(dir, spoolsOnly ? Math.max(limit, 400) : limit);
361
+ let writeBudget = limit;
287
362
  const existingKeys = recordedEpisodeKeys(await loadRecords(projectRoot, { homeDir }));
288
363
  const projectId = (await detectProjectContext(projectRoot, { homeDir })).project.name;
289
364
  for (const name of names) {
365
+ if (spoolsOnly && writeBudget <= 0) break;
366
+ if (summary.scanned > 0 && !(await storageExists())) { summary.reason = 'project gone'; break; }
290
367
  summary.scanned += 1;
291
368
  let stat;
292
369
  try { stat = await fs.stat(path.join(dir, name)); } catch { summary.skipped += 1; continue; }
293
- if (now - stat.mtimeMs < idleMs) { summary.skipped += 1; continue; }
370
+ const live = now - stat.mtimeMs < idleMs;
371
+ const before = summary.recorded;
372
+ try {
373
+ if (live && !spoolsOnly) { summary.skipped += 1; continue; }
294
374
  const ledger = await readJsonIfExists(path.join(dir, name));
295
375
  if (!ledger) { summary.skipped += 1; continue; }
376
+ if (live) {
377
+ let any = false;
378
+ for (const entry of await readSpool(projectRoot, name)) {
379
+ if (entry.taskInstanceId === ledger.taskInstanceId) continue;
380
+ any = true;
381
+ const res = await writeTaskEpisode(projectRoot, {
382
+ ledger: entry, ledgerKey: name, config, homeDir, projectId, existingKeys,
383
+ });
384
+ if (res.status === 'recorded') summary.recorded += 1;
385
+ else if (res.status === 'duplicate' || res.status === 'empty') summary.skipped += 1;
386
+ else summary.failed += 1;
387
+ }
388
+ if (!any) summary.skipped += 1;
389
+ continue;
390
+ }
296
391
  const res = await writeEpisode(projectRoot, {
297
392
  ledger, ledgerKey: name, config, homeDir, projectId, existingKeys,
298
393
  });
@@ -300,10 +395,14 @@ export async function backfillEpisodes(projectRoot, {
300
395
  if (res.status === 'recorded') summary.recorded += 1;
301
396
  else if (res.status === 'duplicate' || res.status === 'empty') summary.skipped += 1;
302
397
  else summary.failed += 1;
398
+ } finally {
399
+ writeBudget -= summary.recorded - before;
400
+ }
303
401
  }
304
402
  // Orphan spools: the ledger file was evicted by the dir sweep but its spool
305
403
  // survived. Same dedupe keys, same idle rule (on the spool's mtime).
306
404
  let dirNames = [];
405
+ if (summary.reason === 'project gone' || !(await storageExists())) return { ...summary, reason: 'project gone' };
307
406
  try { dirNames = await fs.readdir(dir); } catch { /* none */ }
308
407
  const present = new Set(dirNames);
309
408
  for (const spoolName of dirNames) {
@@ -0,0 +1,241 @@
1
+ // memoryCheck.js — memory rules + deterministic checker for memory source files
2
+ // (3.5.4). ONE file, node: builtins only, shipped byte-identical as
3
+ // template_project/.claude/ukit/runtime/memory-check.mjs (the PostToolUse advisory
4
+ // and Codex-less hosts use the mirror; `ukit memory check` and the record write
5
+ // gate use this copy). The recall engine and the secret scanner are injected, so
6
+ // there is exactly one tokenizer and one scanner. Async fs only; never throws.
7
+ //
8
+ // violation (exit 1) warning (exit 0)
9
+ // core-over-lines / core-over-bytes no-evidence (bullet names no existing
10
+ // line-too-long (core) / bullet-too-long file path or commit)
11
+ // secret-detected / near-duplicate
12
+ // unknown-supersedes / duplicate-id
13
+ //
14
+ // no-evidence is only a warning for markdown: pre-3.5.4 notes without a path are
15
+ // not rejected retroactively. Record writes (writeGate.js) enforce it.
16
+ import fsp from 'node:fs/promises';
17
+ import path from 'node:path';
18
+
19
+ // Caps. Measured on the live corpus before the split: longest hand-written
20
+ // bullet 1315 chars, longest record text 114, episodes sliced to 900.
21
+ export const RECORD_TEXT_MAX = 1200;
22
+ export const CORE_MAX_LINES = 150;
23
+ export const CORE_MAX_BYTES = 12000; // "12 KB" core cap, decimal so 12,000 is a hard number
24
+ export const CORE_LINE_MAX = 400; // one line per item in the core
25
+ export const DETAIL_BULLET_MAX = 1500; // verbatim detail bullets in docs/memory/*
26
+ export const CORE_FILE = 'docs/MEMORY.md';
27
+
28
+ // Lexical near-duplicate: Jaccard over the BM25 tokens. Measured (3.5.4) on
29
+ // docs/MEMORY.md (68 notes) + live records: the highest similarity between two
30
+ // distinct hand-written notes is 0.250 (markdown/markdown), 0.080 (markdown/
31
+ // record); 0.80 flags only near-restatements. Generated "Verification pattern
32
+ // ... failed N x" records reach 0.917 with each other, so generated writers are
33
+ // exempt in writeGate.js (GENERATED_CREATORS) and never rejected.
34
+ export const NEAR_DUP_JACCARD = 0.8;
35
+ export const NEAR_DUP_MIN_TOKENS = 6;
36
+
37
+ // Machine-generated writers: templated text, expected to resemble each other.
38
+ export const GENERATED_CREATORS = Object.freeze(new Set([
39
+ 'episode-hook', 'propose', 'learning-observe', 'learning-candidate',
40
+ 'delta-overlay', 'migration', 'system', 'observe',
41
+ ]));
42
+
43
+ export function jaccard(aSet, bSet) {
44
+ if (aSet.size === 0 || bSet.size === 0) return 0;
45
+ let inter = 0;
46
+ for (const t of aSet) if (bSet.has(t)) inter += 1;
47
+ return inter / (aSet.size + bSet.size - inter);
48
+ }
49
+
50
+ /**
51
+ * findNearDuplicate(text, candidates, tokenize, threshold) -> { id, similarity } | null
52
+ * candidates: [{ id, text }]. Notes under NEAR_DUP_MIN_TOKENS tokens are skipped
53
+ * (a short note shares most of its tokens by chance).
54
+ */
55
+ export function findNearDuplicate(text, candidates, tokenize, threshold = NEAR_DUP_JACCARD) {
56
+ const a = new Set(tokenize(String(text ?? '')));
57
+ if (a.size < NEAR_DUP_MIN_TOKENS) return null;
58
+ let best = null;
59
+ for (const c of candidates) {
60
+ const b = new Set(tokenize(String(c?.text ?? '')));
61
+ if (b.size < NEAR_DUP_MIN_TOKENS) continue;
62
+ const similarity = jaccard(a, b);
63
+ if (similarity >= threshold && (!best || similarity > best.similarity)) best = { id: c.id, similarity };
64
+ }
65
+ return best;
66
+ }
67
+
68
+ // Markdown supersession convention (honoured by recallEngine.parseMarkdownBullets):
69
+ // [id: slug] names a bullet (slug = [a-z0-9][a-z0-9._-]*)
70
+ // [supersedes: slug] the bullet replaces the bullet(s) so named
71
+ const MD_ID_RE = /\[id:\s*([a-z0-9][a-z0-9._-]*)\s*\]/gi;
72
+ // dir/file paths, or a bare file name with a known extension
73
+ const PATH_WORD_RE = /(?<![\w./@-])(?:[\w@.-]+\/)+[\w@.-]+(?::\d+)?|(?<![\w./@-])[\w-]+\.(?:m?js|cjs|json|md|sh|ya?ml|ts|py)\b/g;
74
+ const SHA_RE = /\b[0-9a-f]{7,40}\b/;
75
+
76
+ async function exists(root, rel) {
77
+ const clean = rel.replace(/[:#].*$/, '').replace(/^\.\//, '');
78
+ if (!clean || clean.startsWith('/') || clean.split('/').includes('..')) return false;
79
+ try {
80
+ await fsp.stat(path.join(root, clean));
81
+ return true;
82
+ } catch {
83
+ return false;
84
+ }
85
+ }
86
+
87
+ // A note is "about code" when it names a code token (camelCase, snake_case, call(),
88
+ // --flag, dotted config key). Owner preferences, process rules and release history
89
+ // name none, cannot point at a file, and never warn. The heading skip covers prose
90
+ // sections whose notes happen to mention a flag.
91
+ const CODE_TOKEN_RE = /\b[a-z]+[A-Z]\w*\b|\b[a-z0-9]+_[a-z0-9_]+\b|\b\w+\(\)|(?<![\w-])--[a-z][\w-]+|\b[a-z]+\.[a-z]+\.[a-z]\w*\b|\b[\w-]+\.(?:m?js|cjs|json|sh|ya?ml|ts)\b/;
92
+ const PROSE_HEADING_RE = /protocol|release history|preference|process|owner/i;
93
+ export function isCodeNote(doc) {
94
+ return !PROSE_HEADING_RE.test(doc?.heading ?? '') && CODE_TOKEN_RE.test(doc?.text ?? '');
95
+ }
96
+
97
+ /** A bullet is evidenced when it names an existing project path (backticked) or a commit. */
98
+ export async function bulletNamesExistingFile(root, text) {
99
+ // Parsed bullet text has its backticks stripped, so match path-shaped words.
100
+ let tried = 0;
101
+ for (const m of text.matchAll(PATH_WORD_RE)) {
102
+ if (tried >= 12) break;
103
+ tried += 1;
104
+ if (await exists(root, m[0])) return true;
105
+ }
106
+ return SHA_RE.test(text) && /\b(commit|sha)\b/i.test(text);
107
+ }
108
+
109
+ const DETAIL_RE = /Detail:\s+(\S+\.md)/g;
110
+
111
+ /**
112
+ * missingDetailPointers(root, text) -> [{ line, rel, why }]: core lines whose
113
+ * `Detail: <file>.md` pointer escapes the project root or names a missing file.
114
+ */
115
+ export async function missingDetailPointers(root, text) {
116
+ const out = [];
117
+ const base = path.resolve(root);
118
+ const lines = String(text).split('\n');
119
+ for (let i = 0; i < lines.length; i += 1) {
120
+ for (const m of lines[i].replace(/`/g, '').matchAll(DETAIL_RE)) {
121
+ const abs = path.resolve(base, m[1]);
122
+ if (abs !== base && !abs.startsWith(base + path.sep)) { out.push({ line: i + 1, rel: m[1], why: 'escapes the project root' }); continue; }
123
+ try { await fsp.stat(abs); } catch { out.push({ line: i + 1, rel: m[1], why: 'does not exist' }); }
124
+ }
125
+ }
126
+ return out;
127
+ }
128
+
129
+ /**
130
+ * checkMemorySources(projectRoot, { sources, engine, scan, files }) ->
131
+ * { ok, violations: [{file,line,code,message}], warnings, stats }
132
+ * sources: relative .md paths (normalizeMemorySources). engine: { tokenize,
133
+ * parseMarkdownBullets }. scan: (text) -> { labels: string[] } (optional).
134
+ * files: limit the report to these paths (every source is still read, so
135
+ * cross-file duplicates and supersession slugs resolve).
136
+ */
137
+ export async function checkMemorySources(projectRoot, { sources, engine, scan = null, files = null } = {}) {
138
+ const root = path.resolve(projectRoot);
139
+ const violations = [];
140
+ const warnings = [];
141
+ const stats = { files: 0, bullets: 0, coreLines: 0, coreBytes: 0 };
142
+ const all = [];
143
+ const raw = new Map();
144
+ for (const rel of sources ?? []) {
145
+ let text;
146
+ try { text = await fsp.readFile(path.join(root, rel), 'utf8'); } catch { continue; }
147
+ stats.files += 1;
148
+ raw.set(rel, text);
149
+ for (const doc of engine.parseMarkdownBullets(text, rel)) all.push({ ...doc, rel });
150
+ }
151
+ const focus = (rel) => !files || files.includes(rel);
152
+ const add = (list, rel, line, code, message) => { if (focus(rel)) list.push({ file: rel, line, code, message }); };
153
+
154
+ const core = raw.get(CORE_FILE);
155
+ if (core !== undefined) {
156
+ const lines = core.split('\n');
157
+ const lineCount = core.endsWith('\n') ? lines.length - 1 : lines.length;
158
+ const bytes = Buffer.byteLength(core, 'utf8');
159
+ stats.coreLines = lineCount;
160
+ stats.coreBytes = bytes;
161
+ if (lineCount > CORE_MAX_LINES) {
162
+ add(violations, CORE_FILE, lineCount, 'core-over-lines',
163
+ `core has ${lineCount} lines (max ${CORE_MAX_LINES}); merge items or move detail to docs/memory/`);
164
+ }
165
+ if (bytes > CORE_MAX_BYTES) {
166
+ add(violations, CORE_FILE, 1, 'core-over-bytes',
167
+ `core is ${bytes} bytes (max ${CORE_MAX_BYTES}); merge items or move detail to docs/memory/`);
168
+ }
169
+ for (const p of await missingDetailPointers(root, core)) {
170
+ add(violations, CORE_FILE, p.line, 'missing-detail', `Detail pointer ${p.rel} ${p.why}; point at an existing file under docs/memory/`);
171
+ }
172
+ lines.forEach((line, i) => {
173
+ if (/^[-*]\s/.test(line) && line.length > CORE_LINE_MAX) {
174
+ add(violations, CORE_FILE, i + 1, 'line-too-long',
175
+ `core item is ${line.length} chars (max ${CORE_LINE_MAX}); keep one line, move detail to docs/memory/`);
176
+ }
177
+ });
178
+ }
179
+ for (const [rel, text] of raw) {
180
+ if (rel === CORE_FILE) continue;
181
+ text.split('\n').forEach((line, i) => {
182
+ if (/^[-*]\s/.test(line) && line.length > DETAIL_BULLET_MAX) {
183
+ add(violations, rel, i + 1, 'bullet-too-long', `bullet is ${line.length} chars (max ${DETAIL_BULLET_MAX}); split it`);
184
+ }
185
+ });
186
+ }
187
+
188
+ stats.bullets = all.length;
189
+ const slugs = new Map();
190
+ for (const [rel, text] of raw) {
191
+ text.split('\n').forEach((line, i) => {
192
+ for (const m of line.matchAll(MD_ID_RE)) {
193
+ const slug = m[1].toLowerCase();
194
+ if (slugs.has(slug)) add(violations, rel, i + 1, 'duplicate-id', `id "${slug}" already used at ${slugs.get(slug)}`);
195
+ else slugs.set(slug, `${rel}:${i + 1}`);
196
+ }
197
+ });
198
+ }
199
+ for (const doc of all) {
200
+ for (const slug of doc.supersedesSlugs ?? []) {
201
+ if (!slugs.has(slug)) add(violations, doc.rel, doc.line, 'unknown-supersedes', `supersedes "${slug}" but no bullet has [id: ${slug}]`);
202
+ }
203
+ if (scan && focus(doc.rel)) {
204
+ let labels = [];
205
+ try { labels = scan(doc.text)?.labels ?? []; } catch { labels = []; }
206
+ if (labels.length > 0) add(violations, doc.rel, doc.line, 'secret-detected', `secret-like value (${labels.join(', ')}); remove it`);
207
+ }
208
+ if (focus(doc.rel) && isCodeNote(doc) && !(await bulletNamesExistingFile(root, doc.text))) {
209
+ add(warnings, doc.rel, doc.line, 'no-evidence', 'names no existing file path or commit; add the path this note is about');
210
+ }
211
+ }
212
+ // A superseded bullet may legitimately resemble its replacement.
213
+ const replaced = new Set();
214
+ for (const doc of all) for (const s of doc.supersedesSlugs ?? []) { const t = slugs.get(s); if (t) replaced.add(t); }
215
+ const live = all.filter((d) => !replaced.has(`${d.rel}:${d.line}`));
216
+ const seen = new Set();
217
+ const tokenCache = new Map();
218
+ const memoTokenize = (t) => {
219
+ let v = tokenCache.get(t);
220
+ if (!v) { v = engine.tokenize(t); tokenCache.set(t, v); }
221
+ return v;
222
+ };
223
+ for (let i = 0; i < live.length; i += 1) {
224
+ if (!focus(live[i].rel)) continue;
225
+ const dup = findNearDuplicate(live[i].text, live.slice(0, i).map((d) => ({ id: `${d.rel}:${d.line}`, text: d.text })), memoTokenize);
226
+ if (dup && !seen.has(dup.id)) {
227
+ seen.add(dup.id);
228
+ add(violations, live[i].rel, live[i].line, 'near-duplicate',
229
+ `${Math.round(dup.similarity * 100)}% lexical overlap with ${dup.id}; merge them or tag one [supersedes: <slug>]`);
230
+ }
231
+ }
232
+ return { ok: violations.length === 0, violations, warnings, stats };
233
+ }
234
+
235
+ /** One advisory line for the first violation, '' when clean. */
236
+ export function formatCheckLine(res) {
237
+ const v = res?.violations?.[0];
238
+ if (!v) return '';
239
+ const more = res.violations.length > 1 ? ` (+${res.violations.length - 1} more)` : '';
240
+ return `[UKit memory] ${v.file}:${v.line} ${v.code}: ${v.message}${more}. Run \`ukit memory check\`.`;
241
+ }