@ngockhoale/ukit 3.0.3 → 3.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/bin/ukit +12 -4
  3. package/manifests/engineConformance.yaml +29 -0
  4. package/manifests/platform.full.yaml +13 -0
  5. package/package.json +2 -1
  6. package/scripts/audit/decision-coverage.mjs +295 -0
  7. package/scripts/bench/outline-savings.mjs +19 -4
  8. package/scripts/bench/parallel-agents.mjs +15 -4
  9. package/scripts/bench/runGold.mjs +22 -4
  10. package/scripts/bench/v3-ceremony.mjs +7 -1
  11. package/scripts/bug/triage.mjs +56 -17
  12. package/scripts/index/build-index.mjs +94 -28
  13. package/scripts/index/query-index.mjs +48 -14
  14. package/scripts/index/refresh-index.mjs +142 -62
  15. package/scripts/perf/audit-perf.mjs +8 -2
  16. package/scripts/skill/audit-skill.mjs +54 -25
  17. package/src/bug/triageBug.js +9 -6
  18. package/src/cli/adapters.js +6 -0
  19. package/src/cli/commands/code.js +7 -1
  20. package/src/cli/commands/indexArgs.js +4 -2
  21. package/src/cli/commands/indexTools.js +8 -1
  22. package/src/cli/commands/install.js +13 -0
  23. package/src/cli/commands/memory.js +13 -9
  24. package/src/cli/commands/status.js +17 -1
  25. package/src/cli/commands/update.js +7 -0
  26. package/src/context/detectProjectContext.js +3 -1
  27. package/src/core/codeintel/analogy.js +1 -1
  28. package/src/core/codeintel/diagnostics.js +9 -6
  29. package/src/core/codeintel/graph.js +14 -8
  30. package/src/core/codeintel/impact.js +0 -1
  31. package/src/core/codeintel/invalidation.js +5 -3
  32. package/src/core/codeintel/packet.js +11 -0
  33. package/src/core/codeintel/router.js +17 -3
  34. package/src/core/codeintel/semanticProvider.js +1 -1
  35. package/src/core/codeintel/summaries.js +11 -7
  36. package/src/core/compact/contextBudget.js +26 -12
  37. package/src/core/compact/index.js +15 -8
  38. package/src/core/docContracts.js +10 -2
  39. package/src/core/experiments/deliberation.js +321 -0
  40. package/src/core/experiments/dynamicWorkflow.js +492 -0
  41. package/src/core/fileOps.js +8 -1
  42. package/src/core/gatewayProbe.js +29 -1
  43. package/src/core/gatewayResilienceEnv.js +44 -3
  44. package/src/core/handoffDocValidator.js +3 -1
  45. package/src/core/hookChainDoctor.js +16 -1
  46. package/src/core/memory/deltaOverlays.js +448 -0
  47. package/src/core/memory/learningCandidates.js +302 -0
  48. package/src/core/memory/migrate.js +59 -32
  49. package/src/core/memory/recordStore.js +24 -1
  50. package/src/core/memory/store.js +44 -8
  51. package/src/core/memory/storeV2.js +48 -33
  52. package/src/core/memory/userMemory.js +26 -10
  53. package/src/core/output/index.js +15 -10
  54. package/src/core/permissionPolicy.js +8 -0
  55. package/src/core/runtimeConfig.js +224 -4
  56. package/src/core/sensitiveValueScanner.js +10 -2
  57. package/src/core/taskBudgetValidator.js +12 -17
  58. package/src/core/taskProgressGuard.js +59 -9
  59. package/src/core/unattendedDoctor.js +5 -2
  60. package/src/core/uninstall.js +37 -8
  61. package/src/decision/client.js +371 -0
  62. package/src/decision/lease.js +198 -0
  63. package/src/decision/preflight.js +492 -0
  64. package/src/decision/protocol.js +308 -0
  65. package/src/decision/registry.js +384 -0
  66. package/src/decision/shadow.js +281 -0
  67. package/src/decision/statePacket.js +165 -0
  68. package/src/diagnostics/failurePatterns.js +2 -1
  69. package/src/diagnostics/ledgerFiles.js +3 -1
  70. package/src/diagnostics/routeOutcomes.js +1 -29
  71. package/src/index/buildIndex.js +23 -11
  72. package/src/index/impactContext.js +21 -2
  73. package/src/index/importResolution.js +13 -7
  74. package/src/index/queryIndex.js +11 -5
  75. package/src/index/resolveContext.js +22 -9
  76. package/src/index/taskRouting.js +63 -2
  77. package/src/index/verificationPlan.js +12 -1
  78. package/src/learning/patternProposals.js +6 -0
  79. package/src/render/renderTemplate.js +1 -1
  80. package/src/skill/auditSkill.js +3 -1
  81. package/src/stack/detectStack.js +3 -1
  82. package/template_project/.claude/agents/handoff-planner.md +2 -5
  83. package/template_project/.claude/hooks/auto-allow-bash.sh +5 -0
  84. package/template_project/.claude/hooks/block-dangerous.mjs +11 -4
  85. package/template_project/.claude/hooks/context-hardcap-gate.sh +10 -1
  86. package/template_project/.claude/hooks/handoff-model-guard.sh +14 -4
  87. package/template_project/.claude/hooks/handoff-resume.sh +10 -1
  88. package/template_project/.claude/hooks/protect-files.sh +0 -1
  89. package/template_project/.claude/hooks/record-execution.mjs +13 -1
  90. package/template_project/.claude/hooks/sensitive-data-guard.mjs +80 -5
  91. package/template_project/.claude/hooks/session-episode.sh +9 -2
  92. package/template_project/.claude/skills/pdf-processing-pro/SKILL.md +1 -1
  93. package/template_project/.claude/ukit/index/handoff-doc-validator.mjs +3 -1
  94. package/template_project/.claude/ukit/index/lib/index-core.mjs +129 -42
  95. package/template_project/.claude/ukit/index/route-task.mjs +444 -0
  96. package/template_project/.claude/ukit/index/task-budget-validator.mjs +12 -16
  97. package/template_project/.claude/ukit/index/unic-decision.mjs +786 -0
  98. package/template_project/.claude/ukit/index/verify-context.mjs +11 -0
  99. package/template_project/.claude/ukit/runtime/execution-ledger.mjs +116 -9
  100. package/template_project/.claude/ukit/runtime/project-important.mjs +9 -7
  101. package/template_project/.claude/ukit/runtime/reinject-context.mjs +48 -0
  102. package/template_project/.claude/ukit/runtime/resumable-run.mjs +596 -0
  103. package/template_project/.claude/ukit/runtime/sensitive-value-scanner.mjs +4 -7
  104. package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +1 -1
  105. package/template_project/docs/AI_HANDOFF/PLAN.md +7 -7
  106. package/template_project/docs/AI_HANDOFF/RULES.md +1 -1
  107. package/template_project/ukit/storage/config.json +34 -1
  108. package/template_project/.claude/ukit/skill-router-state.json +0 -1
  109. package/template_project/.ukit/storage/cache/hook-latency/unknown.jsonl +0 -4
@@ -0,0 +1,302 @@
1
+ // learningCandidates.js — M04.2 learning candidates (SPEC §5 FR-016, FR-018; C11).
2
+ //
3
+ // Normalizes repeated corrections/suppressions/escalations/verification-gaps
4
+ // into observation records, then promotes them to LearningCandidate records
5
+ // (C11) only after configured recurrence AND cross-session evidence.
6
+ //
7
+ // Contracts:
8
+ // * Gated by `learning.candidates.stage` — 'off' (default) means zero
9
+ // writes: observeLearningSignal is a no-op, maybeCreateCandidate returns
10
+ // null. At 'shadow'/'canary'/'default' everything is still REPORT-ONLY:
11
+ // candidates are created with meta.status 'proposed' and promotion stays
12
+ // manual via the existing pending-candidate flow in store.js.
13
+ // * Observations are memoryV2 records (provenance 'learning-observation');
14
+ // candidates are memoryV2 records (provenance 'learning-candidate').
15
+ // No second store.
16
+ // * Scope classification is conservative: any session-scoped signal pins
17
+ // the candidate to 'session' — session signals never leak to
18
+ // project/user.
19
+ // * Conflict detection prefers conditional refinement; project constraints
20
+ // outrank user preferences (C11).
21
+ // * NEVER THROWS on malformed signals — they are skipped, not fatal.
22
+
23
+ import crypto from 'node:crypto';
24
+ import { loadRuntimeConfig, resolveConfigStage } from '../runtimeConfig.js';
25
+ import { addRecord, queryRecords } from './storeV2.js';
26
+
27
+ export const OBSERVATION_PROVENANCE = 'learning-observation';
28
+ export const CANDIDATE_PROVENANCE = 'learning-candidate';
29
+
30
+ const SIGNAL_KINDS = new Set([
31
+ 'correction',
32
+ 'suppression',
33
+ 'escalation',
34
+ 'verification-gap',
35
+ ]);
36
+
37
+ // C11 proposalType per signal kind.
38
+ const PROPOSAL_TYPE_BY_KIND = {
39
+ correction: 'preference',
40
+ suppression: 'constraint',
41
+ escalation: 'knowledge',
42
+ 'verification-gap': 'overlay',
43
+ };
44
+
45
+ // C11 candidate scopes → memoryV2 record scopes.
46
+ const CANDIDATE_TO_RECORD_SCOPE = {
47
+ session: 'session',
48
+ project: 'repo',
49
+ user: 'user',
50
+ };
51
+
52
+ const DEFAULT_MIN_OCCURRENCES = 3;
53
+ const DEFAULT_MIN_SESSIONS = 2;
54
+ const MAX_EVIDENCE_REFS = 24;
55
+ // Records live in the single records.json document — bound every field that
56
+ // carries caller-controlled text so a signal can never grow it unboundedly.
57
+ const MAX_SIGNAL_TEXT = 512;
58
+ const MAX_META_FIELD = 512;
59
+
60
+ function boundText(value, max = MAX_META_FIELD) {
61
+ if (value == null) return null;
62
+ const text = typeof value === 'string' ? value : JSON.stringify(value);
63
+ return typeof text === 'string' ? text.slice(0, max) : null;
64
+ }
65
+
66
+ // Like boundText but preserves non-string scalars (numbers/booleans) so a
67
+ // proposedDelta value keeps its type for downstream overlay conversion.
68
+ function boundField(value, max = MAX_META_FIELD) {
69
+ if (value == null) return null;
70
+ if (typeof value !== 'string') return value;
71
+ return value.slice(0, max);
72
+ }
73
+
74
+ // Bounds a proposedDelta in place-shape: {field, value} stay typed fields —
75
+ // stringifying the whole object would break detectCandidateConflicts and
76
+ // overlayFromCandidate, which read delta.field/delta.value.
77
+ function boundDelta(delta) {
78
+ if (delta == null || typeof delta !== 'object') return boundText(delta);
79
+ return {
80
+ ...delta,
81
+ field: boundField(delta.field),
82
+ value: boundField(delta.value),
83
+ };
84
+ }
85
+
86
+ function normalizeText(text) {
87
+ return String(text ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
88
+ }
89
+
90
+ function normalizeValue(value) {
91
+ if (value == null) return '';
92
+ return normalizeText(typeof value === 'string' ? value : JSON.stringify(value));
93
+ }
94
+
95
+
96
+ function signalScopeHint(signal) {
97
+ const hint = signal?.scopeHint ?? signal?.scope;
98
+ return hint === 'project' || hint === 'user' ? hint : 'session';
99
+ }
100
+
101
+ async function candidatesConfig(projectRoot) {
102
+ const config = await loadRuntimeConfig(projectRoot);
103
+ const stage = resolveConfigStage(config, 'learning.candidates.stage');
104
+ const raw = config?.learning?.candidates ?? {};
105
+ return {
106
+ stage,
107
+ minOccurrences: Number.isFinite(raw.minOccurrences) && raw.minOccurrences > 0
108
+ ? raw.minOccurrences
109
+ : DEFAULT_MIN_OCCURRENCES,
110
+ minSessions: Number.isFinite(raw.minSessions) && raw.minSessions > 0
111
+ ? raw.minSessions
112
+ : DEFAULT_MIN_SESSIONS,
113
+ };
114
+ }
115
+
116
+ /**
117
+ * classifyCandidateScope(signal) → 'session' | 'project' | 'user'
118
+ * Conservative: unknown/absent hints resolve to 'session'.
119
+ */
120
+ export function classifyCandidateScope(signal) {
121
+ return signalScopeHint(signal);
122
+ }
123
+
124
+ /**
125
+ * detectCandidateConflicts(candidate, existing) → conflicts[]
126
+ * A conflict is a contradiction on the same target: the candidate proposes a
127
+ * different value for a field an existing project_rule/constraint already
128
+ * pins. Project (repo-scoped) constraints outrank user-scoped candidates —
129
+ * the conflict is flagged `outranked` and resolved by conditional refinement
130
+ * rather than silent override.
131
+ */
132
+ export function detectCandidateConflicts(candidate, existing = []) {
133
+ const target = normalizeText(candidate?.target ?? candidate?.proposedDelta?.field);
134
+ if (!target) return [];
135
+ const proposedValue = normalizeValue(
136
+ candidate?.proposedDelta?.value ?? candidate?.proposedDelta,
137
+ );
138
+
139
+ const conflicts = [];
140
+ for (const record of existing ?? []) {
141
+ if (!record || record.status === 'archived') continue;
142
+ if (record.provenance === CANDIDATE_PROVENANCE
143
+ && record.meta?.status !== 'approved') continue;
144
+ const recordTarget = normalizeText(record.meta?.target ?? record.meta?.constraint);
145
+ if (!recordTarget || recordTarget !== target) continue;
146
+ const recordValue = normalizeValue(record.meta?.value);
147
+ if (!recordValue || recordValue === proposedValue) continue;
148
+ const isProjectConstraint = record.type === 'project_rule' || record.scope === 'repo';
149
+ conflicts.push({
150
+ type: 'contradiction',
151
+ recordId: record.id,
152
+ target,
153
+ existingValue: record.meta?.value ?? null,
154
+ proposedValue: candidate?.proposedDelta?.value ?? null,
155
+ outranked: isProjectConstraint && candidate?.scope === 'user',
156
+ resolution: 'conditional-refinement',
157
+ });
158
+ }
159
+ return conflicts;
160
+ }
161
+
162
+ /**
163
+ * observeLearningSignal(projectRoot, signal) → observation record | null
164
+ * signal: { kind, text, sessionId?, projectId?, scopeHint?, target?,
165
+ * condition?, proposedDelta?, evidenceRef? }
166
+ * Returns null when the feature stage is 'off' or the signal is malformed.
167
+ */
168
+ export async function observeLearningSignal(projectRoot, signal = {}) {
169
+ const { stage } = await candidatesConfig(projectRoot);
170
+ if (stage === 'off') return null;
171
+ if (!SIGNAL_KINDS.has(signal?.kind) || !normalizeText(signal?.text)) return null;
172
+
173
+ // Every caller-controlled string is bounded — records.json is a single
174
+ // document, so an unbounded signal would grow it without limit.
175
+ const text = boundText(signal.text, MAX_SIGNAL_TEXT);
176
+ const target = boundText(signal.target);
177
+ const signature = `${signal.kind}:${normalizeText(target ?? text)}`.slice(0, MAX_META_FIELD);
178
+
179
+ return addRecord(projectRoot, {
180
+ type: 'derived_fact',
181
+ scope: CANDIDATE_TO_RECORD_SCOPE[signalScopeHint(signal)],
182
+ text: `learning-signal ${signal.kind}: ${String(text).trim()}`,
183
+ provenance: OBSERVATION_PROVENANCE,
184
+ confidence: 0.3,
185
+ createdBy: 'learning-observe',
186
+ projectId: boundText(signal.projectId),
187
+ meta: {
188
+ kind: signal.kind,
189
+ signature,
190
+ sessionId: boundText(signal.sessionId),
191
+ scopeHint: signalScopeHint(signal),
192
+ target,
193
+ condition: boundText(signal.condition),
194
+ proposedDelta: boundDelta(signal.proposedDelta),
195
+ evidenceRef: boundText(signal.evidenceRef),
196
+ observedAt: Date.now(),
197
+ },
198
+ });
199
+ }
200
+
201
+ function candidateText(kind, signalText, occurrences, sessions) {
202
+ return `Learning candidate (${kind}): "${String(signalText).trim()}" — observed ${occurrences}× across ${sessions} sessions.`;
203
+ }
204
+
205
+ /**
206
+ * maybeCreateCandidate(projectRoot) → candidate record | null
207
+ * Scans observation records, groups by signature + projectId, and promotes
208
+ * the first eligible group (occurrences >= minOccurrences AND distinct
209
+ * sessions >= minSessions) into a proposed LearningCandidate. Idempotent —
210
+ * an existing proposed/approved candidate for the signature is returned
211
+ * instead of duplicated.
212
+ */
213
+ export async function maybeCreateCandidate(projectRoot) {
214
+ const { stage, minOccurrences, minSessions } = await candidatesConfig(projectRoot);
215
+ if (stage === 'off') return null;
216
+
217
+ const records = await queryRecords(projectRoot, {});
218
+ const observations = records
219
+ .filter((r) => r.provenance === OBSERVATION_PROVENANCE && r.status !== 'archived')
220
+ .sort((a, b) => (a.meta?.observedAt ?? a.created_at ?? 0) - (b.meta?.observedAt ?? b.created_at ?? 0));
221
+ const existingCandidates = records.filter((r) => r.provenance === CANDIDATE_PROVENANCE);
222
+
223
+ const groups = new Map();
224
+ for (const obs of observations) {
225
+ const key = `${obs.project_id ?? ''}|${obs.meta?.signature ?? ''}`;
226
+ if (!groups.has(key)) groups.set(key, []);
227
+ groups.get(key).push(obs);
228
+ }
229
+
230
+ let firstExisting = null;
231
+ for (const group of groups.values()) {
232
+ const signature = group[0]?.meta?.signature;
233
+ const projectId = group[0]?.project_id ?? null;
234
+
235
+ const existing = existingCandidates.find(
236
+ (r) => r.meta?.signature === signature
237
+ && (r.project_id ?? null) === projectId
238
+ && (r.meta?.status === 'proposed' || r.meta?.status === 'approved'),
239
+ );
240
+ // Remember the first existing candidate but KEEP SCANNING — returning it
241
+ // here starved every later eligible group: group A's candidate blocked
242
+ // group B's from ever being created.
243
+ if (existing) {
244
+ firstExisting ??= existing;
245
+ continue;
246
+ }
247
+
248
+ const sessions = new Set(group.map((o) => o.meta?.sessionId ?? null));
249
+ if (group.length < minOccurrences || sessions.size < minSessions) continue;
250
+
251
+ const scope = group.reduce(
252
+ (narrowest, o) => {
253
+ const s = o.meta?.scopeHint ?? 'session';
254
+ return s === 'session' ? 'session' : narrowest === 'session' ? 'session' : s === 'project' ? 'project' : narrowest;
255
+ },
256
+ 'user',
257
+ );
258
+
259
+ const first = group[0];
260
+ const evidenceRefs = group
261
+ .flatMap((o) => [o.meta?.evidenceRef, o.id])
262
+ .filter(Boolean)
263
+ .slice(0, MAX_EVIDENCE_REFS);
264
+
265
+ const draft = {
266
+ scope,
267
+ target: first.meta?.target ?? null,
268
+ proposedDelta: first.meta?.proposedDelta ?? { field: first.meta?.target ?? null, value: first.text },
269
+ };
270
+ const conflicts = detectCandidateConflicts(draft, records);
271
+
272
+ const candidateId = `lc_${crypto.randomBytes(6).toString('hex')}`;
273
+ return addRecord(projectRoot, {
274
+ type: 'derived_fact',
275
+ scope: CANDIDATE_TO_RECORD_SCOPE[scope],
276
+ text: candidateText(first.meta?.kind, first.text, group.length, sessions.size),
277
+ provenance: CANDIDATE_PROVENANCE,
278
+ confidence: Math.min(0.9, 0.4 + group.length * 0.1),
279
+ createdBy: 'learning-candidate',
280
+ projectId,
281
+ meta: {
282
+ id: candidateId,
283
+ candidateId,
284
+ schemaVersion: 1,
285
+ status: 'proposed',
286
+ signature,
287
+ kind: first.meta?.kind ?? null,
288
+ scope,
289
+ proposalType: PROPOSAL_TYPE_BY_KIND[first.meta?.kind] ?? 'knowledge',
290
+ condition: first.meta?.condition ?? `when ${first.meta?.kind ?? 'signal'} recurs`,
291
+ proposedDelta: draft.proposedDelta,
292
+ target: draft.target,
293
+ evidenceRefs,
294
+ occurrences: group.length,
295
+ sessions: sessions.size,
296
+ conflicts,
297
+ },
298
+ });
299
+ }
300
+
301
+ return firstExisting;
302
+ }
@@ -10,7 +10,7 @@ import { buildRuntimePaths } from '../runtimePaths.js';
10
10
  import { readJsonIfExists, writeJson } from '../fileOps.js';
11
11
  import { loadRuntimeConfig } from '../runtimeConfig.js';
12
12
  import { createRecord } from './records.js';
13
- import { saveRecords } from './storeV2.js';
13
+ import { mutateRecordStore } from './recordStore.js';
14
14
 
15
15
  const MARKER_NAME = 'migrated-from-v1.json';
16
16
  const DAY_MS = 24 * 60 * 60 * 1000;
@@ -24,11 +24,14 @@ async function pathExists(targetPath) {
24
24
  }
25
25
  }
26
26
 
27
+ // Returns { value, corrupt } — a parse failure is reported, never silently
28
+ // dropped: corrupt legacy files count toward runMigration's `skipped` total
29
+ // and keep needsMigration true so the loss is surfaced, not hidden.
27
30
  async function readJsonTolerant(filePath) {
28
31
  try {
29
- return await readJsonIfExists(filePath);
32
+ return { value: await readJsonIfExists(filePath), corrupt: false };
30
33
  } catch {
31
- return null;
34
+ return { value: null, corrupt: true };
32
35
  }
33
36
  }
34
37
 
@@ -64,9 +67,10 @@ function fingerprint(type, text) {
64
67
  return crypto.createHash('sha1').update(`v1:${type}:${normalized}`).digest('hex');
65
68
  }
66
69
 
67
- const skippedCounter = { count: 0 };
70
+ // Per-run skip counter — threaded explicitly so concurrent runMigration calls
71
+ // can never cross-report each other's skips (a module-level counter did).
68
72
 
69
- function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }) {
73
+ function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }, skip) {
70
74
  try {
71
75
  const record = createRecord({
72
76
  type,
@@ -84,12 +88,12 @@ function makeRecord({ type, scope, text, confidence, provenance, projectId = nul
84
88
  if (status) record.status = status;
85
89
  return record;
86
90
  } catch {
87
- skippedCounter.count += 1;
91
+ skip.count += 1;
88
92
  return null;
89
93
  }
90
94
  }
91
95
 
92
- function migrateUserMemory(user, records) {
96
+ function migrateUserMemory(user, records, skip) {
93
97
  if (!user || typeof user !== 'object') return;
94
98
  for (const rule of Array.isArray(user.rules) ? user.rules : []) {
95
99
  if (typeof rule !== 'string' || !rule) continue;
@@ -97,7 +101,7 @@ function migrateUserMemory(user, records) {
97
101
  type: 'project_rule', scope: 'user', text: rule,
98
102
  confidence: 1.0, provenance: 'legacy:user-rules',
99
103
  createdAt: user.updatedAt,
100
- });
104
+ }, skip);
101
105
  if (rec) records.push(rec);
102
106
  }
103
107
  const prefs = user.preferences && typeof user.preferences === 'object' ? user.preferences : {};
@@ -107,12 +111,12 @@ function migrateUserMemory(user, records) {
107
111
  type: 'derived_fact', scope: 'user', text,
108
112
  confidence: 1.0, provenance: 'legacy:user-preferences',
109
113
  createdAt: user.updatedAt,
110
- });
114
+ }, skip);
111
115
  if (rec) records.push(rec);
112
116
  }
113
117
  }
114
118
 
115
- function migrateProjectMemory(project, projectId, records) {
119
+ function migrateProjectMemory(project, projectId, records, skip) {
116
120
  if (!project || typeof project !== 'object') return;
117
121
  const createdAt = project.updatedAt;
118
122
 
@@ -122,7 +126,7 @@ function migrateProjectMemory(project, projectId, records) {
122
126
  type: 'project_rule', scope: 'repo', text,
123
127
  confidence: 0.9, provenance: 'legacy:conventions',
124
128
  projectId, createdAt,
125
- });
129
+ }, skip);
126
130
  if (rec) records.push(rec);
127
131
  }
128
132
 
@@ -132,7 +136,7 @@ function migrateProjectMemory(project, projectId, records) {
132
136
  type: 'project_rule', scope: 'repo', text,
133
137
  confidence: 1.0, provenance: 'legacy:activeRules',
134
138
  projectId, createdAt,
135
- });
139
+ }, skip);
136
140
  if (rec) records.push(rec);
137
141
  }
138
142
 
@@ -144,7 +148,7 @@ function migrateProjectMemory(project, projectId, records) {
144
148
  type: 'derived_fact', scope: 'repo', text,
145
149
  confidence: 0.9, provenance: 'legacy:decisions',
146
150
  projectId, createdAt: decision.when ?? createdAt,
147
- });
151
+ }, skip);
148
152
  if (rec) records.push(rec);
149
153
  }
150
154
 
@@ -157,7 +161,7 @@ function migrateProjectMemory(project, projectId, records) {
157
161
  type: 'derived_fact', scope: 'repo', text,
158
162
  confidence: 0.8, provenance: 'legacy:project-profile',
159
163
  projectId, createdAt,
160
- });
164
+ }, skip);
161
165
  if (rec) records.push(rec);
162
166
  }
163
167
 
@@ -172,11 +176,11 @@ function migrateProjectMemory(project, projectId, records) {
172
176
  };
173
177
  let rec = null;
174
178
  if (legacyStatus === 'approved') {
175
- rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 });
179
+ rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 }, skip);
176
180
  } else if (legacyStatus === 'rejected') {
177
- rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' });
181
+ rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' }, skip);
178
182
  } else {
179
- rec = makeRecord({ ...base, type: 'derived_fact' });
183
+ rec = makeRecord({ ...base, type: 'derived_fact' }, skip);
180
184
  }
181
185
  if (rec) records.push(rec);
182
186
  }
@@ -190,7 +194,7 @@ function sessionText(session) {
190
194
  ].filter(Boolean).join(' — ');
191
195
  }
192
196
 
193
- function migrateSession(session, episodeTtlMs, records) {
197
+ function migrateSession(session, episodeTtlMs, records, skip) {
194
198
  if (!session || typeof session !== 'object') return;
195
199
  const text = sessionText(session);
196
200
  if (!text) return;
@@ -202,19 +206,24 @@ function migrateSession(session, episodeTtlMs, records) {
202
206
  projectId: typeof session.projectId === 'string' ? session.projectId : null,
203
207
  createdAt, validUntil,
204
208
  meta: { legacyId: session.id ?? null },
205
- });
209
+ }, skip);
206
210
  if (rec) records.push(rec);
207
211
  }
208
212
 
209
213
  async function collectLegacySources(paths) {
210
- const sources = { user: null, projects: [], sessions: [] };
214
+ const sources = { user: null, projects: [], sessions: [], corruptFiles: 0 };
211
215
 
212
216
  const user = await readJsonTolerant(paths.userMemoryPath);
213
- if (user) sources.user = user;
217
+ if (user.corrupt) sources.corruptFiles += 1;
218
+ if (user.value) sources.user = user.value;
214
219
 
215
220
  for (const name of await listJsonFiles(paths.projectsDir)) {
216
221
  const filePath = path.join(paths.projectsDir, name);
217
- const content = await readJsonTolerant(filePath);
222
+ const { value: content, corrupt } = await readJsonTolerant(filePath);
223
+ if (corrupt) {
224
+ sources.corruptFiles += 1;
225
+ continue;
226
+ }
218
227
  if (!content) continue;
219
228
  if (name.endsWith('.archive.json')) {
220
229
  for (const s of Array.isArray(content.sessions) ? content.sessions : []) {
@@ -227,7 +236,11 @@ async function collectLegacySources(paths) {
227
236
 
228
237
  for (const name of await listJsonFiles(paths.sessionsDir)) {
229
238
  const filePath = path.join(paths.sessionsDir, name);
230
- const content = await readJsonTolerant(filePath);
239
+ const { value: content, corrupt } = await readJsonTolerant(filePath);
240
+ if (corrupt) {
241
+ sources.corruptFiles += 1;
242
+ continue;
243
+ }
231
244
  if (!content) continue;
232
245
  if (name.endsWith('.archive.json') && Array.isArray(content.sessions)) {
233
246
  sources.sessions.push(...content.sessions);
@@ -240,10 +253,11 @@ async function collectLegacySources(paths) {
240
253
  }
241
254
 
242
255
  function hasLegacyContent(sources) {
243
- return Boolean(sources.user) || sources.projects.length > 0 || sources.sessions.length > 0;
256
+ return Boolean(sources.user) || sources.projects.length > 0
257
+ || sources.sessions.length > 0 || sources.corruptFiles > 0;
244
258
  }
245
259
 
246
- async function buildRecords(projectRoot, paths, sources) {
260
+ async function buildRecords(projectRoot, paths, sources, skip) {
247
261
  const config = await loadRuntimeConfig(projectRoot);
248
262
  const ttlDays = Number.isFinite(config?.memoryV2?.episodeTtlDays) && config.memoryV2.episodeTtlDays > 0
249
263
  ? config.memoryV2.episodeTtlDays
@@ -251,9 +265,9 @@ async function buildRecords(projectRoot, paths, sources) {
251
265
  const episodeTtlMs = ttlDays * DAY_MS;
252
266
 
253
267
  const records = [];
254
- if (sources.user) migrateUserMemory(sources.user, records);
255
- for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records);
256
- for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records);
268
+ if (sources.user) migrateUserMemory(sources.user, records, skip);
269
+ for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records, skip);
270
+ for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records, skip);
257
271
  return records;
258
272
  }
259
273
 
@@ -285,9 +299,9 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
285
299
  const sources = await collectLegacySources(paths);
286
300
  if (!hasLegacyContent(sources)) return empty;
287
301
 
288
- skippedCounter.count = 0;
289
- const records = await buildRecords(projectRoot, paths, sources);
290
- const skipped = skippedCounter.count;
302
+ const skip = { count: 0 };
303
+ const records = await buildRecords(projectRoot, paths, sources, skip);
304
+ const skipped = skip.count + sources.corruptFiles;
291
305
  if (dryRun) {
292
306
  return { migrated: records.length, skipped, markerPath, backupDir: null };
293
307
  }
@@ -307,7 +321,20 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
307
321
  }
308
322
  }
309
323
 
310
- await saveRecords(projectRoot, records);
324
+ // Merge into the existing store under the shared lock — never clobber.
325
+ // records.json can already hold records when the marker is absent (a crash
326
+ // between saveRecords and the marker write, or a manual re-run); overwriting
327
+ // it would silently destroy post-migration data. source_fingerprint dedupes
328
+ // re-migrated records so the merge stays idempotent.
329
+ await mutateRecordStore(paths.memoryV2RecordsPath, (existing) => {
330
+ const fingerprints = new Set(
331
+ existing.map((r) => r.source_fingerprint).filter(Boolean),
332
+ );
333
+ const additions = records.filter(
334
+ (r) => !r.source_fingerprint || !fingerprints.has(r.source_fingerprint),
335
+ );
336
+ return { result: additions.length, records: [...existing, ...additions] };
337
+ });
311
338
 
312
339
  const counts = {
313
340
  total: records.length,
@@ -4,7 +4,7 @@
4
4
  // (storeV2.js) and the user store (userMemory.js) share one implementation.
5
5
  // Writes go through fileOps.writeJson (tmp+rename).
6
6
 
7
- import { readJsonIfExists, writeJson } from '../fileOps.js';
7
+ import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
8
8
  import { normalizeRecord } from './records.js';
9
9
 
10
10
  const SCHEMA_VERSION = 2;
@@ -79,3 +79,26 @@ export function applyRecordPatch(records, id, patch = {}) {
79
79
  updated[index] = normalized;
80
80
  return { record: normalized, records: updated };
81
81
  }
82
+
83
+ /**
84
+ * mutateRecordStore(recordsPath, mutate) → mutate's result | null
85
+ * Serializes a read-modify-write cycle on the records document under the
86
+ * shared file lock (fileOps.withFileLock) so concurrent add/update flows —
87
+ * in-process or across hook processes — never lose each other's writes.
88
+ * `mutate(records)` returns { result, records } to persist, or null to skip
89
+ * the write (e.g. id not found). Throws when the lock wait expires: a
90
+ * silently dropped mutation would corrupt caller state worse than an error.
91
+ */
92
+ export async function mutateRecordStore(recordsPath, mutate) {
93
+ const outcome = await withFileLock(recordsPath, async () => {
94
+ const { records } = await readRecordStore(recordsPath);
95
+ const mutation = await mutate(records);
96
+ if (!mutation) return null;
97
+ await writeRecordStore(recordsPath, mutation.records);
98
+ return mutation.result ?? null;
99
+ });
100
+ if (outcome === undefined) {
101
+ throw new Error(`mutateRecordStore: lock wait expired for ${recordsPath} — mutation skipped`);
102
+ }
103
+ return outcome;
104
+ }
@@ -365,9 +365,11 @@ async function forgetMemoryItemV2(projectRoot, memoryId, runtimePaths) {
365
365
 
366
366
  if (memoryId === 'user:user' || memoryId === 'user') {
367
367
  const userRecords = await v2.queryRecords(projectRoot, { scope: 'user' });
368
- const archived = await archiveAll(userRecords);
368
+ await archiveAll(userRecords);
369
369
  await writeJson(runtimePaths.userMemoryPath, defaultUserMemory());
370
- return { removed: archived > 0 || true, type: 'user', path: runtimePaths.userMemoryPath };
370
+ // The reset above always removes user memory — `removed` reports that the
371
+ // forget happened, not how many v2 records were archived.
372
+ return { removed: true, type: 'user', path: runtimePaths.userMemoryPath };
371
373
  }
372
374
 
373
375
  const id = String(memoryId);
@@ -539,13 +541,28 @@ export async function runProjectHygiene(projectRoot, projectId) {
539
541
  }
540
542
 
541
543
  function patternCandidateFromRecord(record) {
544
+ const isLearningCandidate = record.provenance === 'learning-candidate';
542
545
  return {
543
546
  id: record.meta?.id ?? record.id,
544
547
  text: record.text,
545
548
  category: record.meta?.category ?? null,
546
549
  detectedFrom: record.meta?.detectedFrom ?? null,
547
550
  detectedAt: record.meta?.detectedAt ?? record.created_at ?? null,
548
- status: record.meta?.legacyStatus ?? 'pending',
551
+ // Learning candidates carry C11 status 'proposed'; the pending listing
552
+ // keeps the legacy 'pending' label so consumers see one shape.
553
+ status: record.meta?.legacyStatus
554
+ ?? (isLearningCandidate && record.meta?.status === 'proposed' ? 'pending' : record.meta?.status)
555
+ ?? 'pending',
556
+ ...(isLearningCandidate ? {
557
+ kind: 'learning-candidate',
558
+ candidateStatus: record.meta?.status ?? 'proposed',
559
+ scope: record.meta?.scope ?? null,
560
+ proposalType: record.meta?.proposalType ?? null,
561
+ condition: record.meta?.condition ?? null,
562
+ proposedDelta: record.meta?.proposedDelta ?? null,
563
+ evidenceRefs: record.meta?.evidenceRefs ?? [],
564
+ conflicts: record.meta?.conflicts ?? [],
565
+ } : {}),
549
566
  _recordId: record.id,
550
567
  };
551
568
  }
@@ -594,32 +611,51 @@ async function listPendingPatternCandidatesV2(projectRoot, projectId) {
594
611
  const v2 = await storeV2();
595
612
  return (await v2.queryRecords(projectRoot, { projectId }))
596
613
  .filter((record) => record.type === 'derived_fact'
597
- && record.meta?.legacyStatus === 'pending'
598
- && record.status !== 'archived')
614
+ && record.status !== 'archived'
615
+ && (record.meta?.legacyStatus === 'pending'
616
+ // Learning candidates (TASK-008) surface through the same pending
617
+ // listing — no second store. C11 'proposed' maps to 'pending'.
618
+ || (record.provenance === 'learning-candidate' && record.meta?.status === 'proposed')))
599
619
  .map(patternCandidateFromRecord);
600
620
  }
601
621
 
602
622
  async function resolvePatternCandidateV2(projectRoot, projectId, candidateId, decision) {
603
623
  const v2 = await storeV2();
604
624
  const records = await v2.queryRecords(projectRoot, { projectId });
605
- const target = records.find((record) => record.meta?.id === candidateId || record.id === candidateId);
625
+ // Only candidate records are resolvable — matching every record by meta.id
626
+ // let a delta-overlay (meta.id 'ov_*') be "approved" into a project_rule or
627
+ // archived, corrupting the overlay store.
628
+ const target = records.find((record) => (
629
+ String(record.provenance ?? '').includes('pattern-candidate')
630
+ || record.provenance === 'learning-candidate'
631
+ ) && (record.meta?.id === candidateId || record.id === candidateId));
606
632
  if (!target) {
607
633
  throw new Error(`Pattern candidate not found: ${candidateId}`);
608
634
  }
609
635
 
610
636
  let updated;
637
+ const isLearningCandidate = target.provenance === 'learning-candidate';
611
638
  if (decision === 'approve') {
612
639
  updated = await v2.updateRecord(projectRoot, target.id, {
613
640
  type: 'project_rule',
614
641
  scope: 'repo',
615
642
  confidence: 0.9,
616
643
  provenance: `${target.provenance ?? 'pattern-candidate'};promoted-from:${target.id}`,
617
- meta: { ...target.meta, legacyStatus: 'approved' },
644
+ meta: {
645
+ ...target.meta,
646
+ legacyStatus: 'approved',
647
+ // Keep the C11 status field in sync for learning candidates.
648
+ ...(isLearningCandidate ? { status: 'approved' } : {}),
649
+ },
618
650
  });
619
651
  } else {
620
652
  updated = await v2.updateRecord(projectRoot, target.id, {
621
653
  status: 'archived',
622
- meta: { ...target.meta, legacyStatus: 'rejected' },
654
+ meta: {
655
+ ...target.meta,
656
+ legacyStatus: 'rejected',
657
+ ...(isLearningCandidate ? { status: 'rejected' } : {}),
658
+ },
623
659
  });
624
660
  }
625
661
  return patternCandidateFromRecord(updated ?? target);