@ngockhoale/ukit 3.0.3 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/bin/ukit +12 -4
- package/manifests/engineConformance.yaml +29 -0
- package/manifests/platform.full.yaml +13 -0
- package/package.json +2 -1
- package/scripts/audit/decision-coverage.mjs +295 -0
- package/scripts/bench/outline-savings.mjs +19 -4
- package/scripts/bench/parallel-agents.mjs +15 -4
- package/scripts/bench/runGold.mjs +22 -4
- package/scripts/bench/v3-ceremony.mjs +7 -1
- package/scripts/bug/triage.mjs +56 -17
- package/scripts/index/build-index.mjs +94 -28
- package/scripts/index/query-index.mjs +48 -14
- package/scripts/index/refresh-index.mjs +142 -62
- package/scripts/perf/audit-perf.mjs +8 -2
- package/scripts/skill/audit-skill.mjs +54 -25
- package/src/bug/triageBug.js +9 -6
- package/src/cli/adapters.js +6 -0
- package/src/cli/commands/code.js +7 -1
- package/src/cli/commands/indexArgs.js +4 -2
- package/src/cli/commands/indexTools.js +8 -1
- package/src/cli/commands/install.js +13 -0
- package/src/cli/commands/memory.js +10 -3
- package/src/cli/commands/status.js +17 -1
- package/src/cli/commands/update.js +7 -0
- package/src/context/detectProjectContext.js +3 -1
- package/src/core/codeintel/analogy.js +1 -1
- package/src/core/codeintel/diagnostics.js +9 -6
- package/src/core/codeintel/graph.js +14 -8
- package/src/core/codeintel/impact.js +0 -1
- package/src/core/codeintel/invalidation.js +5 -3
- package/src/core/codeintel/packet.js +11 -0
- package/src/core/codeintel/router.js +17 -3
- package/src/core/codeintel/semanticProvider.js +1 -1
- package/src/core/codeintel/summaries.js +11 -7
- package/src/core/compact/contextBudget.js +26 -12
- package/src/core/compact/index.js +15 -8
- package/src/core/docContracts.js +10 -2
- package/src/core/experiments/deliberation.js +321 -0
- package/src/core/experiments/dynamicWorkflow.js +492 -0
- package/src/core/fileOps.js +8 -1
- package/src/core/gatewayProbe.js +29 -1
- package/src/core/gatewayResilienceEnv.js +44 -3
- package/src/core/handoffDocValidator.js +3 -1
- package/src/core/hookChainDoctor.js +16 -1
- package/src/core/memory/deltaOverlays.js +448 -0
- package/src/core/memory/learningCandidates.js +302 -0
- package/src/core/memory/migrate.js +59 -32
- package/src/core/memory/recordStore.js +24 -1
- package/src/core/memory/store.js +44 -8
- package/src/core/memory/storeV2.js +48 -33
- package/src/core/memory/userMemory.js +11 -10
- package/src/core/output/index.js +15 -10
- package/src/core/permissionPolicy.js +8 -0
- package/src/core/runtimeConfig.js +224 -4
- package/src/core/sensitiveValueScanner.js +10 -2
- package/src/core/taskBudgetValidator.js +12 -17
- package/src/core/taskProgressGuard.js +59 -9
- package/src/core/unattendedDoctor.js +5 -2
- package/src/core/uninstall.js +37 -8
- package/src/decision/client.js +371 -0
- package/src/decision/lease.js +198 -0
- package/src/decision/preflight.js +492 -0
- package/src/decision/protocol.js +308 -0
- package/src/decision/registry.js +384 -0
- package/src/decision/shadow.js +281 -0
- package/src/decision/statePacket.js +165 -0
- package/src/diagnostics/failurePatterns.js +2 -1
- package/src/diagnostics/ledgerFiles.js +3 -1
- package/src/diagnostics/routeOutcomes.js +1 -29
- package/src/index/buildIndex.js +23 -11
- package/src/index/impactContext.js +21 -2
- package/src/index/importResolution.js +13 -7
- package/src/index/queryIndex.js +11 -5
- package/src/index/resolveContext.js +16 -6
- package/src/index/taskRouting.js +63 -2
- package/src/index/verificationPlan.js +12 -1
- package/src/learning/patternProposals.js +6 -0
- package/src/render/renderTemplate.js +1 -1
- package/src/skill/auditSkill.js +3 -1
- package/src/stack/detectStack.js +3 -1
- package/template_project/.claude/agents/handoff-planner.md +2 -5
- package/template_project/.claude/hooks/auto-allow-bash.sh +5 -0
- package/template_project/.claude/hooks/block-dangerous.mjs +11 -4
- package/template_project/.claude/hooks/context-hardcap-gate.sh +10 -1
- package/template_project/.claude/hooks/handoff-model-guard.sh +14 -4
- package/template_project/.claude/hooks/handoff-resume.sh +10 -1
- package/template_project/.claude/hooks/protect-files.sh +0 -1
- package/template_project/.claude/hooks/record-execution.mjs +13 -1
- package/template_project/.claude/hooks/sensitive-data-guard.mjs +71 -5
- package/template_project/.claude/hooks/session-episode.sh +9 -2
- package/template_project/.claude/skills/pdf-processing-pro/SKILL.md +1 -1
- package/template_project/.claude/ukit/index/handoff-doc-validator.mjs +3 -1
- package/template_project/.claude/ukit/index/lib/index-core.mjs +123 -39
- package/template_project/.claude/ukit/index/route-task.mjs +444 -0
- package/template_project/.claude/ukit/index/task-budget-validator.mjs +12 -16
- package/template_project/.claude/ukit/index/unic-decision.mjs +786 -0
- package/template_project/.claude/ukit/index/verify-context.mjs +11 -0
- package/template_project/.claude/ukit/runtime/execution-ledger.mjs +116 -9
- package/template_project/.claude/ukit/runtime/project-important.mjs +9 -7
- package/template_project/.claude/ukit/runtime/reinject-context.mjs +48 -0
- package/template_project/.claude/ukit/runtime/resumable-run.mjs +596 -0
- package/template_project/.claude/ukit/runtime/sensitive-value-scanner.mjs +4 -7
- package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +1 -1
- package/template_project/docs/AI_HANDOFF/PLAN.md +7 -7
- package/template_project/docs/AI_HANDOFF/RULES.md +1 -1
- package/template_project/ukit/storage/config.json +34 -1
- package/template_project/.claude/ukit/skill-router-state.json +0 -1
- package/template_project/.ukit/storage/cache/hook-latency/unknown.jsonl +0 -4
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
// learningCandidates.js — M04.2 learning candidates (SPEC §5 FR-016, FR-018; C11).
|
|
2
|
+
//
|
|
3
|
+
// Normalizes repeated corrections/suppressions/escalations/verification-gaps
|
|
4
|
+
// into observation records, then promotes them to LearningCandidate records
|
|
5
|
+
// (C11) only after configured recurrence AND cross-session evidence.
|
|
6
|
+
//
|
|
7
|
+
// Contracts:
|
|
8
|
+
// * Gated by `learning.candidates.stage` — 'off' (default) means zero
|
|
9
|
+
// writes: observeLearningSignal is a no-op, maybeCreateCandidate returns
|
|
10
|
+
// null. At 'shadow'/'canary'/'default' everything is still REPORT-ONLY:
|
|
11
|
+
// candidates are created with meta.status 'proposed' and promotion stays
|
|
12
|
+
// manual via the existing pending-candidate flow in store.js.
|
|
13
|
+
// * Observations are memoryV2 records (provenance 'learning-observation');
|
|
14
|
+
// candidates are memoryV2 records (provenance 'learning-candidate').
|
|
15
|
+
// No second store.
|
|
16
|
+
// * Scope classification is conservative: any session-scoped signal pins
|
|
17
|
+
// the candidate to 'session' — session signals never leak to
|
|
18
|
+
// project/user.
|
|
19
|
+
// * Conflict detection prefers conditional refinement; project constraints
|
|
20
|
+
// outrank user preferences (C11).
|
|
21
|
+
// * NEVER THROWS on malformed signals — they are skipped, not fatal.
|
|
22
|
+
|
|
23
|
+
import crypto from 'node:crypto';
|
|
24
|
+
import { loadRuntimeConfig, resolveConfigStage } from '../runtimeConfig.js';
|
|
25
|
+
import { addRecord, queryRecords } from './storeV2.js';
|
|
26
|
+
|
|
27
|
+
export const OBSERVATION_PROVENANCE = 'learning-observation';
|
|
28
|
+
export const CANDIDATE_PROVENANCE = 'learning-candidate';
|
|
29
|
+
|
|
30
|
+
const SIGNAL_KINDS = new Set([
|
|
31
|
+
'correction',
|
|
32
|
+
'suppression',
|
|
33
|
+
'escalation',
|
|
34
|
+
'verification-gap',
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
// C11 proposalType per signal kind.
|
|
38
|
+
const PROPOSAL_TYPE_BY_KIND = {
|
|
39
|
+
correction: 'preference',
|
|
40
|
+
suppression: 'constraint',
|
|
41
|
+
escalation: 'knowledge',
|
|
42
|
+
'verification-gap': 'overlay',
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// C11 candidate scopes → memoryV2 record scopes.
|
|
46
|
+
const CANDIDATE_TO_RECORD_SCOPE = {
|
|
47
|
+
session: 'session',
|
|
48
|
+
project: 'repo',
|
|
49
|
+
user: 'user',
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const DEFAULT_MIN_OCCURRENCES = 3;
|
|
53
|
+
const DEFAULT_MIN_SESSIONS = 2;
|
|
54
|
+
const MAX_EVIDENCE_REFS = 24;
|
|
55
|
+
// Records live in the single records.json document — bound every field that
|
|
56
|
+
// carries caller-controlled text so a signal can never grow it unboundedly.
|
|
57
|
+
const MAX_SIGNAL_TEXT = 512;
|
|
58
|
+
const MAX_META_FIELD = 512;
|
|
59
|
+
|
|
60
|
+
function boundText(value, max = MAX_META_FIELD) {
|
|
61
|
+
if (value == null) return null;
|
|
62
|
+
const text = typeof value === 'string' ? value : JSON.stringify(value);
|
|
63
|
+
return typeof text === 'string' ? text.slice(0, max) : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Like boundText but preserves non-string scalars (numbers/booleans) so a
|
|
67
|
+
// proposedDelta value keeps its type for downstream overlay conversion.
|
|
68
|
+
function boundField(value, max = MAX_META_FIELD) {
|
|
69
|
+
if (value == null) return null;
|
|
70
|
+
if (typeof value !== 'string') return value;
|
|
71
|
+
return value.slice(0, max);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Bounds a proposedDelta in place-shape: {field, value} stay typed fields —
|
|
75
|
+
// stringifying the whole object would break detectCandidateConflicts and
|
|
76
|
+
// overlayFromCandidate, which read delta.field/delta.value.
|
|
77
|
+
function boundDelta(delta) {
|
|
78
|
+
if (delta == null || typeof delta !== 'object') return boundText(delta);
|
|
79
|
+
return {
|
|
80
|
+
...delta,
|
|
81
|
+
field: boundField(delta.field),
|
|
82
|
+
value: boundField(delta.value),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function normalizeText(text) {
|
|
87
|
+
return String(text ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function normalizeValue(value) {
|
|
91
|
+
if (value == null) return '';
|
|
92
|
+
return normalizeText(typeof value === 'string' ? value : JSON.stringify(value));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
function signalScopeHint(signal) {
|
|
97
|
+
const hint = signal?.scopeHint ?? signal?.scope;
|
|
98
|
+
return hint === 'project' || hint === 'user' ? hint : 'session';
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function candidatesConfig(projectRoot) {
|
|
102
|
+
const config = await loadRuntimeConfig(projectRoot);
|
|
103
|
+
const stage = resolveConfigStage(config, 'learning.candidates.stage');
|
|
104
|
+
const raw = config?.learning?.candidates ?? {};
|
|
105
|
+
return {
|
|
106
|
+
stage,
|
|
107
|
+
minOccurrences: Number.isFinite(raw.minOccurrences) && raw.minOccurrences > 0
|
|
108
|
+
? raw.minOccurrences
|
|
109
|
+
: DEFAULT_MIN_OCCURRENCES,
|
|
110
|
+
minSessions: Number.isFinite(raw.minSessions) && raw.minSessions > 0
|
|
111
|
+
? raw.minSessions
|
|
112
|
+
: DEFAULT_MIN_SESSIONS,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* classifyCandidateScope(signal) → 'session' | 'project' | 'user'
|
|
118
|
+
* Conservative: unknown/absent hints resolve to 'session'.
|
|
119
|
+
*/
|
|
120
|
+
export function classifyCandidateScope(signal) {
|
|
121
|
+
return signalScopeHint(signal);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* detectCandidateConflicts(candidate, existing) → conflicts[]
|
|
126
|
+
* A conflict is a contradiction on the same target: the candidate proposes a
|
|
127
|
+
* different value for a field an existing project_rule/constraint already
|
|
128
|
+
* pins. Project (repo-scoped) constraints outrank user-scoped candidates —
|
|
129
|
+
* the conflict is flagged `outranked` and resolved by conditional refinement
|
|
130
|
+
* rather than silent override.
|
|
131
|
+
*/
|
|
132
|
+
export function detectCandidateConflicts(candidate, existing = []) {
|
|
133
|
+
const target = normalizeText(candidate?.target ?? candidate?.proposedDelta?.field);
|
|
134
|
+
if (!target) return [];
|
|
135
|
+
const proposedValue = normalizeValue(
|
|
136
|
+
candidate?.proposedDelta?.value ?? candidate?.proposedDelta,
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
const conflicts = [];
|
|
140
|
+
for (const record of existing ?? []) {
|
|
141
|
+
if (!record || record.status === 'archived') continue;
|
|
142
|
+
if (record.provenance === CANDIDATE_PROVENANCE
|
|
143
|
+
&& record.meta?.status !== 'approved') continue;
|
|
144
|
+
const recordTarget = normalizeText(record.meta?.target ?? record.meta?.constraint);
|
|
145
|
+
if (!recordTarget || recordTarget !== target) continue;
|
|
146
|
+
const recordValue = normalizeValue(record.meta?.value);
|
|
147
|
+
if (!recordValue || recordValue === proposedValue) continue;
|
|
148
|
+
const isProjectConstraint = record.type === 'project_rule' || record.scope === 'repo';
|
|
149
|
+
conflicts.push({
|
|
150
|
+
type: 'contradiction',
|
|
151
|
+
recordId: record.id,
|
|
152
|
+
target,
|
|
153
|
+
existingValue: record.meta?.value ?? null,
|
|
154
|
+
proposedValue: candidate?.proposedDelta?.value ?? null,
|
|
155
|
+
outranked: isProjectConstraint && candidate?.scope === 'user',
|
|
156
|
+
resolution: 'conditional-refinement',
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
return conflicts;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* observeLearningSignal(projectRoot, signal) → observation record | null
|
|
164
|
+
* signal: { kind, text, sessionId?, projectId?, scopeHint?, target?,
|
|
165
|
+
* condition?, proposedDelta?, evidenceRef? }
|
|
166
|
+
* Returns null when the feature stage is 'off' or the signal is malformed.
|
|
167
|
+
*/
|
|
168
|
+
export async function observeLearningSignal(projectRoot, signal = {}) {
|
|
169
|
+
const { stage } = await candidatesConfig(projectRoot);
|
|
170
|
+
if (stage === 'off') return null;
|
|
171
|
+
if (!SIGNAL_KINDS.has(signal?.kind) || !normalizeText(signal?.text)) return null;
|
|
172
|
+
|
|
173
|
+
// Every caller-controlled string is bounded — records.json is a single
|
|
174
|
+
// document, so an unbounded signal would grow it without limit.
|
|
175
|
+
const text = boundText(signal.text, MAX_SIGNAL_TEXT);
|
|
176
|
+
const target = boundText(signal.target);
|
|
177
|
+
const signature = `${signal.kind}:${normalizeText(target ?? text)}`.slice(0, MAX_META_FIELD);
|
|
178
|
+
|
|
179
|
+
return addRecord(projectRoot, {
|
|
180
|
+
type: 'derived_fact',
|
|
181
|
+
scope: CANDIDATE_TO_RECORD_SCOPE[signalScopeHint(signal)],
|
|
182
|
+
text: `learning-signal ${signal.kind}: ${String(text).trim()}`,
|
|
183
|
+
provenance: OBSERVATION_PROVENANCE,
|
|
184
|
+
confidence: 0.3,
|
|
185
|
+
createdBy: 'learning-observe',
|
|
186
|
+
projectId: boundText(signal.projectId),
|
|
187
|
+
meta: {
|
|
188
|
+
kind: signal.kind,
|
|
189
|
+
signature,
|
|
190
|
+
sessionId: boundText(signal.sessionId),
|
|
191
|
+
scopeHint: signalScopeHint(signal),
|
|
192
|
+
target,
|
|
193
|
+
condition: boundText(signal.condition),
|
|
194
|
+
proposedDelta: boundDelta(signal.proposedDelta),
|
|
195
|
+
evidenceRef: boundText(signal.evidenceRef),
|
|
196
|
+
observedAt: Date.now(),
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function candidateText(kind, signalText, occurrences, sessions) {
|
|
202
|
+
return `Learning candidate (${kind}): "${String(signalText).trim()}" — observed ${occurrences}× across ${sessions} sessions.`;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* maybeCreateCandidate(projectRoot) → candidate record | null
|
|
207
|
+
* Scans observation records, groups by signature + projectId, and promotes
|
|
208
|
+
* the first eligible group (occurrences >= minOccurrences AND distinct
|
|
209
|
+
* sessions >= minSessions) into a proposed LearningCandidate. Idempotent —
|
|
210
|
+
* an existing proposed/approved candidate for the signature is returned
|
|
211
|
+
* instead of duplicated.
|
|
212
|
+
*/
|
|
213
|
+
export async function maybeCreateCandidate(projectRoot) {
|
|
214
|
+
const { stage, minOccurrences, minSessions } = await candidatesConfig(projectRoot);
|
|
215
|
+
if (stage === 'off') return null;
|
|
216
|
+
|
|
217
|
+
const records = await queryRecords(projectRoot, {});
|
|
218
|
+
const observations = records
|
|
219
|
+
.filter((r) => r.provenance === OBSERVATION_PROVENANCE && r.status !== 'archived')
|
|
220
|
+
.sort((a, b) => (a.meta?.observedAt ?? a.created_at ?? 0) - (b.meta?.observedAt ?? b.created_at ?? 0));
|
|
221
|
+
const existingCandidates = records.filter((r) => r.provenance === CANDIDATE_PROVENANCE);
|
|
222
|
+
|
|
223
|
+
const groups = new Map();
|
|
224
|
+
for (const obs of observations) {
|
|
225
|
+
const key = `${obs.project_id ?? ''}|${obs.meta?.signature ?? ''}`;
|
|
226
|
+
if (!groups.has(key)) groups.set(key, []);
|
|
227
|
+
groups.get(key).push(obs);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
let firstExisting = null;
|
|
231
|
+
for (const group of groups.values()) {
|
|
232
|
+
const signature = group[0]?.meta?.signature;
|
|
233
|
+
const projectId = group[0]?.project_id ?? null;
|
|
234
|
+
|
|
235
|
+
const existing = existingCandidates.find(
|
|
236
|
+
(r) => r.meta?.signature === signature
|
|
237
|
+
&& (r.project_id ?? null) === projectId
|
|
238
|
+
&& (r.meta?.status === 'proposed' || r.meta?.status === 'approved'),
|
|
239
|
+
);
|
|
240
|
+
// Remember the first existing candidate but KEEP SCANNING — returning it
|
|
241
|
+
// here starved every later eligible group: group A's candidate blocked
|
|
242
|
+
// group B's from ever being created.
|
|
243
|
+
if (existing) {
|
|
244
|
+
firstExisting ??= existing;
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const sessions = new Set(group.map((o) => o.meta?.sessionId ?? null));
|
|
249
|
+
if (group.length < minOccurrences || sessions.size < minSessions) continue;
|
|
250
|
+
|
|
251
|
+
const scope = group.reduce(
|
|
252
|
+
(narrowest, o) => {
|
|
253
|
+
const s = o.meta?.scopeHint ?? 'session';
|
|
254
|
+
return s === 'session' ? 'session' : narrowest === 'session' ? 'session' : s === 'project' ? 'project' : narrowest;
|
|
255
|
+
},
|
|
256
|
+
'user',
|
|
257
|
+
);
|
|
258
|
+
|
|
259
|
+
const first = group[0];
|
|
260
|
+
const evidenceRefs = group
|
|
261
|
+
.flatMap((o) => [o.meta?.evidenceRef, o.id])
|
|
262
|
+
.filter(Boolean)
|
|
263
|
+
.slice(0, MAX_EVIDENCE_REFS);
|
|
264
|
+
|
|
265
|
+
const draft = {
|
|
266
|
+
scope,
|
|
267
|
+
target: first.meta?.target ?? null,
|
|
268
|
+
proposedDelta: first.meta?.proposedDelta ?? { field: first.meta?.target ?? null, value: first.text },
|
|
269
|
+
};
|
|
270
|
+
const conflicts = detectCandidateConflicts(draft, records);
|
|
271
|
+
|
|
272
|
+
const candidateId = `lc_${crypto.randomBytes(6).toString('hex')}`;
|
|
273
|
+
return addRecord(projectRoot, {
|
|
274
|
+
type: 'derived_fact',
|
|
275
|
+
scope: CANDIDATE_TO_RECORD_SCOPE[scope],
|
|
276
|
+
text: candidateText(first.meta?.kind, first.text, group.length, sessions.size),
|
|
277
|
+
provenance: CANDIDATE_PROVENANCE,
|
|
278
|
+
confidence: Math.min(0.9, 0.4 + group.length * 0.1),
|
|
279
|
+
createdBy: 'learning-candidate',
|
|
280
|
+
projectId,
|
|
281
|
+
meta: {
|
|
282
|
+
id: candidateId,
|
|
283
|
+
candidateId,
|
|
284
|
+
schemaVersion: 1,
|
|
285
|
+
status: 'proposed',
|
|
286
|
+
signature,
|
|
287
|
+
kind: first.meta?.kind ?? null,
|
|
288
|
+
scope,
|
|
289
|
+
proposalType: PROPOSAL_TYPE_BY_KIND[first.meta?.kind] ?? 'knowledge',
|
|
290
|
+
condition: first.meta?.condition ?? `when ${first.meta?.kind ?? 'signal'} recurs`,
|
|
291
|
+
proposedDelta: draft.proposedDelta,
|
|
292
|
+
target: draft.target,
|
|
293
|
+
evidenceRefs,
|
|
294
|
+
occurrences: group.length,
|
|
295
|
+
sessions: sessions.size,
|
|
296
|
+
conflicts,
|
|
297
|
+
},
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
return firstExisting;
|
|
302
|
+
}
|
|
@@ -10,7 +10,7 @@ import { buildRuntimePaths } from '../runtimePaths.js';
|
|
|
10
10
|
import { readJsonIfExists, writeJson } from '../fileOps.js';
|
|
11
11
|
import { loadRuntimeConfig } from '../runtimeConfig.js';
|
|
12
12
|
import { createRecord } from './records.js';
|
|
13
|
-
import {
|
|
13
|
+
import { mutateRecordStore } from './recordStore.js';
|
|
14
14
|
|
|
15
15
|
const MARKER_NAME = 'migrated-from-v1.json';
|
|
16
16
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
@@ -24,11 +24,14 @@ async function pathExists(targetPath) {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
// Returns { value, corrupt } — a parse failure is reported, never silently
|
|
28
|
+
// dropped: corrupt legacy files count toward runMigration's `skipped` total
|
|
29
|
+
// and keep needsMigration true so the loss is surfaced, not hidden.
|
|
27
30
|
async function readJsonTolerant(filePath) {
|
|
28
31
|
try {
|
|
29
|
-
return await readJsonIfExists(filePath);
|
|
32
|
+
return { value: await readJsonIfExists(filePath), corrupt: false };
|
|
30
33
|
} catch {
|
|
31
|
-
return null;
|
|
34
|
+
return { value: null, corrupt: true };
|
|
32
35
|
}
|
|
33
36
|
}
|
|
34
37
|
|
|
@@ -64,9 +67,10 @@ function fingerprint(type, text) {
|
|
|
64
67
|
return crypto.createHash('sha1').update(`v1:${type}:${normalized}`).digest('hex');
|
|
65
68
|
}
|
|
66
69
|
|
|
67
|
-
|
|
70
|
+
// Per-run skip counter — threaded explicitly so concurrent runMigration calls
|
|
71
|
+
// can never cross-report each other's skips (a module-level counter did).
|
|
68
72
|
|
|
69
|
-
function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }) {
|
|
73
|
+
function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }, skip) {
|
|
70
74
|
try {
|
|
71
75
|
const record = createRecord({
|
|
72
76
|
type,
|
|
@@ -84,12 +88,12 @@ function makeRecord({ type, scope, text, confidence, provenance, projectId = nul
|
|
|
84
88
|
if (status) record.status = status;
|
|
85
89
|
return record;
|
|
86
90
|
} catch {
|
|
87
|
-
|
|
91
|
+
skip.count += 1;
|
|
88
92
|
return null;
|
|
89
93
|
}
|
|
90
94
|
}
|
|
91
95
|
|
|
92
|
-
function migrateUserMemory(user, records) {
|
|
96
|
+
function migrateUserMemory(user, records, skip) {
|
|
93
97
|
if (!user || typeof user !== 'object') return;
|
|
94
98
|
for (const rule of Array.isArray(user.rules) ? user.rules : []) {
|
|
95
99
|
if (typeof rule !== 'string' || !rule) continue;
|
|
@@ -97,7 +101,7 @@ function migrateUserMemory(user, records) {
|
|
|
97
101
|
type: 'project_rule', scope: 'user', text: rule,
|
|
98
102
|
confidence: 1.0, provenance: 'legacy:user-rules',
|
|
99
103
|
createdAt: user.updatedAt,
|
|
100
|
-
});
|
|
104
|
+
}, skip);
|
|
101
105
|
if (rec) records.push(rec);
|
|
102
106
|
}
|
|
103
107
|
const prefs = user.preferences && typeof user.preferences === 'object' ? user.preferences : {};
|
|
@@ -107,12 +111,12 @@ function migrateUserMemory(user, records) {
|
|
|
107
111
|
type: 'derived_fact', scope: 'user', text,
|
|
108
112
|
confidence: 1.0, provenance: 'legacy:user-preferences',
|
|
109
113
|
createdAt: user.updatedAt,
|
|
110
|
-
});
|
|
114
|
+
}, skip);
|
|
111
115
|
if (rec) records.push(rec);
|
|
112
116
|
}
|
|
113
117
|
}
|
|
114
118
|
|
|
115
|
-
function migrateProjectMemory(project, projectId, records) {
|
|
119
|
+
function migrateProjectMemory(project, projectId, records, skip) {
|
|
116
120
|
if (!project || typeof project !== 'object') return;
|
|
117
121
|
const createdAt = project.updatedAt;
|
|
118
122
|
|
|
@@ -122,7 +126,7 @@ function migrateProjectMemory(project, projectId, records) {
|
|
|
122
126
|
type: 'project_rule', scope: 'repo', text,
|
|
123
127
|
confidence: 0.9, provenance: 'legacy:conventions',
|
|
124
128
|
projectId, createdAt,
|
|
125
|
-
});
|
|
129
|
+
}, skip);
|
|
126
130
|
if (rec) records.push(rec);
|
|
127
131
|
}
|
|
128
132
|
|
|
@@ -132,7 +136,7 @@ function migrateProjectMemory(project, projectId, records) {
|
|
|
132
136
|
type: 'project_rule', scope: 'repo', text,
|
|
133
137
|
confidence: 1.0, provenance: 'legacy:activeRules',
|
|
134
138
|
projectId, createdAt,
|
|
135
|
-
});
|
|
139
|
+
}, skip);
|
|
136
140
|
if (rec) records.push(rec);
|
|
137
141
|
}
|
|
138
142
|
|
|
@@ -144,7 +148,7 @@ function migrateProjectMemory(project, projectId, records) {
|
|
|
144
148
|
type: 'derived_fact', scope: 'repo', text,
|
|
145
149
|
confidence: 0.9, provenance: 'legacy:decisions',
|
|
146
150
|
projectId, createdAt: decision.when ?? createdAt,
|
|
147
|
-
});
|
|
151
|
+
}, skip);
|
|
148
152
|
if (rec) records.push(rec);
|
|
149
153
|
}
|
|
150
154
|
|
|
@@ -157,7 +161,7 @@ function migrateProjectMemory(project, projectId, records) {
|
|
|
157
161
|
type: 'derived_fact', scope: 'repo', text,
|
|
158
162
|
confidence: 0.8, provenance: 'legacy:project-profile',
|
|
159
163
|
projectId, createdAt,
|
|
160
|
-
});
|
|
164
|
+
}, skip);
|
|
161
165
|
if (rec) records.push(rec);
|
|
162
166
|
}
|
|
163
167
|
|
|
@@ -172,11 +176,11 @@ function migrateProjectMemory(project, projectId, records) {
|
|
|
172
176
|
};
|
|
173
177
|
let rec = null;
|
|
174
178
|
if (legacyStatus === 'approved') {
|
|
175
|
-
rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 });
|
|
179
|
+
rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 }, skip);
|
|
176
180
|
} else if (legacyStatus === 'rejected') {
|
|
177
|
-
rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' });
|
|
181
|
+
rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' }, skip);
|
|
178
182
|
} else {
|
|
179
|
-
rec = makeRecord({ ...base, type: 'derived_fact' });
|
|
183
|
+
rec = makeRecord({ ...base, type: 'derived_fact' }, skip);
|
|
180
184
|
}
|
|
181
185
|
if (rec) records.push(rec);
|
|
182
186
|
}
|
|
@@ -190,7 +194,7 @@ function sessionText(session) {
|
|
|
190
194
|
].filter(Boolean).join(' — ');
|
|
191
195
|
}
|
|
192
196
|
|
|
193
|
-
function migrateSession(session, episodeTtlMs, records) {
|
|
197
|
+
function migrateSession(session, episodeTtlMs, records, skip) {
|
|
194
198
|
if (!session || typeof session !== 'object') return;
|
|
195
199
|
const text = sessionText(session);
|
|
196
200
|
if (!text) return;
|
|
@@ -202,19 +206,24 @@ function migrateSession(session, episodeTtlMs, records) {
|
|
|
202
206
|
projectId: typeof session.projectId === 'string' ? session.projectId : null,
|
|
203
207
|
createdAt, validUntil,
|
|
204
208
|
meta: { legacyId: session.id ?? null },
|
|
205
|
-
});
|
|
209
|
+
}, skip);
|
|
206
210
|
if (rec) records.push(rec);
|
|
207
211
|
}
|
|
208
212
|
|
|
209
213
|
async function collectLegacySources(paths) {
|
|
210
|
-
const sources = { user: null, projects: [], sessions: [] };
|
|
214
|
+
const sources = { user: null, projects: [], sessions: [], corruptFiles: 0 };
|
|
211
215
|
|
|
212
216
|
const user = await readJsonTolerant(paths.userMemoryPath);
|
|
213
|
-
if (user) sources.
|
|
217
|
+
if (user.corrupt) sources.corruptFiles += 1;
|
|
218
|
+
if (user.value) sources.user = user.value;
|
|
214
219
|
|
|
215
220
|
for (const name of await listJsonFiles(paths.projectsDir)) {
|
|
216
221
|
const filePath = path.join(paths.projectsDir, name);
|
|
217
|
-
const content = await readJsonTolerant(filePath);
|
|
222
|
+
const { value: content, corrupt } = await readJsonTolerant(filePath);
|
|
223
|
+
if (corrupt) {
|
|
224
|
+
sources.corruptFiles += 1;
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
218
227
|
if (!content) continue;
|
|
219
228
|
if (name.endsWith('.archive.json')) {
|
|
220
229
|
for (const s of Array.isArray(content.sessions) ? content.sessions : []) {
|
|
@@ -227,7 +236,11 @@ async function collectLegacySources(paths) {
|
|
|
227
236
|
|
|
228
237
|
for (const name of await listJsonFiles(paths.sessionsDir)) {
|
|
229
238
|
const filePath = path.join(paths.sessionsDir, name);
|
|
230
|
-
const content = await readJsonTolerant(filePath);
|
|
239
|
+
const { value: content, corrupt } = await readJsonTolerant(filePath);
|
|
240
|
+
if (corrupt) {
|
|
241
|
+
sources.corruptFiles += 1;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
231
244
|
if (!content) continue;
|
|
232
245
|
if (name.endsWith('.archive.json') && Array.isArray(content.sessions)) {
|
|
233
246
|
sources.sessions.push(...content.sessions);
|
|
@@ -240,10 +253,11 @@ async function collectLegacySources(paths) {
|
|
|
240
253
|
}
|
|
241
254
|
|
|
242
255
|
function hasLegacyContent(sources) {
|
|
243
|
-
return Boolean(sources.user) || sources.projects.length > 0
|
|
256
|
+
return Boolean(sources.user) || sources.projects.length > 0
|
|
257
|
+
|| sources.sessions.length > 0 || sources.corruptFiles > 0;
|
|
244
258
|
}
|
|
245
259
|
|
|
246
|
-
async function buildRecords(projectRoot, paths, sources) {
|
|
260
|
+
async function buildRecords(projectRoot, paths, sources, skip) {
|
|
247
261
|
const config = await loadRuntimeConfig(projectRoot);
|
|
248
262
|
const ttlDays = Number.isFinite(config?.memoryV2?.episodeTtlDays) && config.memoryV2.episodeTtlDays > 0
|
|
249
263
|
? config.memoryV2.episodeTtlDays
|
|
@@ -251,9 +265,9 @@ async function buildRecords(projectRoot, paths, sources) {
|
|
|
251
265
|
const episodeTtlMs = ttlDays * DAY_MS;
|
|
252
266
|
|
|
253
267
|
const records = [];
|
|
254
|
-
if (sources.user) migrateUserMemory(sources.user, records);
|
|
255
|
-
for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records);
|
|
256
|
-
for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records);
|
|
268
|
+
if (sources.user) migrateUserMemory(sources.user, records, skip);
|
|
269
|
+
for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records, skip);
|
|
270
|
+
for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records, skip);
|
|
257
271
|
return records;
|
|
258
272
|
}
|
|
259
273
|
|
|
@@ -285,9 +299,9 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
|
|
|
285
299
|
const sources = await collectLegacySources(paths);
|
|
286
300
|
if (!hasLegacyContent(sources)) return empty;
|
|
287
301
|
|
|
288
|
-
|
|
289
|
-
const records = await buildRecords(projectRoot, paths, sources);
|
|
290
|
-
const skipped =
|
|
302
|
+
const skip = { count: 0 };
|
|
303
|
+
const records = await buildRecords(projectRoot, paths, sources, skip);
|
|
304
|
+
const skipped = skip.count + sources.corruptFiles;
|
|
291
305
|
if (dryRun) {
|
|
292
306
|
return { migrated: records.length, skipped, markerPath, backupDir: null };
|
|
293
307
|
}
|
|
@@ -307,7 +321,20 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
|
|
|
307
321
|
}
|
|
308
322
|
}
|
|
309
323
|
|
|
310
|
-
|
|
324
|
+
// Merge into the existing store under the shared lock — never clobber.
|
|
325
|
+
// records.json can already hold records when the marker is absent (a crash
|
|
326
|
+
// between saveRecords and the marker write, or a manual re-run); overwriting
|
|
327
|
+
// it would silently destroy post-migration data. source_fingerprint dedupes
|
|
328
|
+
// re-migrated records so the merge stays idempotent.
|
|
329
|
+
await mutateRecordStore(paths.memoryV2RecordsPath, (existing) => {
|
|
330
|
+
const fingerprints = new Set(
|
|
331
|
+
existing.map((r) => r.source_fingerprint).filter(Boolean),
|
|
332
|
+
);
|
|
333
|
+
const additions = records.filter(
|
|
334
|
+
(r) => !r.source_fingerprint || !fingerprints.has(r.source_fingerprint),
|
|
335
|
+
);
|
|
336
|
+
return { result: additions.length, records: [...existing, ...additions] };
|
|
337
|
+
});
|
|
311
338
|
|
|
312
339
|
const counts = {
|
|
313
340
|
total: records.length,
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// (storeV2.js) and the user store (userMemory.js) share one implementation.
|
|
5
5
|
// Writes go through fileOps.writeJson (tmp+rename).
|
|
6
6
|
|
|
7
|
-
import { readJsonIfExists, writeJson } from '../fileOps.js';
|
|
7
|
+
import { readJsonIfExists, writeJson, withFileLock } from '../fileOps.js';
|
|
8
8
|
import { normalizeRecord } from './records.js';
|
|
9
9
|
|
|
10
10
|
const SCHEMA_VERSION = 2;
|
|
@@ -79,3 +79,26 @@ export function applyRecordPatch(records, id, patch = {}) {
|
|
|
79
79
|
updated[index] = normalized;
|
|
80
80
|
return { record: normalized, records: updated };
|
|
81
81
|
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* mutateRecordStore(recordsPath, mutate) → mutate's result | null
|
|
85
|
+
* Serializes a read-modify-write cycle on the records document under the
|
|
86
|
+
* shared file lock (fileOps.withFileLock) so concurrent add/update flows —
|
|
87
|
+
* in-process or across hook processes — never lose each other's writes.
|
|
88
|
+
* `mutate(records)` returns { result, records } to persist, or null to skip
|
|
89
|
+
* the write (e.g. id not found). Throws when the lock wait expires: a
|
|
90
|
+
* silently dropped mutation would corrupt caller state worse than an error.
|
|
91
|
+
*/
|
|
92
|
+
export async function mutateRecordStore(recordsPath, mutate) {
|
|
93
|
+
const outcome = await withFileLock(recordsPath, async () => {
|
|
94
|
+
const { records } = await readRecordStore(recordsPath);
|
|
95
|
+
const mutation = await mutate(records);
|
|
96
|
+
if (!mutation) return null;
|
|
97
|
+
await writeRecordStore(recordsPath, mutation.records);
|
|
98
|
+
return mutation.result ?? null;
|
|
99
|
+
});
|
|
100
|
+
if (outcome === undefined) {
|
|
101
|
+
throw new Error(`mutateRecordStore: lock wait expired for ${recordsPath} — mutation skipped`);
|
|
102
|
+
}
|
|
103
|
+
return outcome;
|
|
104
|
+
}
|
package/src/core/memory/store.js
CHANGED
|
@@ -365,9 +365,11 @@ async function forgetMemoryItemV2(projectRoot, memoryId, runtimePaths) {
|
|
|
365
365
|
|
|
366
366
|
if (memoryId === 'user:user' || memoryId === 'user') {
|
|
367
367
|
const userRecords = await v2.queryRecords(projectRoot, { scope: 'user' });
|
|
368
|
-
|
|
368
|
+
await archiveAll(userRecords);
|
|
369
369
|
await writeJson(runtimePaths.userMemoryPath, defaultUserMemory());
|
|
370
|
-
|
|
370
|
+
// The reset above always removes user memory — `removed` reports that the
|
|
371
|
+
// forget happened, not how many v2 records were archived.
|
|
372
|
+
return { removed: true, type: 'user', path: runtimePaths.userMemoryPath };
|
|
371
373
|
}
|
|
372
374
|
|
|
373
375
|
const id = String(memoryId);
|
|
@@ -539,13 +541,28 @@ export async function runProjectHygiene(projectRoot, projectId) {
|
|
|
539
541
|
}
|
|
540
542
|
|
|
541
543
|
function patternCandidateFromRecord(record) {
|
|
544
|
+
const isLearningCandidate = record.provenance === 'learning-candidate';
|
|
542
545
|
return {
|
|
543
546
|
id: record.meta?.id ?? record.id,
|
|
544
547
|
text: record.text,
|
|
545
548
|
category: record.meta?.category ?? null,
|
|
546
549
|
detectedFrom: record.meta?.detectedFrom ?? null,
|
|
547
550
|
detectedAt: record.meta?.detectedAt ?? record.created_at ?? null,
|
|
548
|
-
status
|
|
551
|
+
// Learning candidates carry C11 status 'proposed'; the pending listing
|
|
552
|
+
// keeps the legacy 'pending' label so consumers see one shape.
|
|
553
|
+
status: record.meta?.legacyStatus
|
|
554
|
+
?? (isLearningCandidate && record.meta?.status === 'proposed' ? 'pending' : record.meta?.status)
|
|
555
|
+
?? 'pending',
|
|
556
|
+
...(isLearningCandidate ? {
|
|
557
|
+
kind: 'learning-candidate',
|
|
558
|
+
candidateStatus: record.meta?.status ?? 'proposed',
|
|
559
|
+
scope: record.meta?.scope ?? null,
|
|
560
|
+
proposalType: record.meta?.proposalType ?? null,
|
|
561
|
+
condition: record.meta?.condition ?? null,
|
|
562
|
+
proposedDelta: record.meta?.proposedDelta ?? null,
|
|
563
|
+
evidenceRefs: record.meta?.evidenceRefs ?? [],
|
|
564
|
+
conflicts: record.meta?.conflicts ?? [],
|
|
565
|
+
} : {}),
|
|
549
566
|
_recordId: record.id,
|
|
550
567
|
};
|
|
551
568
|
}
|
|
@@ -594,32 +611,51 @@ async function listPendingPatternCandidatesV2(projectRoot, projectId) {
|
|
|
594
611
|
const v2 = await storeV2();
|
|
595
612
|
return (await v2.queryRecords(projectRoot, { projectId }))
|
|
596
613
|
.filter((record) => record.type === 'derived_fact'
|
|
597
|
-
&& record.
|
|
598
|
-
&& record.
|
|
614
|
+
&& record.status !== 'archived'
|
|
615
|
+
&& (record.meta?.legacyStatus === 'pending'
|
|
616
|
+
// Learning candidates (TASK-008) surface through the same pending
|
|
617
|
+
// listing — no second store. C11 'proposed' maps to 'pending'.
|
|
618
|
+
|| (record.provenance === 'learning-candidate' && record.meta?.status === 'proposed')))
|
|
599
619
|
.map(patternCandidateFromRecord);
|
|
600
620
|
}
|
|
601
621
|
|
|
602
622
|
async function resolvePatternCandidateV2(projectRoot, projectId, candidateId, decision) {
|
|
603
623
|
const v2 = await storeV2();
|
|
604
624
|
const records = await v2.queryRecords(projectRoot, { projectId });
|
|
605
|
-
|
|
625
|
+
// Only candidate records are resolvable — matching every record by meta.id
|
|
626
|
+
// let a delta-overlay (meta.id 'ov_*') be "approved" into a project_rule or
|
|
627
|
+
// archived, corrupting the overlay store.
|
|
628
|
+
const target = records.find((record) => (
|
|
629
|
+
String(record.provenance ?? '').includes('pattern-candidate')
|
|
630
|
+
|| record.provenance === 'learning-candidate'
|
|
631
|
+
) && (record.meta?.id === candidateId || record.id === candidateId));
|
|
606
632
|
if (!target) {
|
|
607
633
|
throw new Error(`Pattern candidate not found: ${candidateId}`);
|
|
608
634
|
}
|
|
609
635
|
|
|
610
636
|
let updated;
|
|
637
|
+
const isLearningCandidate = target.provenance === 'learning-candidate';
|
|
611
638
|
if (decision === 'approve') {
|
|
612
639
|
updated = await v2.updateRecord(projectRoot, target.id, {
|
|
613
640
|
type: 'project_rule',
|
|
614
641
|
scope: 'repo',
|
|
615
642
|
confidence: 0.9,
|
|
616
643
|
provenance: `${target.provenance ?? 'pattern-candidate'};promoted-from:${target.id}`,
|
|
617
|
-
meta: {
|
|
644
|
+
meta: {
|
|
645
|
+
...target.meta,
|
|
646
|
+
legacyStatus: 'approved',
|
|
647
|
+
// Keep the C11 status field in sync for learning candidates.
|
|
648
|
+
...(isLearningCandidate ? { status: 'approved' } : {}),
|
|
649
|
+
},
|
|
618
650
|
});
|
|
619
651
|
} else {
|
|
620
652
|
updated = await v2.updateRecord(projectRoot, target.id, {
|
|
621
653
|
status: 'archived',
|
|
622
|
-
meta: {
|
|
654
|
+
meta: {
|
|
655
|
+
...target.meta,
|
|
656
|
+
legacyStatus: 'rejected',
|
|
657
|
+
...(isLearningCandidate ? { status: 'rejected' } : {}),
|
|
658
|
+
},
|
|
623
659
|
});
|
|
624
660
|
}
|
|
625
661
|
return patternCandidateFromRecord(updated ?? target);
|