mixdog 0.9.13 → 0.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/defaults/cycle3-review-prompt.md +14 -0
- package/src/defaults/memory-promote-prompt.md +9 -9
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +132 -11
- package/src/runtime/agent/orchestrator/session/manager.mjs +3 -1
- package/src/runtime/memory/index.mjs +7 -2
- package/src/runtime/memory/lib/core-memory-store.mjs +108 -4
- package/src/runtime/memory/lib/cycle-scheduler.mjs +46 -14
- package/src/runtime/memory/lib/memory-cycle1.mjs +166 -23
- package/src/runtime/memory/lib/memory-cycle2-gate.mjs +15 -8
- package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +37 -0
- package/src/runtime/memory/lib/memory-cycle2.mjs +17 -7
- package/src/runtime/memory/lib/memory-cycle3.mjs +106 -6
- package/src/runtime/memory/lib/memory.mjs +7 -1
- package/src/runtime/memory/lib/query-handlers.mjs +1 -0
- package/src/tui/dist/index.mjs +283 -28
- package/src/tui/engine/tui-steering-persist.mjs +175 -0
- package/src/tui/engine.mjs +137 -4
- package/src/ui/statusline.mjs +2 -4
package/package.json
CHANGED
|
@@ -49,6 +49,16 @@ or inconclusive, keep the CORE entry.
|
|
|
49
49
|
- `keep` — durable and already one short clause.
|
|
50
50
|
- `update` — durable but verbose or multi-sentence → rewrite as one ≤120-char clause.
|
|
51
51
|
- `merge` — duplicates another entry → fold into the survivor (same project pool).
|
|
52
|
+
- `superseded` — a NEWER active core entry directly CONTRADICTS the value/state
|
|
53
|
+
this older entry asserts (same
|
|
54
|
+
subject, changed fact: a renamed thing, a moved location, a reversed
|
|
55
|
+
preference, a superseded decision). The older entry is no longer true → retire
|
|
56
|
+
it. You MUST name the newer core id that replaces it as `<newer_id>` — it must
|
|
57
|
+
be another entry id under review that holds the current fact, and not the same
|
|
58
|
+
id. Requires clear evidence of that newer contradicting core; when the two
|
|
59
|
+
could both hold, or you cannot cite a concrete newer id → `keep`. Unlike
|
|
60
|
+
`delete`, `superseded` archives (never physically removes) and applies in
|
|
61
|
+
default mode.
|
|
52
62
|
- `delete` — records a past event, not a current rule or structure; OR merely
|
|
53
63
|
restates a rule already in **Current rules** below (rules load every session,
|
|
54
64
|
so a CORE copy is redundant); OR is sourced from code, rules files, or skill
|
|
@@ -59,6 +69,9 @@ or inconclusive, keep the CORE entry.
|
|
|
59
69
|
A verbose durable entry is always `update`, never `keep`.
|
|
60
70
|
Delete is the rarest verdict. Prefer `keep` for durable rules/preferences and
|
|
61
71
|
`update` for compression when the current behavior is still valid.
|
|
72
|
+
Use `superseded` (not `delete`) when the entry WAS a valid durable rule/fact but
|
|
73
|
+
a newer active fact has replaced it — supersession is fact-change retirement,
|
|
74
|
+
delete is not-durable-in-the-first-place removal.
|
|
62
75
|
|
|
63
76
|
## Current rules (Source Of Truth — loaded into the session every turn)
|
|
64
77
|
|
|
@@ -83,6 +96,7 @@ One line per entry id, any order:
|
|
|
83
96
|
<id>|keep
|
|
84
97
|
<id>|update|<element>|<summary>
|
|
85
98
|
<id>|merge|<target_id>|<source_ids_csv>
|
|
99
|
+
<id>|superseded|<newer_id>
|
|
86
100
|
<id>|delete
|
|
87
101
|
```
|
|
88
102
|
|
|
@@ -105,11 +105,12 @@ Archive everything else: implementation specs, code-internal constants,
|
|
|
105
105
|
measurements, resolved-bug stories, and status snapshots.
|
|
106
106
|
|
|
107
107
|
The cap is an upper bound, not a target. When `Active < cap`, seed and grow the
|
|
108
|
-
active set
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
strictly: any active entry without a concrete
|
|
108
|
+
active set with **durable** L1/L2/L3 lessons only (not task/status snapshots,
|
|
109
|
+
open issues, or in-flight work state). Prefer promoting clear, non-duplicate
|
|
110
|
+
A/B pending rows that encode lasting behavior or map anchors; transient
|
|
111
|
+
`task`/`issue` rows should archive unless they distill to a durable lesson.
|
|
112
|
+
When `Active > cap`, contract strictly: any active entry without a concrete
|
|
113
|
+
A/B reason must archive.
|
|
113
114
|
|
|
114
115
|
If useful content is buried inside work narrative, keep only the durable L2
|
|
115
116
|
behavior lesson (via `update` on an active row, or `active` for a pending row);
|
|
@@ -123,10 +124,9 @@ archive the surrounding story.
|
|
|
123
124
|
- Never merge across project_id boundaries.
|
|
124
125
|
- `element` <=100 chars. `summary` <=200 chars. No literal `|` or newlines.
|
|
125
126
|
- When a row genuinely fails the A/B bar or you are unsure it qualifies, prefer
|
|
126
|
-
`archived`.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
user-curated core still archive per Source Of Truth).
|
|
127
|
+
`archived`. Pending rows that clearly meet A/B as a durable lesson may use
|
|
128
|
+
`active`; pending task/status churn and narrative snapshots should archive
|
|
129
|
+
even under cap (Source-Of-Truth duplicates still archive per above).
|
|
130
130
|
|
|
131
131
|
## Output
|
|
132
132
|
|
|
@@ -4,10 +4,13 @@ import { join } from 'path';
|
|
|
4
4
|
import { resolvePluginData } from '../../../../shared/plugin-paths.mjs';
|
|
5
5
|
import { updateJsonAtomicSync } from '../../../../shared/atomic-file.mjs';
|
|
6
6
|
import { promptContentText, isInternalRuntimeNotificationText } from './prompt-utils.mjs';
|
|
7
|
+
import { loadSession } from '../store.mjs';
|
|
7
8
|
|
|
8
9
|
const _sessionPendingMessages = new Map();
|
|
9
10
|
const PENDING_MESSAGES_FILE = 'session-pending-messages.json';
|
|
10
11
|
const PENDING_MESSAGES_MODE = 0o600;
|
|
12
|
+
const PENDING_ORPHAN_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
13
|
+
const PENDING_ORPHAN_GRACE_MS = 60 * 60 * 1000;
|
|
11
14
|
const _pendingPersistBuffers = new Map();
|
|
12
15
|
let _pendingPersistImmediate = null;
|
|
13
16
|
|
|
@@ -19,25 +22,58 @@ function isValidPendingSessionId(sessionId) {
|
|
|
19
22
|
return typeof sessionId === 'string' && /^[A-Za-z0-9_-]+$/.test(sessionId);
|
|
20
23
|
}
|
|
21
24
|
|
|
25
|
+
function isTuiSteeringPendingKey(sessionId) {
|
|
26
|
+
return typeof sessionId === 'string' && sessionId.startsWith('tui_');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function normalizeTuiSteeringQueueEntry(entry) {
|
|
30
|
+
if (typeof entry === 'string') {
|
|
31
|
+
const text = entry.trim();
|
|
32
|
+
return text || null;
|
|
33
|
+
}
|
|
34
|
+
if (!entry || typeof entry !== 'object') return null;
|
|
35
|
+
if (typeof entry.text === 'string' && entry.text.trim()) {
|
|
36
|
+
const text = entry.text.trim();
|
|
37
|
+
const id = typeof entry.id === 'string' && entry.id.trim() ? entry.id.trim() : null;
|
|
38
|
+
return id ? { id, text } : text;
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
|
|
22
43
|
function normalizePendingStore(raw) {
|
|
23
44
|
const sessions = raw && typeof raw === 'object' && raw.sessions && typeof raw.sessions === 'object'
|
|
24
45
|
? raw.sessions
|
|
25
46
|
: {};
|
|
26
|
-
const
|
|
47
|
+
const storeUpdatedAt = Number(raw?.updatedAt) || Date.now();
|
|
48
|
+
const touchedRaw = raw && typeof raw === 'object' && raw.sessionTouchedAt && typeof raw.sessionTouchedAt === 'object'
|
|
49
|
+
? raw.sessionTouchedAt
|
|
50
|
+
: {};
|
|
51
|
+
const out = { version: 1, updatedAt: storeUpdatedAt, sessions: {}, sessionTouchedAt: {} };
|
|
27
52
|
for (const [sid, value] of Object.entries(sessions)) {
|
|
28
53
|
if (!isValidPendingSessionId(sid) || !Array.isArray(value)) continue;
|
|
29
|
-
const q =
|
|
30
|
-
.map((
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
54
|
+
const q = isTuiSteeringPendingKey(sid)
|
|
55
|
+
? value.map(normalizeTuiSteeringQueueEntry).filter(Boolean)
|
|
56
|
+
: value
|
|
57
|
+
.map((entry) => {
|
|
58
|
+
if (typeof entry === 'string') return entry;
|
|
59
|
+
if (entry && typeof entry === 'object' && typeof entry.message === 'string') return entry.message;
|
|
60
|
+
return '';
|
|
61
|
+
})
|
|
62
|
+
.filter(Boolean);
|
|
63
|
+
if (q.length > 0) {
|
|
64
|
+
out.sessions[sid] = q;
|
|
65
|
+
const touched = Number(touchedRaw[sid]);
|
|
66
|
+
out.sessionTouchedAt[sid] = Number.isFinite(touched) && touched > 0 ? touched : storeUpdatedAt;
|
|
67
|
+
}
|
|
37
68
|
}
|
|
38
69
|
return out;
|
|
39
70
|
}
|
|
40
71
|
|
|
72
|
+
function touchPendingSessionEntry(next, sessionId, now = Date.now()) {
|
|
73
|
+
if (!next.sessionTouchedAt || typeof next.sessionTouchedAt !== 'object') next.sessionTouchedAt = {};
|
|
74
|
+
next.sessionTouchedAt[sessionId] = now;
|
|
75
|
+
}
|
|
76
|
+
|
|
41
77
|
function normalizePendingMessageEntry(entry) {
|
|
42
78
|
if (typeof entry === 'string') {
|
|
43
79
|
const text = entry.trim();
|
|
@@ -86,7 +122,9 @@ function persistPendingMessages(sessionId, messages) {
|
|
|
86
122
|
const q = Array.isArray(next.sessions[sessionId]) ? next.sessions[sessionId] : [];
|
|
87
123
|
q.push(...persistedMessages);
|
|
88
124
|
next.sessions[sessionId] = q;
|
|
89
|
-
|
|
125
|
+
const now = Date.now();
|
|
126
|
+
next.updatedAt = now;
|
|
127
|
+
touchPendingSessionEntry(next, sessionId, now);
|
|
90
128
|
depth = q.length;
|
|
91
129
|
return next;
|
|
92
130
|
}, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
|
|
@@ -143,6 +181,7 @@ function drainPersistedPendingMessages(sessionId) {
|
|
|
143
181
|
drained = q.filter((m) => typeof m === 'string' && m.length > 0);
|
|
144
182
|
if (drained.length === 0) return undefined;
|
|
145
183
|
delete next.sessions[sessionId];
|
|
184
|
+
if (next.sessionTouchedAt) delete next.sessionTouchedAt[sessionId];
|
|
146
185
|
next.updatedAt = Date.now();
|
|
147
186
|
return next;
|
|
148
187
|
}, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
|
|
@@ -152,6 +191,75 @@ function drainPersistedPendingMessages(sessionId) {
|
|
|
152
191
|
return drained;
|
|
153
192
|
}
|
|
154
193
|
|
|
194
|
+
function clearPersistedPendingMessages(sessionId) {
|
|
195
|
+
if (!isValidPendingSessionId(sessionId)) return;
|
|
196
|
+
try {
|
|
197
|
+
updateJsonAtomicSync(pendingMessagesPath(), (raw) => {
|
|
198
|
+
const next = normalizePendingStore(raw);
|
|
199
|
+
if (!Object.prototype.hasOwnProperty.call(next.sessions, sessionId)) return undefined;
|
|
200
|
+
delete next.sessions[sessionId];
|
|
201
|
+
if (next.sessionTouchedAt) delete next.sessionTouchedAt[sessionId];
|
|
202
|
+
next.updatedAt = Date.now();
|
|
203
|
+
return next;
|
|
204
|
+
}, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
|
|
205
|
+
} catch (err) {
|
|
206
|
+
try { process.stderr.write(`[session] pending-message clear failed sessionId=${sessionId}: ${err?.message || err}\n`); } catch {}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function shouldEvictPendingSession(sessionId, ttlMs, entryTouchedAt, now = Date.now()) {
|
|
211
|
+
if (isTuiSteeringPendingKey(sessionId)) {
|
|
212
|
+
const entryTouch = Number(entryTouchedAt) || 0;
|
|
213
|
+
if (entryTouch <= 0) return false;
|
|
214
|
+
return (now - entryTouch) > ttlMs;
|
|
215
|
+
}
|
|
216
|
+
const session = loadSession(sessionId);
|
|
217
|
+
if (session) {
|
|
218
|
+
const touched = Math.max(
|
|
219
|
+
Number(session.updatedAt) || 0,
|
|
220
|
+
Number(session.lastHeartbeatAt) || 0,
|
|
221
|
+
Number(session.createdAt) || 0,
|
|
222
|
+
);
|
|
223
|
+
return touched > 0 && (now - touched) > ttlMs;
|
|
224
|
+
}
|
|
225
|
+
const entryTouch = Number(entryTouchedAt) || 0;
|
|
226
|
+
return entryTouch > 0 && (now - entryTouch) > PENDING_ORPHAN_GRACE_MS;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function sweepOrphanedPendingMessages({ ttlMs = PENDING_ORPHAN_TTL_MS } = {}) {
|
|
230
|
+
const now = Date.now();
|
|
231
|
+
const removed = [];
|
|
232
|
+
try {
|
|
233
|
+
updateJsonAtomicSync(pendingMessagesPath(), (raw) => {
|
|
234
|
+
const next = normalizePendingStore(raw);
|
|
235
|
+
const ids = Object.keys(next.sessions);
|
|
236
|
+
if (ids.length === 0) return undefined;
|
|
237
|
+
for (const sid of ids) {
|
|
238
|
+
const entryTouchedAt = next.sessionTouchedAt?.[sid];
|
|
239
|
+
if (shouldEvictPendingSession(sid, ttlMs, entryTouchedAt, now)) {
|
|
240
|
+
delete next.sessions[sid];
|
|
241
|
+
if (next.sessionTouchedAt) delete next.sessionTouchedAt[sid];
|
|
242
|
+
removed.push(sid);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (removed.length === 0) return undefined;
|
|
246
|
+
next.updatedAt = now;
|
|
247
|
+
return next;
|
|
248
|
+
}, { compact: true, lock: true, mode: PENDING_MESSAGES_MODE, fsync: false });
|
|
249
|
+
} catch (err) {
|
|
250
|
+
try { process.stderr.write(`[session] pending-message sweep failed: ${err?.message || err}\n`); } catch {}
|
|
251
|
+
return 0;
|
|
252
|
+
}
|
|
253
|
+
if (removed.length > 0) {
|
|
254
|
+
try {
|
|
255
|
+
process.stderr.write(
|
|
256
|
+
`[session] pending-message sweep: removed ${removed.length} stale/orphan queue(s) (ttl=${Math.round(ttlMs / 86400000)}d) (${removed.slice(0, 5).join(', ')}${removed.length > 5 ? `, +${removed.length - 5} more` : ''})\n`,
|
|
257
|
+
);
|
|
258
|
+
} catch { /* ignore */ }
|
|
259
|
+
}
|
|
260
|
+
return removed.length;
|
|
261
|
+
}
|
|
262
|
+
|
|
155
263
|
function modelVisiblePendingMessages(messages) {
|
|
156
264
|
return (Array.isArray(messages) ? messages : [])
|
|
157
265
|
.map(pendingMessageQueueEntry)
|
|
@@ -229,7 +337,20 @@ export function drainPendingMessages(sessionId) {
|
|
|
229
337
|
|
|
230
338
|
// Cleanup hook for closeSession — drop the in-memory queue and buffered-persist
|
|
231
339
|
// entry so both Maps do not accumulate one entry per closed session.
|
|
232
|
-
export function _dropPendingMessageState(id) {
|
|
340
|
+
export function _dropPendingMessageState(id, { clearPersisted = true } = {}) {
|
|
341
|
+
if (!clearPersisted) {
|
|
342
|
+
const buffered = _pendingPersistBuffers.get(id);
|
|
343
|
+
if (buffered?.length) {
|
|
344
|
+
try { persistPendingMessages(id, buffered); } catch { /* ignore */ }
|
|
345
|
+
}
|
|
346
|
+
}
|
|
233
347
|
try { _sessionPendingMessages.delete(id); } catch { /* ignore */ }
|
|
234
348
|
try { _pendingPersistBuffers.delete(id); } catch { /* ignore */ }
|
|
349
|
+
if (clearPersisted) {
|
|
350
|
+
try { clearPersistedPendingMessages(id); } catch { /* ignore */ }
|
|
351
|
+
}
|
|
235
352
|
}
|
|
353
|
+
|
|
354
|
+
setImmediate(() => {
|
|
355
|
+
try { sweepOrphanedPendingMessages(); } catch { /* ignore */ }
|
|
356
|
+
});
|
|
@@ -72,6 +72,7 @@ import {
|
|
|
72
72
|
enqueuePendingMessage,
|
|
73
73
|
drainPendingMessages,
|
|
74
74
|
_dropPendingMessageState,
|
|
75
|
+
sweepOrphanedPendingMessages,
|
|
75
76
|
} from './manager/pending-messages.mjs';
|
|
76
77
|
import {
|
|
77
78
|
bumpUsageMetricsTurnId,
|
|
@@ -2082,7 +2083,7 @@ export function closeSession(id, reason = 'manual', opts = {}) {
|
|
|
2082
2083
|
// Drop the in-memory pending-message queue and any buffered-persist entry
|
|
2083
2084
|
// for this session — otherwise both Maps accumulate one entry per closed
|
|
2084
2085
|
// session for the life of the mcp-server.
|
|
2085
|
-
_dropPendingMessageState(id);
|
|
2086
|
+
_dropPendingMessageState(id, { clearPersisted: tombstone });
|
|
2086
2087
|
// 4. Defer runtime map clear to next tick so any settling askSession can
|
|
2087
2088
|
// observe `closed=true` / bumped generation before we yank the entry.
|
|
2088
2089
|
// Disk tombstone remains — that's what blocks resurrection.
|
|
@@ -2222,6 +2223,7 @@ function hasActiveRuntimeWork() {
|
|
|
2222
2223
|
|
|
2223
2224
|
function _runCleanupCycle() {
|
|
2224
2225
|
if (hasActiveRuntimeWork()) return;
|
|
2226
|
+
sweepOrphanedPendingMessages();
|
|
2225
2227
|
sweepIdleSessions({ includeTombstones: true });
|
|
2226
2228
|
}
|
|
2227
2229
|
|
|
@@ -580,6 +580,7 @@ const _cycleScheduler = createCycleScheduler({
|
|
|
580
580
|
const _startCycle1Run = _cycleScheduler.startCycle1Run
|
|
581
581
|
const _awaitCycle1Run = _cycleScheduler.awaitCycle1Run
|
|
582
582
|
const _finalizeCycle2Run = _cycleScheduler.finalizeCycle2Run
|
|
583
|
+
const _finalizeCycle3Run = _cycleScheduler.finalizeCycle3Run
|
|
583
584
|
|
|
584
585
|
// Transcript watcher lifecycle stays in the facade (owns _transcriptIngest);
|
|
585
586
|
// the cycle tick loop start/stop is delegated to the scheduler.
|
|
@@ -769,6 +770,10 @@ async function _handleMemCycle3(args, config, signal) {
|
|
|
769
770
|
}
|
|
770
771
|
const result = await runCycle3(db, config || {}, DATA_DIR, c3Options)
|
|
771
772
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
773
|
+
// Stamp cycle3 success: the MCP path bypasses the scheduler's coalesced
|
|
774
|
+
// onCoalescedSuccess, so without this last_success_at stayed 0 despite
|
|
775
|
+
// successful runs.
|
|
776
|
+
await _finalizeCycle3Run(result)
|
|
772
777
|
const parts = ['reviewed', 'kept', 'updated', 'merged', 'deleted']
|
|
773
778
|
.map(k => `${k}=${result?.[k] || 0}`)
|
|
774
779
|
if (result?.proposed) {
|
|
@@ -1523,11 +1528,11 @@ async function buildSessionCoreMemoryPayload(cwd) {
|
|
|
1523
1528
|
ORDER BY score DESC, last_seen_at DESC
|
|
1524
1529
|
`, projectId !== null ? [projectId] : [])).rows
|
|
1525
1530
|
const commonRows = (await db.query(
|
|
1526
|
-
`SELECT summary FROM core_entries WHERE project_id IS NULL ORDER BY id ASC`
|
|
1531
|
+
`SELECT summary FROM core_entries WHERE project_id IS NULL AND (status IS NULL OR status = 'active') ORDER BY id ASC`
|
|
1527
1532
|
)).rows
|
|
1528
1533
|
const scopedRows = projectId !== null
|
|
1529
1534
|
? (await db.query(
|
|
1530
|
-
`SELECT summary FROM core_entries WHERE project_id = $1 ORDER BY id ASC`,
|
|
1535
|
+
`SELECT summary FROM core_entries WHERE project_id = $1 AND (status IS NULL OR status = 'active') ORDER BY id ASC`,
|
|
1531
1536
|
[projectId]
|
|
1532
1537
|
)).rows
|
|
1533
1538
|
: []
|
|
@@ -63,7 +63,9 @@ function throwIfAborted(signal) {
|
|
|
63
63
|
async function _backfillNullEmbeddings(db, options = {}) {
|
|
64
64
|
const signal = options?.signal
|
|
65
65
|
throwIfAborted(signal)
|
|
66
|
-
|
|
66
|
+
// Only refill live cores — archiveCore intentionally nulls the embedding to
|
|
67
|
+
// drop archived rows from recall; refilling them would resurrect them.
|
|
68
|
+
const r = await db.query(`SELECT id, element, summary FROM core_entries WHERE embedding IS NULL AND (status IS NULL OR status = 'active')`)
|
|
67
69
|
if (r.rows.length === 0) return 0
|
|
68
70
|
let filled = 0
|
|
69
71
|
for (const row of r.rows) {
|
|
@@ -144,6 +146,7 @@ async function _findTopKCore(db, projectId, embedding, excludeId, { forUpdate =
|
|
|
144
146
|
FROM core_entries
|
|
145
147
|
WHERE embedding IS NOT NULL
|
|
146
148
|
AND project_id IS NOT DISTINCT FROM $2
|
|
149
|
+
AND (status IS NULL OR status = 'active')
|
|
147
150
|
${exclusion}
|
|
148
151
|
ORDER BY embedding <=> $1::halfvec
|
|
149
152
|
LIMIT ${CORE_DEDUP_TOP_K}${forUpdate ? ' FOR UPDATE' : ''}`
|
|
@@ -188,15 +191,18 @@ async function _llmJudgeMerge(existing, incoming) {
|
|
|
188
191
|
export async function listCore(dataDir, projectId = null) {
|
|
189
192
|
const db = _getDb(dataDir)
|
|
190
193
|
const cols = `id, element, summary, category, project_id, created_at, updated_at`
|
|
194
|
+
// Only live cores enter recall/review — archived (superseded) rows are
|
|
195
|
+
// retired but retained for audit. Legacy NULL status = active.
|
|
196
|
+
const live = `(status IS NULL OR status = 'active')`
|
|
191
197
|
if (projectId === '*') {
|
|
192
|
-
const r = await db.query(`SELECT ${cols} FROM core_entries ORDER BY project_id NULLS FIRST, id ASC`)
|
|
198
|
+
const r = await db.query(`SELECT ${cols} FROM core_entries WHERE ${live} ORDER BY project_id NULLS FIRST, id ASC`)
|
|
193
199
|
return r.rows
|
|
194
200
|
}
|
|
195
201
|
if (projectId === null) {
|
|
196
|
-
const r = await db.query(`SELECT ${cols} FROM core_entries WHERE project_id IS NULL ORDER BY id ASC`)
|
|
202
|
+
const r = await db.query(`SELECT ${cols} FROM core_entries WHERE project_id IS NULL AND ${live} ORDER BY id ASC`)
|
|
197
203
|
return r.rows
|
|
198
204
|
}
|
|
199
|
-
const r = await db.query(`SELECT ${cols} FROM core_entries WHERE project_id = $1 ORDER BY id ASC`, [projectId])
|
|
205
|
+
const r = await db.query(`SELECT ${cols} FROM core_entries WHERE project_id = $1 AND ${live} ORDER BY id ASC`, [projectId])
|
|
200
206
|
return r.rows
|
|
201
207
|
}
|
|
202
208
|
|
|
@@ -242,14 +248,37 @@ export async function addCore(dataDir, { element, summary, category }, projectId
|
|
|
242
248
|
}
|
|
243
249
|
let r
|
|
244
250
|
try {
|
|
251
|
+
// Savepoint so a unique-collision doesn't abort the whole tx — we recover
|
|
252
|
+
// by reviving an archived row on the same connection below.
|
|
253
|
+
await client.query('SAVEPOINT ins')
|
|
245
254
|
r = await client.query(
|
|
246
255
|
`INSERT INTO core_entries(element, summary, category, project_id, embedding, created_at, updated_at)
|
|
247
256
|
VALUES ($1, $2, $3, $4, $5::halfvec, $6, $7)
|
|
248
257
|
RETURNING id, element, summary, category, project_id, created_at, updated_at`,
|
|
249
258
|
[el, sm, cat, projectId, embedding ? embeddingToSql(embedding) : null, now, now],
|
|
250
259
|
)
|
|
260
|
+
await client.query('RELEASE SAVEPOINT ins')
|
|
251
261
|
} catch (err) {
|
|
252
262
|
if (err.code === '23505') {
|
|
263
|
+
await client.query('ROLLBACK TO SAVEPOINT ins')
|
|
264
|
+
// Unique (project_id, element) collision. If the colliding row is an
|
|
265
|
+
// archived (superseded) row, the fact is being re-asserted → revive it
|
|
266
|
+
// in place: flip back to active, clear archived_at, overwrite content.
|
|
267
|
+
// Avoids a partial-index migration. An active collision is a genuine
|
|
268
|
+
// duplicate → surface the error.
|
|
269
|
+
const revived = await client.query(
|
|
270
|
+
`UPDATE core_entries
|
|
271
|
+
SET summary = $1, category = $2, embedding = $3::halfvec,
|
|
272
|
+
status = 'active', archived_at = NULL, updated_at = $4
|
|
273
|
+
WHERE project_id IS NOT DISTINCT FROM $5 AND element = $6
|
|
274
|
+
AND status = 'archived'
|
|
275
|
+
RETURNING id, element, summary, category, project_id, created_at, updated_at`,
|
|
276
|
+
[sm, cat, embedding ? embeddingToSql(embedding) : null, now, projectId, el],
|
|
277
|
+
)
|
|
278
|
+
if (revived.rows.length > 0) {
|
|
279
|
+
await client.query('COMMIT')
|
|
280
|
+
return { ...revived.rows[0], revived_from_archived: true }
|
|
281
|
+
}
|
|
253
282
|
throw new Error(`core entry already exists: project=${projectId ?? 'COMMON'} element=${JSON.stringify(el.slice(0, 200))}`)
|
|
254
283
|
}
|
|
255
284
|
throw err
|
|
@@ -339,6 +368,80 @@ export async function deleteCore(dataDir, id) {
|
|
|
339
368
|
return r.rows[0]
|
|
340
369
|
}
|
|
341
370
|
|
|
371
|
+
// Archive (retire without physical removal) a core entry whose fact was
|
|
372
|
+
// superseded by a newer active fact. Non-destructive: flips status to
|
|
373
|
+
// 'archived' + stamps archived_at so the row can be recovered/audited, and
|
|
374
|
+
// drops it from the recall/review pool. Safe-by-default: unlike deleteCore this
|
|
375
|
+
// is reversible and runs in conservative mode. Relies on the nullable status/
|
|
376
|
+
// archived_at columns added in ensureCurrentSchemaExtensions (no migration
|
|
377
|
+
// beyond those additive columns).
|
|
378
|
+
//
|
|
379
|
+
// Takes the same core:${project} advisory lock as addCore/editCore and
|
|
380
|
+
// re-checks the row inside it, so a concurrent addCore merge/overwrite that
|
|
381
|
+
// changed the fact after cycle3 read it is NOT clobbered. `expect` carries the
|
|
382
|
+
// element/summary cycle3 reviewed; if the live row drifted from it the archive
|
|
383
|
+
// is skipped (returns { skipped:true, reason:'content drift' }).
|
|
384
|
+
export async function archiveCore(dataDir, id, expect = null) {
|
|
385
|
+
const numId = Number(id)
|
|
386
|
+
if (!Number.isInteger(numId) || numId <= 0) throw new Error('integer id > 0 required')
|
|
387
|
+
const db = _getDb(dataDir)
|
|
388
|
+
const now = Date.now()
|
|
389
|
+
const client = await checkedConnect(db._pool, 'memory')
|
|
390
|
+
try {
|
|
391
|
+
await client.query('BEGIN')
|
|
392
|
+
await client.query(`SET LOCAL lock_timeout = '5s'`)
|
|
393
|
+
// Lock ORDER must match addCore/editCore: advisory (pool) FIRST, then the
|
|
394
|
+
// row FOR UPDATE — otherwise same-row concurrency deadlocks. The advisory
|
|
395
|
+
// key needs project_id, so read it first WITHOUT a row lock (plain SELECT,
|
|
396
|
+
// no FOR UPDATE → acquires no row lock, can't invert ordering), take the
|
|
397
|
+
// pool advisory lock, THEN FOR UPDATE the row and re-validate under it.
|
|
398
|
+
const pre = (await client.query(
|
|
399
|
+
`SELECT project_id FROM core_entries WHERE id = $1`,
|
|
400
|
+
[numId],
|
|
401
|
+
)).rows[0]
|
|
402
|
+
if (!pre) throw new Error(`no entry with id=${numId}`)
|
|
403
|
+
const poolKey = `core:${pre.project_id == null ? 'COMMON' : pre.project_id}`
|
|
404
|
+
await client.query(`SELECT pg_advisory_xact_lock(hashtext($1))`, [poolKey])
|
|
405
|
+
// Now take the row lock under the pool lock and re-read — a concurrent
|
|
406
|
+
// addCore holding the same advisory lock may have merged/overwritten or
|
|
407
|
+
// moved the row's project_id between our pre-read and the lock.
|
|
408
|
+
const locked = (await client.query(
|
|
409
|
+
`SELECT id, element, summary, project_id, status FROM core_entries WHERE id = $1 FOR UPDATE`,
|
|
410
|
+
[numId],
|
|
411
|
+
)).rows[0]
|
|
412
|
+
if (!locked || !(locked.status == null || locked.status === 'active')) {
|
|
413
|
+
await client.query('ROLLBACK')
|
|
414
|
+
return { id: numId, skipped: true, reason: 'concurrently archived/removed' }
|
|
415
|
+
}
|
|
416
|
+
// If project_id moved pools between pre-read and lock, our advisory lock is
|
|
417
|
+
// on the wrong pool → bail rather than archive under a mismatched lock.
|
|
418
|
+
if ((locked.project_id ?? null) !== (pre.project_id ?? null)) {
|
|
419
|
+
await client.query('ROLLBACK')
|
|
420
|
+
return { id: numId, skipped: true, reason: 'pool changed under lock' }
|
|
421
|
+
}
|
|
422
|
+
if (expect && (String(expect.element ?? '') !== String(locked.element ?? '') ||
|
|
423
|
+
String(expect.summary ?? '') !== String(locked.summary ?? ''))) {
|
|
424
|
+
await client.query('ROLLBACK')
|
|
425
|
+
return { id: numId, skipped: true, reason: 'content drift' }
|
|
426
|
+
}
|
|
427
|
+
const r = await client.query(
|
|
428
|
+
`UPDATE core_entries
|
|
429
|
+
SET status = 'archived', archived_at = $1, updated_at = $2, embedding = NULL
|
|
430
|
+
WHERE id = $3 AND (status IS NULL OR status = 'active')
|
|
431
|
+
RETURNING *`,
|
|
432
|
+
[now, now, numId],
|
|
433
|
+
)
|
|
434
|
+
await client.query('COMMIT')
|
|
435
|
+
if (r.rows.length === 0) return { id: numId, skipped: true, reason: 'concurrently archived/removed' }
|
|
436
|
+
return r.rows[0]
|
|
437
|
+
} catch (err) {
|
|
438
|
+
try { await client.query('ROLLBACK') } catch {}
|
|
439
|
+
throw err
|
|
440
|
+
} finally {
|
|
441
|
+
client.release()
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
342
445
|
// ─── Core-candidate promotion pipeline (proposal mode) ───────────────────────
|
|
343
446
|
//
|
|
344
447
|
// nominateCoreCandidates flags strong active entries as core-memory
|
|
@@ -446,6 +549,7 @@ export async function nominateCoreCandidates(dataDir, options = {}) {
|
|
|
446
549
|
SELECT inner_c.embedding
|
|
447
550
|
FROM core_entries inner_c
|
|
448
551
|
WHERE inner_c.embedding IS NOT NULL
|
|
552
|
+
AND (inner_c.status IS NULL OR inner_c.status = 'active')
|
|
449
553
|
AND (inner_c.project_id IS NULL OR inner_c.project_id IS NOT DISTINCT FROM e.project_id)
|
|
450
554
|
ORDER BY inner_c.embedding <=> e.embedding
|
|
451
555
|
LIMIT 1
|
|
@@ -30,6 +30,8 @@ const CYCLE1_AUTO_RESTART_COOLDOWN_MS = 5 * 60_000
|
|
|
30
30
|
const BACKLOG_WARN_COOLDOWN_MS = 10 * 60_000
|
|
31
31
|
const BACKLOG_WARN_PENDING = 500
|
|
32
32
|
const BACKLOG_WARN_FAILURES = 5
|
|
33
|
+
// Max back-to-back cycle2 passes per scheduled slot (config: cycle2.catchup_passes).
|
|
34
|
+
const CYCLE2_CATCHUP_PASSES = 4
|
|
33
35
|
|
|
34
36
|
export function createCycleScheduler(deps) {
|
|
35
37
|
const {
|
|
@@ -255,20 +257,41 @@ export function createCycleScheduler(deps) {
|
|
|
255
257
|
_cycle2InFlight = true
|
|
256
258
|
markCycleRunning('cycle2')
|
|
257
259
|
try {
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
c2Options = {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
260
|
+
// Catch-up drain: one scheduled slot may run several back-to-back
|
|
261
|
+
// passes while pending roots remain, so a backlog above the batch
|
|
262
|
+
// size drains in one interval instead of one batch per hour.
|
|
263
|
+
const drainPasses = Math.max(1, Number(config?.catchup_passes ?? CYCLE2_CATCHUP_PASSES))
|
|
264
|
+
for (let pass = 0; pass < drainPasses; pass++) {
|
|
265
|
+
let c2Options = {
|
|
266
|
+
coalescedRetry: true,
|
|
267
|
+
onCoalescedSuccess: _finalizeCycle2Run,
|
|
268
|
+
}
|
|
269
|
+
if (typeof c2Options?.callLlm !== 'function') {
|
|
270
|
+
c2Options = { ...c2Options, callLlm: getCycle2CallLlm() }
|
|
271
|
+
}
|
|
272
|
+
const result = await runCycle2(getDb(), config, c2Options, dataDir)
|
|
273
|
+
if (result?.skippedInFlight) {
|
|
274
|
+
scheduleScheduledCycle2(config, signature, attempt + 1)
|
|
275
|
+
break
|
|
276
|
+
}
|
|
277
|
+
if (result?.coalescedRetryNoop) {
|
|
278
|
+
log('[cycle2] scheduled queue noop\n')
|
|
279
|
+
break
|
|
280
|
+
}
|
|
281
|
+
if (result?.ok === false) {
|
|
282
|
+
await _finalizeCycle2Run(result)
|
|
283
|
+
break
|
|
284
|
+
}
|
|
285
|
+
// A gate parse/coverage failure marks the run failed even with
|
|
286
|
+
// ok=true; never chain passes on a failing gate.
|
|
287
|
+
if (result?.gate_failed === true) break
|
|
288
|
+
if (pass + 1 >= drainPasses) break
|
|
289
|
+
const pendingRes = await getDb().query(
|
|
290
|
+
`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`,
|
|
291
|
+
)
|
|
292
|
+
const pendingLeft = Number(pendingRes?.rows?.[0]?.c ?? 0)
|
|
293
|
+
if (pendingLeft <= 0) break
|
|
294
|
+
log(`[cycle2] catch-up pass ${pass + 2}/${drainPasses}: pending=${pendingLeft}\n`)
|
|
272
295
|
}
|
|
273
296
|
} catch (err) {
|
|
274
297
|
log(`[cycle2] scheduled queue failed: ${err?.message || err}\n`)
|
|
@@ -489,6 +512,15 @@ export function createCycleScheduler(deps) {
|
|
|
489
512
|
startCycle1Run: _startCycle1Run,
|
|
490
513
|
awaitCycle1Run: _awaitCycle1Run,
|
|
491
514
|
finalizeCycle2Run: _finalizeCycle2Run,
|
|
515
|
+
// cycle3 success recorder: MCP-driven runs go through runCycle3 directly
|
|
516
|
+
// (no coalesced onCoalescedSuccess), so callers stamp success here to keep
|
|
517
|
+
// cycles.cycle3.last_success_at honest. Non-mutating/failed runs skipped.
|
|
518
|
+
finalizeCycle3Run: async (result) => {
|
|
519
|
+
if (!result || result.skippedInFlight || result.coalescedRetryNoop) return
|
|
520
|
+
if (result.error) { markCycleDone('cycle3', false, result.error); return }
|
|
521
|
+
await setCycleLastRun('cycle3', Date.now())
|
|
522
|
+
markCycleDone('cycle3', true)
|
|
523
|
+
},
|
|
492
524
|
periodicCycle1Config,
|
|
493
525
|
// lifecycle
|
|
494
526
|
startCycles,
|