mixdog 0.9.19 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +100 -34
- package/package.json +3 -2
- package/scripts/build-tui.mjs +6 -0
- package/scripts/hook-bus-test.mjs +8 -0
- package/scripts/log-writer-guard-smoke.mjs +131 -0
- package/scripts/reactive-compact-persist-smoke.mjs +22 -6
- package/scripts/recall-bench-cases.json +0 -1
- package/scripts/recall-quality-cases.json +1 -2
- package/scripts/session-ingest-compaction-smoke.mjs +241 -0
- package/scripts/tool-smoke.mjs +150 -45
- package/src/defaults/skills/setup/SKILL.md +327 -0
- package/src/help.mjs +2 -5
- package/src/lib/mixdog-debug.cjs +13 -0
- package/src/mixdog-session-runtime.mjs +7 -3328
- package/src/runtime/agent/orchestrator/context/collect.mjs +33 -9
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +34 -2
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +67 -10
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +40 -3
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +73 -3
- package/src/runtime/agent/orchestrator/session/agent-loop.mjs +663 -0
- package/src/runtime/agent/orchestrator/session/compact/engine.mjs +6 -0
- package/src/runtime/agent/orchestrator/session/eager-dispatch.mjs +153 -0
- package/src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs +49 -0
- package/src/runtime/agent/orchestrator/session/loop/stored-tool-args.mjs +9 -1
- package/src/runtime/agent/orchestrator/session/loop.mjs +8 -1860
- package/src/runtime/agent/orchestrator/session/manager/agent-runtime-singleton.mjs +29 -0
- package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +686 -0
- package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +60 -12
- package/src/runtime/agent/orchestrator/session/manager/env-utils.mjs +8 -0
- package/src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs +159 -0
- package/src/runtime/agent/orchestrator/session/manager/message-sanitize.mjs +143 -0
- package/src/runtime/agent/orchestrator/session/manager/prefetch-bridge.mjs +268 -0
- package/src/runtime/agent/orchestrator/session/manager/provider-cache-key.mjs +22 -0
- package/src/runtime/agent/orchestrator/session/manager/runtime-loaders.mjs +26 -0
- package/src/runtime/agent/orchestrator/session/manager/session-close.mjs +124 -0
- package/src/runtime/agent/orchestrator/session/manager/session-crud.mjs +258 -0
- package/src/runtime/agent/orchestrator/session/manager/session-errors.mjs +20 -0
- package/src/runtime/agent/orchestrator/session/manager/session-id.mjs +9 -0
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +475 -0
- package/src/runtime/agent/orchestrator/session/manager/session-lock.mjs +23 -0
- package/src/runtime/agent/orchestrator/session/manager.mjs +88 -2285
- package/src/runtime/agent/orchestrator/session/pre-send-compact.mjs +440 -0
- package/src/runtime/agent/orchestrator/session/send-with-recovery.mjs +153 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +4 -1
- package/src/runtime/agent/orchestrator/session/tool-batch.mjs +642 -0
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +23 -3
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-paths.mjs +108 -2
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +15 -3
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +7 -0
- package/src/runtime/agent/orchestrator/tools/builtin/shell-runtime.mjs +24 -2
- package/src/runtime/agent/orchestrator/tools/patch/orchestrator.mjs +2 -2
- package/src/runtime/agent/orchestrator/tools/patch/parsing.mjs +72 -8
- package/src/runtime/agent/orchestrator/tools/shell-policy-danger-target.mjs +5 -1
- package/src/runtime/channels/index.mjs +6 -2183
- package/src/runtime/channels/lib/inbound-handler.mjs +328 -0
- package/src/runtime/channels/lib/interaction-handlers.mjs +260 -0
- package/src/runtime/channels/lib/network-retry.mjs +23 -0
- package/src/runtime/channels/lib/owned-runtime.mjs +627 -0
- package/src/runtime/channels/lib/runtime-paths.mjs +20 -9
- package/src/runtime/channels/lib/transcript-binding.mjs +336 -0
- package/src/runtime/channels/lib/voice-runtime-fetcher.mjs +39 -2
- package/src/runtime/channels/lib/worker-bootstrap.mjs +97 -0
- package/src/runtime/channels/lib/worker-ipc.mjs +201 -0
- package/src/runtime/channels/lib/worker-main.mjs +777 -0
- package/src/runtime/memory/index.mjs +163 -1725
- package/src/runtime/memory/lib/embedding-provider.mjs +4 -2
- package/src/runtime/memory/lib/embedding-worker.mjs +47 -6
- package/src/runtime/memory/lib/http-router.mjs +811 -0
- package/src/runtime/memory/lib/memory-action-handlers.mjs +901 -0
- package/src/runtime/memory/lib/memory-cycle2-gate.mjs +6 -0
- package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +46 -9
- package/src/runtime/memory/lib/memory-cycle2.mjs +87 -11
- package/src/runtime/memory/lib/memory-embed.mjs +28 -7
- package/src/runtime/memory/lib/memory-recall-store.mjs +4 -4
- package/src/runtime/memory/lib/memory.mjs +39 -0
- package/src/runtime/memory/lib/query-handlers.mjs +2 -2
- package/src/runtime/memory/lib/session-ingest-runtime.mjs +401 -0
- package/src/runtime/shared/atomic-file.mjs +138 -80
- package/src/runtime/shared/child-guardian.mjs +61 -3
- package/src/session-runtime/boot-profile.mjs +36 -0
- package/src/session-runtime/channel-config-api.mjs +70 -0
- package/src/session-runtime/context-status.mjs +181 -0
- package/src/session-runtime/env.mjs +17 -0
- package/src/session-runtime/lifecycle-api.mjs +242 -0
- package/src/session-runtime/model-route-api.mjs +198 -0
- package/src/session-runtime/provider-auth-api.mjs +135 -0
- package/src/session-runtime/resource-api.mjs +282 -0
- package/src/session-runtime/runtime-core.mjs +2104 -0
- package/src/session-runtime/session-turn-api.mjs +274 -0
- package/src/session-runtime/tool-catalog.mjs +18 -264
- package/src/session-runtime/tool-defs.mjs +2 -2
- package/src/session-runtime/workflow-agents-api.mjs +238 -0
- package/src/standalone/agent-tool.mjs +2 -2
- package/src/standalone/channel-worker.mjs +67 -7
- package/src/standalone/folder-dialog.mjs +4 -1
- package/src/standalone/memory-runtime-proxy.mjs +154 -17
- package/src/standalone/seeds.mjs +28 -1
- package/src/tui/App.jsx +40 -28
- package/src/tui/app/core-memory-picker.mjs +1 -1
- package/src/tui/app/doctor.mjs +175 -0
- package/src/tui/app/slash-commands.mjs +1 -0
- package/src/tui/app/slash-dispatch.mjs +9 -0
- package/src/tui/app/use-mouse-input.mjs +6 -0
- package/src/tui/app/use-transcript-scroll.mjs +77 -3
- package/src/tui/dist/index.mjs +2851 -2162
- package/src/tui/engine/context-state.mjs +145 -0
- package/src/tui/engine/prompt-history.mjs +27 -0
- package/src/tui/engine/session-api-ext.mjs +478 -0
- package/src/tui/engine/session-api.mjs +564 -0
- package/src/tui/engine/session-flow.mjs +485 -0
- package/src/tui/engine/turn.mjs +1078 -0
- package/src/tui/engine.mjs +68 -2620
- package/src/tui/index.jsx +7 -0
- package/vendor/ink/build/ink.js +16 -1
- package/vendor/ink/build/output.js +30 -4
- package/vendor/ink/build/render.js +5 -0
- package/src/workflows/sequential/WORKFLOW.md +0 -51
|
@@ -83,6 +83,9 @@ import { init as initKoMorph } from './lib/ko-morph.mjs'
|
|
|
83
83
|
import { createCycleLlmAdapters } from './lib/cycle-llm-adapters.mjs'
|
|
84
84
|
import { createCycleScheduler } from './lib/cycle-scheduler.mjs'
|
|
85
85
|
import { createQueryHandlers } from './lib/query-handlers.mjs'
|
|
86
|
+
import { createSessionIngestRuntime } from './lib/session-ingest-runtime.mjs'
|
|
87
|
+
import { createMemoryActionHandlers } from './lib/memory-action-handlers.mjs'
|
|
88
|
+
import { createHttpRouter } from './lib/http-router.mjs'
|
|
86
89
|
import {
|
|
87
90
|
readMainConfig,
|
|
88
91
|
readRecapEnabled,
|
|
@@ -161,6 +164,88 @@ function touchDaemonIdleTimer(reason = 'activity') {
|
|
|
161
164
|
_idleShutdownTimer.unref?.()
|
|
162
165
|
}
|
|
163
166
|
|
|
167
|
+
// ── Connected-client tracking + prompt shutdown ───────────────────────────
|
|
168
|
+
// The daemon is shared by multiple proxy clients (TUI host + channels worker,
|
|
169
|
+
// potentially several sessions). Each client registers/deregisters over HTTP
|
|
170
|
+
// (see /client/register, /client/deregister in lib/http-router.mjs). When the
|
|
171
|
+
// last client goes away we arm a short grace timer (default 10s) so a quick
|
|
172
|
+
// reconnect keeps the daemon warm, but an actually-closed CLI reaps the daemon
|
|
173
|
+
// in seconds instead of waiting out the 10-minute idle TTL (kept as backstop).
|
|
174
|
+
const MEMORY_CLIENT_GRACE_MS = Math.max(0, Number(process.env.MIXDOG_MEMORY_CLIENT_GRACE_MS) || 10_000)
|
|
175
|
+
const _connectedClients = new Map() // clientPid -> lastSeenMs
|
|
176
|
+
let _everHadClient = false
|
|
177
|
+
let _clientGraceTimer = null
|
|
178
|
+
let _clientSweepTimer = null
|
|
179
|
+
|
|
180
|
+
function _clientShutdownEnabled() {
|
|
181
|
+
return MEMORY_DAEMON_MODE && MEMORY_CLIENT_GRACE_MS > 0
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function pruneDeadClients() {
|
|
185
|
+
for (const pid of [..._connectedClients.keys()]) {
|
|
186
|
+
if (!_isPidAliveLocal(pid)) _connectedClients.delete(pid)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function cancelClientGrace() {
|
|
191
|
+
if (_clientGraceTimer) {
|
|
192
|
+
try { clearTimeout(_clientGraceTimer) } catch {}
|
|
193
|
+
_clientGraceTimer = null
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function armClientGrace(reason = 'last client gone') {
|
|
198
|
+
if (!_clientShutdownEnabled() || _clientGraceTimer) return
|
|
199
|
+
_clientGraceTimer = setTimeout(() => {
|
|
200
|
+
_clientGraceTimer = null
|
|
201
|
+
pruneDeadClients()
|
|
202
|
+
if (_connectedClients.size > 0) return
|
|
203
|
+
__mixdogMemoryLog(`[memory-service] daemon client grace elapsed (${reason}); shutting down\n`)
|
|
204
|
+
stop()
|
|
205
|
+
.then(() => process.exit(0))
|
|
206
|
+
.catch((e) => {
|
|
207
|
+
__mixdogMemoryLog(`[memory-service] daemon client-grace shutdown failed: ${e?.message || e}\n`)
|
|
208
|
+
process.exit(1)
|
|
209
|
+
})
|
|
210
|
+
}, MEMORY_CLIENT_GRACE_MS)
|
|
211
|
+
_clientGraceTimer.unref?.()
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function startClientSweep() {
|
|
215
|
+
if (_clientSweepTimer || !_clientShutdownEnabled()) return
|
|
216
|
+
// Reap clients that died without deregistering so a crashed CLI still frees
|
|
217
|
+
// the daemon in grace-scale time rather than waiting for the idle TTL.
|
|
218
|
+
const interval = Math.max(1000, Math.min(MEMORY_CLIENT_GRACE_MS, 5000))
|
|
219
|
+
_clientSweepTimer = setInterval(() => {
|
|
220
|
+
pruneDeadClients()
|
|
221
|
+
if (_everHadClient && _connectedClients.size === 0) armClientGrace('all clients gone (sweep)')
|
|
222
|
+
}, interval)
|
|
223
|
+
_clientSweepTimer.unref?.()
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function registerClient(clientPid) {
|
|
227
|
+
const pid = parsePositivePid(clientPid)
|
|
228
|
+
if (!pid) return true
|
|
229
|
+
// Reject registration once shutdown has begun. stop() sets _stopPromise
|
|
230
|
+
// synchronously (before its first await) and, in daemon mode, always ends
|
|
231
|
+
// in process.exit — so a daemon that is draining will never revive. Signal
|
|
232
|
+
// the proxy (via a distinct 503) to respawn a fresh daemon instead of
|
|
233
|
+
// binding to this dying one, which would fail the subsequent /api/tool.
|
|
234
|
+
if (_stopPromise) return false
|
|
235
|
+
_connectedClients.set(pid, Date.now())
|
|
236
|
+
_everHadClient = true
|
|
237
|
+
cancelClientGrace()
|
|
238
|
+
startClientSweep()
|
|
239
|
+
return true
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function deregisterClient(clientPid) {
|
|
243
|
+
const pid = parsePositivePid(clientPid)
|
|
244
|
+
if (pid) _connectedClients.delete(pid)
|
|
245
|
+
pruneDeadClients()
|
|
246
|
+
if (_everHadClient && _connectedClients.size === 0) armClientGrace('last client deregistered')
|
|
247
|
+
}
|
|
248
|
+
|
|
164
249
|
function advertiseMemoryPort(boundPort, attempt = 0) {
|
|
165
250
|
if (!Number.isFinite(boundPort) || boundPort <= 0) return
|
|
166
251
|
_currentAdvertisedPort = boundPort
|
|
@@ -255,52 +340,6 @@ let _initialized = false
|
|
|
255
340
|
let _initPromise = null
|
|
256
341
|
let _stopPromise = null
|
|
257
342
|
let _bootTimestamp = null
|
|
258
|
-
// Post-ingest raw-embedding flush chain (facade-local). The checkCycles tick
|
|
259
|
-
// keeps its OWN independent guard inside the scheduler; this chain serializes
|
|
260
|
-
// flushes kicked from ingestSessionMessages so ingest bursts never stack
|
|
261
|
-
// embedding work, while still guaranteeing every ingest's rows get their own
|
|
262
|
-
// flush pass (the old boolean guard silently SKIPPED bursts, leaving rows
|
|
263
|
-
// embedding-less until the ~60s tick — a recall (no results) window).
|
|
264
|
-
let _rawEmbedFlushChain = Promise.resolve()
|
|
265
|
-
// Per-session ingest serialization. Concurrent ingest_session calls for the
|
|
266
|
-
// SAME session raced on MAX(source_turn) → duplicate turn allocation. Chain
|
|
267
|
-
// same-session ingests so the MAX read + insert loop for one session never
|
|
268
|
-
// overlaps another for the same session. Different sessions stay parallel.
|
|
269
|
-
const _ingestSessionChains = new Map()
|
|
270
|
-
// Cache of (role, cleaned content) per session message OBJECT, keyed by
|
|
271
|
-
// identity (WeakMap — entries vanish once the message is GC'd, e.g. after
|
|
272
|
-
// compaction drops the array). The subset-reingest ordinal fix below must
|
|
273
|
-
// replay cleanMemoryText/shouldExcludeIngestMessage over messages[0, start)
|
|
274
|
-
// on every call; without caching that turned an O(limit)-bounded ingest into
|
|
275
|
-
// an O(full transcript) regex pass on every hydrate. Message objects are
|
|
276
|
-
// immutable transcript entries reused by reference across calls (recall-
|
|
277
|
-
// fasttrack re-ingests the same in-memory array as the session grows), so
|
|
278
|
-
// once a message's identity fields are computed they never change — caching
|
|
279
|
-
// makes every call after the first pay only for genuinely NEW messages,
|
|
280
|
-
// restoring the limit-bounded cost in steady state.
|
|
281
|
-
const _ingestIdentityFieldsCache = new WeakMap()
|
|
282
|
-
// Return { role, content } for a session message's dedup identity, or null if
|
|
283
|
-
// the message would be skipped by ingest (no role / excluded / empty content
|
|
284
|
-
// after cleaning). Cached per message object so repeated calls over the same
|
|
285
|
-
// (unchanged) transcript prefix never re-run the expensive clean/normalize
|
|
286
|
-
// regex pipeline.
|
|
287
|
-
function _ingestIdentityFields(m) {
|
|
288
|
-
if (_ingestIdentityFieldsCache.has(m)) return _ingestIdentityFieldsCache.get(m)
|
|
289
|
-
let fields = null
|
|
290
|
-
const role = normalizeIngestRole(m.role)
|
|
291
|
-
if (role && !shouldExcludeIngestMessage(m)) {
|
|
292
|
-
const content = cleanMemoryText(sessionMessageContentForIngest(m))
|
|
293
|
-
if (content && content.trim()) fields = { role, content }
|
|
294
|
-
}
|
|
295
|
-
_ingestIdentityFieldsCache.set(m, fields)
|
|
296
|
-
return fields
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
// Ingest now WAITS for its embedding flush (bounded) so freshly ingested rows
|
|
300
|
-
// are dense-searchable the moment ingest_session returns. On a cold ONNX
|
|
301
|
-
// model the flush can stall on warmup; the race below caps the wait and lets
|
|
302
|
-
// the flush finish in the background (tick fallback still sweeps stragglers).
|
|
303
|
-
const INGEST_EMBED_WAIT_MS = 15_000
|
|
304
343
|
// Boot-edge background warmup. ONNX session creation on the embedding worker
|
|
305
344
|
// thread is CPU-heavy, so it must not overlap the worker's own init (DB open,
|
|
306
345
|
// schema, cycle wiring). Previously this was gated behind a fixed setTimeout —
|
|
@@ -311,6 +350,10 @@ const INGEST_EMBED_WAIT_MS = 15_000
|
|
|
311
350
|
|
|
312
351
|
const TRANSCRIPT_OFFSETS_KEY = 'state.transcript_offsets'
|
|
313
352
|
const CYCLE_LAST_RUN_KEY = 'state.cycle_last_run'
|
|
353
|
+
// Per-session durable high-water for untimestamped-repeat ordinals (one small
|
|
354
|
+
// hash→next-ordinal map per session, only for identities that reached a
|
|
355
|
+
// duplicate). Stored in the `meta` kv (entries schema untouched).
|
|
356
|
+
const SESSION_INGEST_ORDINALS_KEY_PREFIX = 'state.session_ingest_ordinals.'
|
|
314
357
|
|
|
315
358
|
// Transcript ingest cluster (extracted to lib/transcript-ingest.mjs). Live
|
|
316
359
|
// db/config coupling is injected so index.mjs keeps lifecycle ownership.
|
|
@@ -331,6 +374,22 @@ const {
|
|
|
331
374
|
parseTsToMs,
|
|
332
375
|
} = _transcriptIngest
|
|
333
376
|
|
|
377
|
+
// Session ingest runtime (extracted to lib/session-ingest-runtime.mjs). Owns
|
|
378
|
+
// the per-session chains, identity cache, and post-ingest raw-embedding flush
|
|
379
|
+
// chain; live db + parseTsToMs are injected so the facade keeps db ownership.
|
|
380
|
+
const _sessionIngest = createSessionIngestRuntime({
|
|
381
|
+
getDb: () => db,
|
|
382
|
+
log: __mixdogMemoryLog,
|
|
383
|
+
parseTsToMs,
|
|
384
|
+
loadOrdinalHighWater: async (sessionId) => {
|
|
385
|
+
const raw = await getMetaValue(db, `${SESSION_INGEST_ORDINALS_KEY_PREFIX}${sessionId}`, 'null')
|
|
386
|
+
try { return JSON.parse(raw) } catch { return null }
|
|
387
|
+
},
|
|
388
|
+
saveOrdinalHighWater: (sessionId, obj) =>
|
|
389
|
+
setMetaValue(db, `${SESSION_INGEST_ORDINALS_KEY_PREFIX}${sessionId}`, JSON.stringify(obj)),
|
|
390
|
+
})
|
|
391
|
+
const { ingestSessionMessages } = _sessionIngest
|
|
392
|
+
|
|
334
393
|
// Boot-edge embedding warmup queue (extracted to lib/embedding-warmup.mjs).
|
|
335
394
|
const _embeddingWarmup = createEmbeddingWarmup({
|
|
336
395
|
canStart: embeddingWarmupCanStart,
|
|
@@ -499,155 +558,6 @@ async function setCycleLastRun(kind, ts) {
|
|
|
499
558
|
await mergeMetaValue(db, CYCLE_LAST_RUN_KEY, { [kind]: ts })
|
|
500
559
|
}
|
|
501
560
|
|
|
502
|
-
async function ingestSessionMessages(args = {}) {
|
|
503
|
-
const sessionId = String(args.sessionId || args.session_id || `session-${Date.now()}`).trim()
|
|
504
|
-
// Serialize per-session so the MAX(source_turn) read + insert loop for one
|
|
505
|
-
// session never overlaps a concurrent ingest for the SAME session (they
|
|
506
|
-
// otherwise race the turn allocator and double-allocate). Distinct sessions
|
|
507
|
-
// run in parallel.
|
|
508
|
-
const prev = _ingestSessionChains.get(sessionId) ?? Promise.resolve()
|
|
509
|
-
const run = prev.catch(() => {}).then(() => _ingestSessionMessagesImpl(sessionId, args))
|
|
510
|
-
_ingestSessionChains.set(sessionId, run.catch(() => {}))
|
|
511
|
-
const tail = _ingestSessionChains.get(sessionId)
|
|
512
|
-
try {
|
|
513
|
-
return await run
|
|
514
|
-
} finally {
|
|
515
|
-
// Best-effort GC: drop the map entry only if no later call chained after us.
|
|
516
|
-
if (_ingestSessionChains.get(sessionId) === tail) _ingestSessionChains.delete(sessionId)
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
async function _ingestSessionMessagesImpl(sessionId, args = {}) {
|
|
521
|
-
const messages = Array.isArray(args.messages) ? args.messages : []
|
|
522
|
-
// Recall fast-track hydrates the current session before compaction; allow
|
|
523
|
-
// callers to ingest the full in-memory transcript instead of silently
|
|
524
|
-
// clipping long sessions at 500 turns. Default remains conservative.
|
|
525
|
-
const limit = Math.max(1, Math.min(5000, Number(args.limit) || 200))
|
|
526
|
-
const start = Math.max(0, messages.length - limit)
|
|
527
|
-
const projectId = resolveProjectScope(typeof args.cwd === 'string' && args.cwd ? args.cwd : null)
|
|
528
|
-
let considered = 0
|
|
529
|
-
let inserted = 0
|
|
530
|
-
// Monotonic ingest order, independent of the current (post-compaction)
|
|
531
|
-
// array index. source_turn used to be `i+1`, but after compaction shrinks /
|
|
532
|
-
// reindexes session.messages a NEWLY appended turn gets a LOW i and thus a
|
|
533
|
-
// LOW source_turn — and since dump_session_roots / recall order by
|
|
534
|
-
// source_turn first, it would sort BEFORE older pre-compaction rows. Seed a
|
|
535
|
-
// running counter from the current max source_turn for this session so every
|
|
536
|
-
// new row is assigned a turn strictly greater than all previously-ingested
|
|
537
|
-
// ones (true continuation order). Re-ingested (ON CONFLICT) rows keep their
|
|
538
|
-
// original turn and do not consume a new one.
|
|
539
|
-
let prevMaxTurn = 0
|
|
540
|
-
try {
|
|
541
|
-
const maxRow = await db.query(
|
|
542
|
-
`SELECT COALESCE(MAX(source_turn), 0) AS max_turn FROM entries WHERE session_id = $1`,
|
|
543
|
-
[sessionId],
|
|
544
|
-
)
|
|
545
|
-
prevMaxTurn = Number(maxRow.rows?.[0]?.max_turn) || 0
|
|
546
|
-
} catch { prevMaxTurn = 0 }
|
|
547
|
-
const turnAllocator = createIngestTurnAllocator(prevMaxTurn)
|
|
548
|
-
const _identityOccurrence = new Map()
|
|
549
|
-
// Per-session-stable ordinal for untimestamped, textually-identical repeats.
|
|
550
|
-
// Must NOT use the global next source_turn (differs every run → re-ingest
|
|
551
|
-
// duplicates). The Nth identical (role,content) untimestamped turn gets
|
|
552
|
-
// ordinal N based on its POSITION IN THE FULL messages ARRAY (0..i), not
|
|
553
|
-
// merely within this call's [start, len) window. Seed the counter by
|
|
554
|
-
// replaying the SAME identity/exclude logic over messages[0, start) below
|
|
555
|
-
// (no DB writes, no query — the full array is already in memory) so a
|
|
556
|
-
// SUBSET re-ingest (e.g. only a session's later half) reproduces the exact
|
|
557
|
-
// ordinal a full re-ingest would assign at that array position. Appended
|
|
558
|
-
// genuine repeats therefore never collide with earlier persisted rows of
|
|
559
|
-
// the same identity, while a full identical re-ingest still reproduces the
|
|
560
|
-
// same ordinals/source_refs and ON CONFLICT dedupes exactly.
|
|
561
|
-
for (let i = 0; i < start; i += 1) {
|
|
562
|
-
const m = messages[i]
|
|
563
|
-
if (!m || typeof m !== 'object') continue
|
|
564
|
-
// Cached (WeakMap-keyed) so a repeated hydrate over the SAME prefix
|
|
565
|
-
// objects only pays the clean/normalize cost once per message, keeping
|
|
566
|
-
// this seed pass effectively free after the first ingest call.
|
|
567
|
-
const fields = _ingestIdentityFields(m)
|
|
568
|
-
if (!fields) continue
|
|
569
|
-
const occKey = `${fields.role}\u0000${fields.content}`
|
|
570
|
-
_identityOccurrence.set(occKey, (_identityOccurrence.get(occKey) ?? 0) + 1)
|
|
571
|
-
}
|
|
572
|
-
for (let i = start; i < messages.length; i += 1) {
|
|
573
|
-
const m = messages[i]
|
|
574
|
-
if (!m || typeof m !== 'object') continue
|
|
575
|
-
const role = normalizeIngestRole(m.role)
|
|
576
|
-
// ingest_session persists user/assistant only; system/developer/tool rows
|
|
577
|
-
// are dropped so recall-fasttrack summaries stay conversation-focused and
|
|
578
|
-
// do not re-inject content already in the protected system prefix.
|
|
579
|
-
if (!role) continue
|
|
580
|
-
// Exclude synthetic / non-conversation rows entirely (reference-files
|
|
581
|
-
// injections, compaction summaries, protected-context `.` acks, internal
|
|
582
|
-
// runtime nudges). These are mechanical noise, not conversation.
|
|
583
|
-
if (shouldExcludeIngestMessage(m)) continue
|
|
584
|
-
// Pure-conversation shaping: only the human/model prose text. The ingest
|
|
585
|
-
// shaper NEVER inlines tool_call / tool_result traces and strips the
|
|
586
|
-
// deterministic manager.mjs user-turn prefix envelopes (# Session /
|
|
587
|
-
// # Additional context / # Prefetch / # Task) so only the real human
|
|
588
|
-
// prompt remains. cleanMemoryText then removes the <system-reminder> block
|
|
589
|
-
// and residual transcript noise.
|
|
590
|
-
const content = cleanMemoryText(sessionMessageContentForIngest(m))
|
|
591
|
-
if (!content || !content.trim()) continue
|
|
592
|
-
considered += 1
|
|
593
|
-
const tsMs = parseTsToMs(m.ts ?? m.timestamp ?? (Date.now() - (messages.length - i)))
|
|
594
|
-
// Assign the next monotonic turn BEFORE building the source_ref so identical
|
|
595
|
-
// untimestamped repeats get distinct identities (peekNext is stable until a
|
|
596
|
-
// row is actually inserted → next()).
|
|
597
|
-
const assignedTurn = turnAllocator.peekNext()
|
|
598
|
-
// Stable occurrence index for this identity (pre-hash; role+content is a
|
|
599
|
-
// sufficient discriminator for the duplicate case the ordinal disambiguates),
|
|
600
|
-
// seeded above from messages[0, start) so it reflects full-array position
|
|
601
|
-
// regardless of where this call's window starts.
|
|
602
|
-
const occKey = `${role}\u0000${content}`
|
|
603
|
-
const occurrence = (_identityOccurrence.get(occKey) ?? 0)
|
|
604
|
-
_identityOccurrence.set(occKey, occurrence + 1)
|
|
605
|
-
// Stable per-message identity. The previous `session:${id}#${i+1}` key was
|
|
606
|
-
// positional, so after compaction shrinks/reindexes session.messages a
|
|
607
|
-
// later turn could reuse an old index and be silently skipped by
|
|
608
|
-
// ON CONFLICT DO NOTHING. stableSessionSourceRef hashes only durable
|
|
609
|
-
// fields (role, original ts if present, tool ids, content) — never the
|
|
610
|
-
// synthesized tsMs fallback or the loop index. For untimestamped turns the
|
|
611
|
-
// monotonic ordinal is folded in so genuine repeats persist (not collapsed).
|
|
612
|
-
const sourceRef = stableSessionSourceRef(sessionId, m, role, content, occurrence)
|
|
613
|
-
const result = await db.query(`
|
|
614
|
-
INSERT INTO entries(ts, role, content, source_ref, session_id, source_turn, project_id)
|
|
615
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
|
616
|
-
ON CONFLICT DO NOTHING
|
|
617
|
-
`, [tsMs, role, content, sourceRef, sessionId, assignedTurn, projectId])
|
|
618
|
-
const rowInserted = Number(result.rowCount ?? result.affectedRows ?? 0) || 0
|
|
619
|
-
if (rowInserted > 0) {
|
|
620
|
-
inserted += rowInserted
|
|
621
|
-
turnAllocator.next()
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
// Always-on post-ingest raw embedding: freshly ingested rows become
|
|
625
|
-
// dense-searchable immediately (autoclear/recall-fasttrack hydration,
|
|
626
|
-
// recall empty-fallback), without waiting for cycle1 chunking or the
|
|
627
|
-
// ~60s background tick. Local ONNX only — no LLM cost, so it runs
|
|
628
|
-
// regardless of the recap toggle. SYNCHRONOUS (bounded): ingest_session
|
|
629
|
-
// resolves only after this session's rows are embedded, so an immediately
|
|
630
|
-
// following recall sees them in the dense leg. Bursts serialize on
|
|
631
|
-
// _rawEmbedFlushChain instead of being skipped; a 15s cap keeps a cold
|
|
632
|
-
// ONNX warmup from wedging ingest (tick fallback sweeps what's left).
|
|
633
|
-
if (inserted > 0) {
|
|
634
|
-
const run = _rawEmbedFlushChain
|
|
635
|
-
.catch(() => {}) // never let a previous flush failure poison the chain
|
|
636
|
-
.then(() => flushRawEmbeddings(db, { limit: 200 }))
|
|
637
|
-
.then((r) => {
|
|
638
|
-
if (r.attempted > 0) __mixdogMemoryLog(`[embed] post-ingest raw flush attempted=${r.attempted} embedded=${r.embedded}\n`)
|
|
639
|
-
return r
|
|
640
|
-
})
|
|
641
|
-
.catch((err) => __mixdogMemoryLog(`[embed] post-ingest raw flush failed: ${err?.message || err}\n`))
|
|
642
|
-
_rawEmbedFlushChain = run
|
|
643
|
-
let timer
|
|
644
|
-
await Promise.race([
|
|
645
|
-
run,
|
|
646
|
-
new Promise((resolve) => { timer = setTimeout(resolve, INGEST_EMBED_WAIT_MS) }),
|
|
647
|
-
]).finally(() => clearTimeout(timer))
|
|
648
|
-
}
|
|
649
|
-
return { text: `ingest_session: considered=${considered} inserted=${inserted} session=${sessionId}` }
|
|
650
|
-
}
|
|
651
561
|
|
|
652
562
|
// ── Cycle scheduling cluster (extracted to lib/cycle-scheduler.mjs) ────────
|
|
653
563
|
// The mutually-referential cycle machinery (health ledger, cycle1 outer
|
|
@@ -782,828 +692,30 @@ const {
|
|
|
782
692
|
entryStats,
|
|
783
693
|
} = __queryHandlers
|
|
784
694
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
if (Number.isFinite(rowsPerSessionOverride) && rowsPerSessionOverride > 0) {
|
|
810
|
-
cycle1Config = { ...cycle1Config, rows_per_session: rowsPerSessionOverride }
|
|
811
|
-
}
|
|
812
|
-
if (sessionIdOverride) {
|
|
813
|
-
cycle1Config = { ...cycle1Config, session_id: sessionIdOverride }
|
|
814
|
-
}
|
|
815
|
-
if (Number.isFinite(concurrencyOverride) && concurrencyOverride > 0) {
|
|
816
|
-
cycle1Config = { ...cycle1Config, concurrency: Math.min(8, Math.floor(concurrencyOverride)) }
|
|
817
|
-
}
|
|
818
|
-
const callerDeadlineMs = Number(args?._callerDeadlineMs) || 0
|
|
819
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
820
|
-
const cycle1Options = callerDeadlineMs > 0 ? { callerDeadlineMs, signal } : { signal }
|
|
821
|
-
if (typeof args?._callLlm === 'function') {
|
|
822
|
-
cycle1Options.callLlm = args._callLlm
|
|
823
|
-
}
|
|
824
|
-
const result = await _awaitCycle1Run(
|
|
825
|
-
cycle1Config,
|
|
826
|
-
cycle1Options,
|
|
827
|
-
)
|
|
828
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
829
|
-
const pendingStr = result?.pendingRows != null ? result.pendingRows : 0
|
|
830
|
-
const inFlightStr = result?.skippedInFlight === true ? 'true' : 'false'
|
|
831
|
-
const timedOutPart = result?.timedOutWaiting === true ? ' timedOut=true' : ''
|
|
832
|
-
const omitted = Array.isArray(result?.omitted_row_ids) ? result.omitted_row_ids.length : Number(result?.quality?.omitted_rows || 0)
|
|
833
|
-
const prefiltered = Array.isArray(result?.prefiltered_row_ids) ? result.prefiltered_row_ids.length : Number(result?.quality?.prefiltered_rows || 0)
|
|
834
|
-
const failedRows = Array.isArray(result?.failed_row_ids) ? result.failed_row_ids.length : Number(result?.quality?.failed_rows || 0)
|
|
835
|
-
const invalidChunks = Array.isArray(result?.invalid_chunks) ? result.invalid_chunks.length : Number(result?.quality?.invalid_chunks || 0)
|
|
836
|
-
return {
|
|
837
|
-
...result,
|
|
838
|
-
text: `cycle1: chunks=${result.chunks} processed=${result.processed} skipped_chunks=${result.skipped}` +
|
|
839
|
-
` omitted=${omitted} prefiltered=${prefiltered} failed_rows=${failedRows} invalid_chunks=${invalidChunks}` +
|
|
840
|
-
` pending=${pendingStr} inFlight=${inFlightStr}${timedOutPart}`,
|
|
841
|
-
}
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
async function _handleMemCycle2(args, config, signal) {
|
|
845
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
846
|
-
let c2Options = { signal }
|
|
847
|
-
if (typeof c2Options?.callLlm !== 'function') {
|
|
848
|
-
c2Options = { ...c2Options, callLlm: getCycle2CallLlm() }
|
|
849
|
-
}
|
|
850
|
-
const result = await runCycle2(db, config?.cycle2 || {}, c2Options, DATA_DIR)
|
|
851
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
852
|
-
await _finalizeCycle2Run(result)
|
|
853
|
-
const counts = {
|
|
854
|
-
promoted: result?.promoted || 0,
|
|
855
|
-
archived: result?.archived || 0,
|
|
856
|
-
merged: result?.merged || 0,
|
|
857
|
-
updated: result?.updated || 0,
|
|
858
|
-
kept: result?.kept || 0,
|
|
859
|
-
rejected_verb: result?.rejected_verb || 0,
|
|
860
|
-
merge_rejected: result?.merge_rejected || 0,
|
|
861
|
-
missing_core: result?.missing_core_summary || 0,
|
|
862
|
-
core_backfill: result?.core_embedding_backfill || 0,
|
|
863
|
-
cascade_drop: result?.cascade?.dropped || 0,
|
|
864
|
-
phase_merge: result?.phase_merge?.merged || 0,
|
|
865
|
-
core_overlap: result?.phase_merge?.core_overlap || 0,
|
|
866
|
-
}
|
|
867
|
-
const parts = Object.entries(counts).filter(([, v]) => v > 0).map(([k, v]) => `${k}=${v}`)
|
|
868
|
-
if (parts.length) return { text: `cycle2 ${parts.join(' ')}` }
|
|
869
|
-
// No applied counts — disambiguate the "noop" so a broken gate is visible
|
|
870
|
-
// instead of looking like a clean, nothing-to-do run.
|
|
871
|
-
let cause = ''
|
|
872
|
-
if (result?.skippedInFlight) cause = ' (skipped: in-flight)'
|
|
873
|
-
else if (result?.ok === false) cause = ` (error: ${result.error || 'unknown'})`
|
|
874
|
-
else if (result?.gate_failed) cause = ' (gate_failed)'
|
|
875
|
-
return { text: `cycle2 noop${cause}` }
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
async function _handleMemCycle3(args, config, signal) {
|
|
879
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
880
|
-
const confirmed = args?.confirm === 'APPLY CYCLE3'
|
|
881
|
-
const requestedMode = typeof args?.cycle3Mode === 'string' ? args.cycle3Mode : null
|
|
882
|
-
const applyMode = confirmed
|
|
883
|
-
? 'confirmed'
|
|
884
|
-
: (requestedMode === 'proposal' || requestedMode === 'dry-run' || requestedMode === 'dryrun')
|
|
885
|
-
? 'proposal'
|
|
886
|
-
: 'conservative'
|
|
887
|
-
let c3Options = { signal, apply: confirmed ? true : undefined, applyMode }
|
|
888
|
-
if (typeof c3Options?.callLlm !== 'function') {
|
|
889
|
-
c3Options = { ...c3Options, callLlm: getCycle3CallLlm() }
|
|
890
|
-
}
|
|
891
|
-
const result = await runCycle3(db, config || {}, DATA_DIR, c3Options)
|
|
892
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
893
|
-
// Stamp cycle3 success: the MCP path bypasses the scheduler's coalesced
|
|
894
|
-
// onCoalescedSuccess, so without this last_success_at stayed 0 despite
|
|
895
|
-
// successful runs.
|
|
896
|
-
await _finalizeCycle3Run(result)
|
|
897
|
-
const parts = ['reviewed', 'kept', 'updated', 'merged', 'deleted']
|
|
898
|
-
.map(k => `${k}=${result?.[k] || 0}`)
|
|
899
|
-
if (result?.proposed) {
|
|
900
|
-
parts.push(`proposal_update=${result.proposed.updated || 0}`)
|
|
901
|
-
parts.push(`proposal_merge=${result.proposed.merged || 0}`)
|
|
902
|
-
parts.push(`proposal_delete=${result.proposed.deleted || 0}`)
|
|
903
|
-
}
|
|
904
|
-
if (result?.held) {
|
|
905
|
-
parts.push(`held_update=${result.held.updated || 0}`)
|
|
906
|
-
parts.push(`held_merge=${result.held.merged || 0}`)
|
|
907
|
-
parts.push(`held_delete=${result.held.deleted || 0}`)
|
|
908
|
-
}
|
|
909
|
-
parts.push(`mode=${result?.applyMode || applyMode}`)
|
|
910
|
-
parts.push(`applied=${result?.applied === true ? 'true' : 'false'}`)
|
|
911
|
-
if (result?.skippedInFlight) parts.push('inFlight=true')
|
|
912
|
-
const errPart = result?.error ? ` error=${result.error}` : ''
|
|
913
|
-
return { text: `cycle3 ${parts.join(' ')}${errPart}` }
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
async function _handleMemFlush(args, config, signal) {
|
|
917
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
918
|
-
const r1 = await _awaitCycle1Run(config?.cycle1 || {}, { signal })
|
|
919
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
920
|
-
let flushC2Options = { signal }
|
|
921
|
-
if (typeof flushC2Options?.callLlm !== 'function') {
|
|
922
|
-
flushC2Options = { ...flushC2Options, callLlm: getCycle2CallLlm() }
|
|
923
|
-
}
|
|
924
|
-
const r2 = await runCycle2(db, config?.cycle2 || {}, flushC2Options, DATA_DIR)
|
|
925
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
926
|
-
await _finalizeCycle2Run(r2)
|
|
927
|
-
return { text: `flush: cycle1 chunks=${r1.chunks} processed=${r1.processed}, cycle2 ${JSON.stringify(r2)}` }
|
|
928
|
-
}
|
|
929
|
-
|
|
930
|
-
async function _handleMemStatus(args, config) {
|
|
931
|
-
const stats = await entryStats()
|
|
932
|
-
const last = await getCycleLastRun()
|
|
933
|
-
let dims = 0
|
|
934
|
-
let dimsErr = null
|
|
935
|
-
try {
|
|
936
|
-
const raw = await getMetaValue(db, 'embedding.current_dims', null)
|
|
937
|
-
if (raw != null) dims = Number(JSON.parse(raw))
|
|
938
|
-
if (!Number.isFinite(dims)) dims = 0
|
|
939
|
-
} catch (e) {
|
|
940
|
-
// Surface the error in the status line instead of masquerading a meta
|
|
941
|
-
// read failure as dims=0 (which is indistinguishable from a fresh,
|
|
942
|
-
// pre-bootstrap DB). Keep status callable so other lines still render.
|
|
943
|
-
dims = 0
|
|
944
|
-
dimsErr = e?.message || String(e)
|
|
945
|
-
}
|
|
946
|
-
const bootstrapComplete = await isBootstrapComplete(db)
|
|
947
|
-
const lastCycle1Ago = last.cycle1 ? `${Math.round((Date.now() - last.cycle1) / 60000)}m ago` : 'never'
|
|
948
|
-
const lastCycle2Ago = last.cycle2 ? `${Math.round((Date.now() - last.cycle2) / 60000)}m ago` : 'never'
|
|
949
|
-
const activeTargetCap = Number.isFinite(Number(config?.cycle2?.active_target_cap))
|
|
950
|
-
? Number(config?.cycle2?.active_target_cap)
|
|
951
|
-
: CYCLE2_ACTIVE_TARGET_CAP
|
|
952
|
-
const mvState = stats.mv_hot_active_populated === null
|
|
953
|
-
? 'missing'
|
|
954
|
-
: stats.mv_hot_active_populated ? 'populated' : 'unpopulated'
|
|
955
|
-
const lines = [
|
|
956
|
-
`entries: total=${stats.total} roots=${stats.roots} cycle1_raw=${stats.unchunked_leaves} (unchunked leaves) cycle2_pending=${stats.cycle2_pending_roots} (awaiting cycle2 review)`,
|
|
957
|
-
`status: ${stats.byStatus.map(r => `${r.status ?? '?'}:${r.c}`).join(', ') || 'empty'}`,
|
|
958
|
-
`categories(active): ${stats.byCategory.map(r => `${r.category ?? 'NULL'}:${r.c}`).join(', ') || 'empty'} active_target_cap=${activeTargetCap}`,
|
|
959
|
-
`core_memory: user=${stats.core_entries} embed_null=${stats.core_embed_null} active_core=${stats.active_core_summaries} active_missing_core=${stats.active_core_summary_missing}`,
|
|
960
|
-
`embedding_index: ready dims=${dims}${dimsErr ? ` (meta_read_error: ${dimsErr})` : ''}`,
|
|
961
|
-
`recall_index: mv_hot_active=${mvState}`,
|
|
962
|
-
`bootstrap: ${bootstrapComplete ? 'complete' : 'incomplete'}`,
|
|
963
|
-
`last_cycle1: ${lastCycle1Ago}`,
|
|
964
|
-
`last_cycle2: ${lastCycle2Ago}`,
|
|
965
|
-
...(last.cycle2_last_error ? [`last_cycle2_error: ${last.cycle2_last_error}`] : []),
|
|
966
|
-
]
|
|
967
|
-
return { text: lines.join('\n') }
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
async function _handleMemRebuild(args, config, signal) {
|
|
971
|
-
if (args.confirm !== 'REBUILD MEMORY') {
|
|
972
|
-
return { text: 'rebuild requires confirm: "REBUILD MEMORY" (truncates classification columns and re-runs cycles)', isError: true }
|
|
973
|
-
}
|
|
974
|
-
// Drain any pre-reset cycle1 BEFORE the destructive truncation so the
|
|
975
|
-
// post-reset run is not started concurrently against the same DB.
|
|
976
|
-
// _awaitCycle1Run() may release the outer handle on a caller deadline while
|
|
977
|
-
// the inner runCycle1 promise still owns the DB writes. Drain both layers,
|
|
978
|
-
// then loop once more if one layer exposed another promise while awaiting.
|
|
979
|
-
const drainedCycle1Promises = new Set()
|
|
980
|
-
for (;;) {
|
|
981
|
-
const pendingCycle1Promises = [_cycleScheduler.getCycle1InFlight(), getInFlightCycle1(db)]
|
|
982
|
-
.filter(p => p && !drainedCycle1Promises.has(p))
|
|
983
|
-
if (pendingCycle1Promises.length === 0) break
|
|
984
|
-
for (const pendingCycle1 of pendingCycle1Promises) {
|
|
985
|
-
drainedCycle1Promises.add(pendingCycle1)
|
|
986
|
-
try { await pendingCycle1 } catch {}
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
990
|
-
// Cleanup must run BEFORE demotion: the original order demoted normal
|
|
991
|
-
// roots (chunk_root = id) to is_root = 0 first, then ran the cleanup
|
|
992
|
-
// WHERE is_root = 1 — which missed exactly those demoted rows, leaving
|
|
993
|
-
// stale element/category/summary/score/embedding/summary_hash on rows that
|
|
994
|
-
// had just become raw leaves. Reorder so all roots get their classification
|
|
995
|
-
// columns cleared while is_root = 1 still selects them, then demote.
|
|
996
|
-
// Wrap the whole destructive sequence in one transaction so a mid-step
|
|
997
|
-
// failure rolls back rather than leaving a mixed state.
|
|
998
|
-
await db.transaction(async (tx) => {
|
|
999
|
-
await tx.query(`
|
|
1000
|
-
UPDATE entries
|
|
1001
|
-
SET element = NULL, category = NULL, summary = NULL,
|
|
1002
|
-
status = 'pending', score = NULL, last_seen_at = NULL,
|
|
1003
|
-
embedding = NULL, summary_hash = NULL,
|
|
1004
|
-
core_summary = NULL, reviewed_at = NULL, promoted_at = NULL,
|
|
1005
|
-
error_count = 0
|
|
1006
|
-
WHERE is_root = 1
|
|
1007
|
-
`)
|
|
1008
|
-
await tx.query(`UPDATE entries SET chunk_root = NULL, is_root = 0 WHERE chunk_root = id`)
|
|
1009
|
-
await tx.query(`UPDATE entries SET chunk_root = NULL WHERE is_root = 0`)
|
|
1010
|
-
await tx.query(`
|
|
1011
|
-
UPDATE entries
|
|
1012
|
-
SET status = NULL,
|
|
1013
|
-
element = NULL, category = NULL, summary = NULL,
|
|
1014
|
-
score = NULL, last_seen_at = NULL,
|
|
1015
|
-
embedding = NULL, summary_hash = NULL,
|
|
1016
|
-
core_summary = NULL, reviewed_at = NULL, promoted_at = NULL,
|
|
1017
|
-
error_count = 0
|
|
1018
|
-
WHERE is_root = 0
|
|
1019
|
-
`)
|
|
1020
|
-
})
|
|
1021
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1022
|
-
// Force a fresh post-reset cycle1: _cycle1InFlight is guaranteed null
|
|
1023
|
-
// here (we drained above and have not awaited any cycle1-starting call
|
|
1024
|
-
// since), so calling _startCycle1Run directly skips the coalesce branch
|
|
1025
|
-
// inside _awaitCycle1Run and guarantees the newly demoted rows are read.
|
|
1026
|
-
const r1 = await _startCycle1Run(config?.cycle1 || {}, { signal })
|
|
1027
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1028
|
-
let rebuildC2Options = { signal }
|
|
1029
|
-
if (typeof rebuildC2Options?.callLlm !== 'function') {
|
|
1030
|
-
rebuildC2Options = { ...rebuildC2Options, callLlm: getCycle2CallLlm() }
|
|
1031
|
-
}
|
|
1032
|
-
const r2 = await runCycle2(db, config?.cycle2 || {}, rebuildC2Options, DATA_DIR)
|
|
1033
|
-
await _finalizeCycle2Run(r2)
|
|
1034
|
-
return { text: `rebuild: cycle1 chunks=${r1.chunks} processed=${r1.processed}, cycle2 ${JSON.stringify(r2)}` }
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
async function _handleMemPrune(args, _config) {
|
|
1038
|
-
if (args.confirm !== 'PRUNE OLD ENTRIES') {
|
|
1039
|
-
return { text: 'prune requires confirm: "PRUNE OLD ENTRIES" (permanently deletes unclassified entries older than maxDays)', isError: true }
|
|
1040
|
-
}
|
|
1041
|
-
const days = Math.max(1, Number(args.maxDays ?? 30))
|
|
1042
|
-
const result = await pruneOldEntries(db, days)
|
|
1043
|
-
return { text: `prune: deleted ${result.deleted} unclassified entries older than ${days} days` }
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
async function _handleMemBackfill(args, config, signal) {
|
|
1047
|
-
// Whole-action mutex (transport-agnostic). _cycle1InFlight only protects
|
|
1048
|
-
// cycle1; ingest workers + cycle2 can still overlap if a second backfill
|
|
1049
|
-
// kicks in (timeout-retry, parallel callers, /api/tool vs /mcp vs
|
|
1050
|
-
// /admin/backfill). Sentinel is set synchronously before any await so a
|
|
1051
|
-
// burst of concurrent calls cannot all pass the check.
|
|
1052
|
-
if (_backfillInFlight) {
|
|
1053
|
-
return { text: 'backfill already in progress', isError: true }
|
|
1054
|
-
}
|
|
1055
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1056
|
-
const window = args.window != null ? String(args.window) : '7d'
|
|
1057
|
-
const scope = args.scope != null ? String(args.scope) : 'all'
|
|
1058
|
-
const limit = args.limit != null ? Math.max(1, Number(args.limit)) : null
|
|
1059
|
-
// Capture the cycle2 envelope so we can route through _finalizeCycle2Run
|
|
1060
|
-
// (which records cycle2_last_error and clears scheduler delay only on
|
|
1061
|
-
// ok:true) rather than stamping cycle2 unconditionally afterward.
|
|
1062
|
-
let _capturedCycle2
|
|
1063
|
-
const promise = runFullBackfill(db, {
|
|
1064
|
-
window,
|
|
1065
|
-
scope,
|
|
1066
|
-
limit,
|
|
1067
|
-
config,
|
|
1068
|
-
dataDir: DATA_DIR,
|
|
1069
|
-
ingestTranscriptFile,
|
|
1070
|
-
cwdFromTranscriptPath,
|
|
1071
|
-
// Re-check the IPC cancel signal at every cycle1/cycle2 iteration the
|
|
1072
|
-
// backfill driver dispatches. handleMemoryAction only checks once
|
|
1073
|
-
// before dispatch; without per-iteration checkpoints a long-running
|
|
1074
|
-
// backfill keeps spinning through ingest + cycle1 + cycle2 batches
|
|
1075
|
-
// after the proxy has already responded "cancelled" to the caller.
|
|
1076
|
-
runCycle1: (dbArg, cycle1Config = {}, options = {}, _dir) => {
|
|
1077
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1078
|
-
return _awaitCycle1Run(cycle1Config, { ...options, signal })
|
|
1079
|
-
},
|
|
1080
|
-
runCycle2: async (dbArg, c2Config, c2Options, c2DataDir) => {
|
|
1081
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1082
|
-
let backfillC2Options = { ...c2Options, signal }
|
|
1083
|
-
if (typeof backfillC2Options?.callLlm !== 'function') {
|
|
1084
|
-
backfillC2Options = { ...backfillC2Options, callLlm: getCycle2CallLlm() }
|
|
1085
|
-
}
|
|
1086
|
-
const r2 = await runCycle2(dbArg, c2Config, backfillC2Options, c2DataDir)
|
|
1087
|
-
_capturedCycle2 = r2
|
|
1088
|
-
return r2
|
|
1089
|
-
},
|
|
1090
|
-
})
|
|
1091
|
-
_backfillInFlight = promise
|
|
1092
|
-
let result
|
|
1093
|
-
try {
|
|
1094
|
-
result = await promise
|
|
1095
|
-
} finally {
|
|
1096
|
-
if (_backfillInFlight === promise) _backfillInFlight = null
|
|
1097
|
-
}
|
|
1098
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1099
|
-
if (_capturedCycle2) {
|
|
1100
|
-
await _finalizeCycle2Run(_capturedCycle2)
|
|
1101
|
-
}
|
|
1102
|
-
return {
|
|
1103
|
-
text: `backfill: window=${result.window} scope=${result.scope} files=${result.files} ingested=${result.ingested} cycle1_iters=${result.cycle1_iters} promoted=${result.promoted} unclassified=${result.unclassified}`,
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
async function handleMemoryAction(args, signal) {
|
|
1108
|
-
// Cooperative abort check: surfaces caller-cancel (IPC cancel handler)
|
|
1109
|
-
// before any long DB work begins on the worker side.
|
|
1110
|
-
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
1111
|
-
const action = String(args.action ?? '')
|
|
1112
|
-
const config = readMainConfig()
|
|
1113
|
-
|
|
1114
|
-
if (action === 'status') {
|
|
1115
|
-
return _handleMemStatus(args, config)
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
if (action === 'cycle1') {
|
|
1119
|
-
return _handleMemCycle1(args, config, signal)
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
if (action === 'cycle2' || action === 'sleep') {
|
|
1123
|
-
return _handleMemCycle2(args, config, signal)
|
|
1124
|
-
}
|
|
1125
|
-
|
|
1126
|
-
if (action === 'cycle3') {
|
|
1127
|
-
return _handleMemCycle3(args, config, signal)
|
|
1128
|
-
}
|
|
1129
|
-
|
|
1130
|
-
// Direct semantic-search surface for callers that want raw ranked rows
|
|
1131
|
-
// without going through the Lead-side recall synthesizer. The
|
|
1132
|
-
// handleSearch executor is exposed through the public `memory` tool action
|
|
1133
|
-
// `search` so callers can hit the hybrid CTE directly.
|
|
1134
|
-
if (action === 'search') {
|
|
1135
|
-
return handleSearch(args, signal)
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
if (action === 'flush') {
|
|
1139
|
-
return _handleMemFlush(args, config, signal)
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
if (action === 'rebuild') {
|
|
1143
|
-
return _handleMemRebuild(args, config, signal)
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
if (action === 'prune') {
|
|
1147
|
-
return _handleMemPrune(args, config)
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
if (action === 'backfill') {
|
|
1151
|
-
return _handleMemBackfill(args, config, signal)
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
if (action === 'ingest_session') {
|
|
1155
|
-
return ingestSessionMessages(args)
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
if (action === 'dump_session_roots') {
|
|
1159
|
-
return dumpSessionRootChunks(args)
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
if (action === 'manage') {
|
|
1163
|
-
const op = String(args.op ?? '').trim().toLowerCase()
|
|
1164
|
-
if (!['add', 'edit', 'delete'].includes(op)) {
|
|
1165
|
-
return { text: 'manage requires op: "add" | "edit" | "delete"', isError: true }
|
|
1166
|
-
}
|
|
1167
|
-
const VALID_CAT = new Set(['rule', 'constraint', 'decision', 'fact', 'goal', 'preference', 'task', 'issue'])
|
|
1168
|
-
const VALID_STATUS = new Set(['pending', 'active', 'archived'])
|
|
1169
|
-
|
|
1170
|
-
if (op === 'add') {
|
|
1171
|
-
const element = String(args.element ?? '').trim()
|
|
1172
|
-
const summary = String(args.summary ?? args.element ?? '').trim()
|
|
1173
|
-
const category = String(args.category ?? 'fact').trim().toLowerCase()
|
|
1174
|
-
if (!element || !summary) {
|
|
1175
|
-
return { text: 'manage add requires element and summary', isError: true }
|
|
1176
|
-
}
|
|
1177
|
-
if (!VALID_CAT.has(category)) {
|
|
1178
|
-
return { text: `manage add: invalid category "${category}". Valid: ${[...VALID_CAT].join(', ')}`, isError: true }
|
|
1179
|
-
}
|
|
1180
|
-
const nowMs = Date.now()
|
|
1181
|
-
const sourceRef = `manual:${nowMs}-${process.pid}`
|
|
1182
|
-
const manageProjectId = resolveProjectScope(typeof args.cwd === 'string' && args.cwd ? args.cwd : null)
|
|
1183
|
-
try {
|
|
1184
|
-
let newId
|
|
1185
|
-
await db.transaction(async (tx) => {
|
|
1186
|
-
const result = await tx.query(`
|
|
1187
|
-
INSERT INTO entries(ts, role, content, source_ref, session_id, project_id)
|
|
1188
|
-
VALUES ($1, 'system', $2, $3, NULL, $4)
|
|
1189
|
-
RETURNING id
|
|
1190
|
-
`, [nowMs, element + ' — ' + summary, sourceRef, manageProjectId])
|
|
1191
|
-
newId = Number(result.rows[0].id)
|
|
1192
|
-
const score = computeEntryScore(category, nowMs, nowMs)
|
|
1193
|
-
await tx.query(`
|
|
1194
|
-
UPDATE entries
|
|
1195
|
-
SET chunk_root = $1, is_root = 1, element = $2, category = $3, summary = $4,
|
|
1196
|
-
status = 'active', score = $5, last_seen_at = $6
|
|
1197
|
-
WHERE id = $7
|
|
1198
|
-
`, [newId, element, category, summary, score, nowMs, newId])
|
|
1199
|
-
})
|
|
1200
|
-
await syncRootEmbedding(db, newId)
|
|
1201
|
-
return { text: `added (id=${newId}): [${category}] ${element} — ${summary.slice(0, 200)}` }
|
|
1202
|
-
} catch (e) {
|
|
1203
|
-
return { text: `manage add failed: ${e.message}`, isError: true }
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
if (op === 'edit') {
|
|
1208
|
-
const id = Number(args.id)
|
|
1209
|
-
if (!Number.isFinite(id) || id <= 0) {
|
|
1210
|
-
return { text: 'manage edit requires numeric id', isError: true }
|
|
1211
|
-
}
|
|
1212
|
-
const existing = (await db.query(
|
|
1213
|
-
`SELECT id, element, summary, category, status, ts, is_root FROM entries WHERE id = $1`,
|
|
1214
|
-
[id]
|
|
1215
|
-
)).rows[0]
|
|
1216
|
-
if (!existing) return { text: `manage edit: no entry with id=${id}`, isError: true }
|
|
1217
|
-
if (existing.is_root !== 1) return { text: `manage edit: id=${id} is not a root`, isError: true }
|
|
1218
|
-
|
|
1219
|
-
const trimOrNull = v => {
|
|
1220
|
-
if (v == null) return null
|
|
1221
|
-
const s = String(v).trim()
|
|
1222
|
-
return s === '' ? null : s
|
|
1223
|
-
}
|
|
1224
|
-
const newElement = trimOrNull(args.element)
|
|
1225
|
-
const newSummary = trimOrNull(args.summary)
|
|
1226
|
-
const newCategory = trimOrNull(args.category)?.toLowerCase() ?? null
|
|
1227
|
-
const newStatus = trimOrNull(args.status)?.toLowerCase() ?? null
|
|
1228
|
-
|
|
1229
|
-
if (!newElement && !newSummary && !newCategory && !newStatus) {
|
|
1230
|
-
return { text: 'manage edit requires at least one field: element, summary, category, status', isError: true }
|
|
1231
|
-
}
|
|
1232
|
-
if (newCategory && !VALID_CAT.has(newCategory)) {
|
|
1233
|
-
return { text: `manage edit: invalid category "${newCategory}". Valid: ${[...VALID_CAT].join(', ')}`, isError: true }
|
|
1234
|
-
}
|
|
1235
|
-
if (newStatus && !VALID_STATUS.has(newStatus)) {
|
|
1236
|
-
return { text: `manage edit: invalid status "${newStatus}". Valid: ${[...VALID_STATUS].join(', ')}`, isError: true }
|
|
1237
|
-
}
|
|
1238
|
-
|
|
1239
|
-
const finalElement = newElement ?? existing.element
|
|
1240
|
-
const finalSummary = newSummary ?? existing.summary
|
|
1241
|
-
const finalCategory = newCategory ?? existing.category
|
|
1242
|
-
const finalStatus = newStatus ?? existing.status
|
|
1243
|
-
const nowMs = Date.now()
|
|
1244
|
-
const score = computeEntryScore(finalCategory, nowMs, nowMs)
|
|
1245
|
-
const textChanged = newElement != null || newSummary != null
|
|
1246
|
-
// Guard null element/summary: a category/status-only edit on a root
|
|
1247
|
-
// whose element or summary is NULL would otherwise persist literal
|
|
1248
|
-
// 'null — null' content and explode on finalSummary.slice() below.
|
|
1249
|
-
// Use empty-string sentinels for the content composition + render so
|
|
1250
|
-
// the row stays consistent with what's actually stored.
|
|
1251
|
-
const elementStr = finalElement == null ? '' : String(finalElement)
|
|
1252
|
-
const summaryStr = finalSummary == null ? '' : String(finalSummary)
|
|
1253
|
-
const composedContent = elementStr || summaryStr
|
|
1254
|
-
? `${elementStr}${summaryStr ? ' — ' + summaryStr : ''}`
|
|
1255
|
-
: ''
|
|
1256
|
-
|
|
1257
|
-
try {
|
|
1258
|
-
await db.query(`
|
|
1259
|
-
UPDATE entries
|
|
1260
|
-
SET element = $1, summary = $2, category = $3, status = $4, score = $5,
|
|
1261
|
-
last_seen_at = $6, content = $7
|
|
1262
|
-
WHERE id = $8
|
|
1263
|
-
`, [finalElement, finalSummary, finalCategory, finalStatus, score,
|
|
1264
|
-
nowMs, composedContent, id])
|
|
1265
|
-
} catch (e) {
|
|
1266
|
-
return { text: `manage edit failed: ${e.message}`, isError: true }
|
|
1267
|
-
}
|
|
1268
|
-
if (textChanged) {
|
|
1269
|
-
try { await syncRootEmbedding(db, id) } catch (e) {
|
|
1270
|
-
__mixdogMemoryLog(`[memory.manage] embedding resync failed (id=${id}): ${e.message}\n`)
|
|
1271
|
-
}
|
|
1272
|
-
}
|
|
1273
|
-
return { text: `edited (id=${id}): [${finalCategory}/${finalStatus}] ${elementStr}${summaryStr ? ' — ' + summaryStr.slice(0, 200) : ''}` }
|
|
1274
|
-
}
|
|
1275
|
-
|
|
1276
|
-
if (op === 'delete') {
|
|
1277
|
-
const id = Number(args.id)
|
|
1278
|
-
if (!Number.isFinite(id) || id <= 0) {
|
|
1279
|
-
return { text: 'manage delete requires numeric id', isError: true }
|
|
1280
|
-
}
|
|
1281
|
-
const info = (await db.query(
|
|
1282
|
-
`SELECT id, category, element, is_root FROM entries WHERE id = $1`,
|
|
1283
|
-
[id]
|
|
1284
|
-
)).rows[0]
|
|
1285
|
-
if (!info) return { text: `manage delete: no entry with id=${id}`, isError: true }
|
|
1286
|
-
try {
|
|
1287
|
-
const result = info.is_root === 1
|
|
1288
|
-
? await db.query(`DELETE FROM entries WHERE id = $1 OR chunk_root = $2`, [id, id])
|
|
1289
|
-
: await db.query(`DELETE FROM entries WHERE id = $1`, [id])
|
|
1290
|
-
return { text: `deleted (id=${id}, rows=${Number(result.rowCount ?? result.affectedRows ?? 0)}): [${info.category ?? '-'}] ${info.element ?? ''}` }
|
|
1291
|
-
} catch (e) {
|
|
1292
|
-
return { text: `manage delete failed: ${e.message}`, isError: true }
|
|
1293
|
-
}
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
return { text: `manage: unhandled op "${op}"`, isError: true }
|
|
1297
|
-
}
|
|
1298
|
-
|
|
1299
|
-
if (action === 'core') {
|
|
1300
|
-
const op = String(args.op ?? '').trim().toLowerCase()
|
|
1301
|
-
if (!['add', 'edit', 'delete', 'list', 'candidates', 'promote', 'dismiss'].includes(op)) {
|
|
1302
|
-
return { text: 'core requires op: "add" | "edit" | "delete" | "list" | "candidates" | "promote" | "dismiss"', isError: true }
|
|
1303
|
-
}
|
|
1304
|
-
const dataDir = (typeof DATA_DIR === 'string' ? DATA_DIR : resolvePluginData())
|
|
1305
|
-
if (!dataDir) return { text: 'core: memory data dir is not initialized', isError: true }
|
|
1306
|
-
// Core-candidate promotion pipeline (proposal mode). The candidate flag
|
|
1307
|
-
// lives on generated `entries`, which carry a project_id, so these ops MUST
|
|
1308
|
-
// be project-scoped just like add/edit/delete — an unscoped listing/promote
|
|
1309
|
-
// would leak candidates across projects. project_id resolution mirrors the
|
|
1310
|
-
// block below: 'common'/null → COMMON (project_id NULL), '*' → all pools
|
|
1311
|
-
// (candidates op only, same escape hatch as op:'list'). UI calls exactly
|
|
1312
|
-
// these op names.
|
|
1313
|
-
if (op === 'candidates' || op === 'promote' || op === 'dismiss') {
|
|
1314
|
-
const hasPid = Object.prototype.hasOwnProperty.call(args, 'project_id')
|
|
1315
|
-
const scope = (() => {
|
|
1316
|
-
if (!hasPid || args.project_id == null) return null
|
|
1317
|
-
const s = String(args.project_id).trim()
|
|
1318
|
-
if (s === '' || s.toLowerCase() === 'common') return null
|
|
1319
|
-
if (s === '*') return '*'
|
|
1320
|
-
return s
|
|
1321
|
-
})()
|
|
1322
|
-
try {
|
|
1323
|
-
if (op === 'candidates') {
|
|
1324
|
-
const list = await listCoreCandidates(dataDir, scope)
|
|
1325
|
-
if (list.length === 0) return { text: 'core candidates: none' }
|
|
1326
|
-
return {
|
|
1327
|
-
text: list.map(c =>
|
|
1328
|
-
// project=<pool> lets the UI thread project_id into the follow-up
|
|
1329
|
-
// promote/dismiss call — matters under project_id:'*' listing where
|
|
1330
|
-
// rows span pools. Uses the same COMMON/slug convention as op:'list'.
|
|
1331
|
-
`id=${c.id} project=${c.project_id == null ? 'COMMON' : c.project_id} [${c.category}] score=${c.score == null ? '-' : c.score.toFixed(2)} ${c.element} — ${String(c.summary || '').slice(0, 200)} (${c.reason})`,
|
|
1332
|
-
).join('\n'),
|
|
1333
|
-
}
|
|
1334
|
-
}
|
|
1335
|
-
// promote/dismiss operate on a single id but are scope-guarded: the
|
|
1336
|
-
// candidate must belong to the resolved scope (or COMMON), never '*'.
|
|
1337
|
-
if (scope === '*') {
|
|
1338
|
-
return { text: `core ${op}: project_id "*" only valid for op="candidates"`, isError: true }
|
|
1339
|
-
}
|
|
1340
|
-
if (op === 'promote') {
|
|
1341
|
-
const entry = await promoteCoreCandidate(dataDir, args.id, { ...args, scope })
|
|
1342
|
-
const mergeNote = entry.merged_with ? ` (merged into core id=${entry.merged_with}, sim=${entry.sim})` : ''
|
|
1343
|
-
return { text: `core promoted candidate id=${args.id} → core id=${entry.id}${mergeNote}: [${entry.category}] ${entry.element}` }
|
|
1344
|
-
}
|
|
1345
|
-
// dismiss
|
|
1346
|
-
const removed = await dismissCoreCandidate(dataDir, args.id, { scope })
|
|
1347
|
-
return { text: `core candidate dismissed (id=${removed.id}): [${removed.category}] ${removed.element}` }
|
|
1348
|
-
} catch (e) {
|
|
1349
|
-
return { text: `core ${op} failed: ${e.message}`, isError: true }
|
|
1350
|
-
}
|
|
1351
|
-
}
|
|
1352
|
-
// Local trim helper — the manage-block trimOrNull at :1807 is scoped to
|
|
1353
|
-
// that branch and unreachable from here.
|
|
1354
|
-
// Normalize project_id: 'common' (case-insensitive) or null → null (COMMON pool); non-empty string → slug.
|
|
1355
|
-
const hasProjectIdKey = Object.prototype.hasOwnProperty.call(args, 'project_id')
|
|
1356
|
-
const projectId = (() => {
|
|
1357
|
-
if (!hasProjectIdKey || args.project_id == null) return null
|
|
1358
|
-
const s = String(args.project_id).trim()
|
|
1359
|
-
if (s === '' || s.toLowerCase() === 'common') return null
|
|
1360
|
-
if (s === '*') return '*'
|
|
1361
|
-
return s
|
|
1362
|
-
})()
|
|
1363
|
-
try {
|
|
1364
|
-
if (projectId === '*' && op !== 'list') {
|
|
1365
|
-
return { text: `core ${op}: project_id "*" only valid for op="list"`, isError: true }
|
|
1366
|
-
}
|
|
1367
|
-
if (op === 'list') {
|
|
1368
|
-
if (projectId !== '*') {
|
|
1369
|
-
const entries = await listCore(dataDir, projectId)
|
|
1370
|
-
if (entries.length === 0) return { text: 'core: empty' }
|
|
1371
|
-
return { text: entries.map(e => `id=${e.id} [${e.category}] ${e.element} — ${String(e.summary || '').slice(0, 200)}`).join('\n') }
|
|
1372
|
-
}
|
|
1373
|
-
// Cross-pool listing — group by project_id, COMMON first
|
|
1374
|
-
const entries = await listCore(dataDir, '*')
|
|
1375
|
-
if (entries.length === 0) return { text: 'core: empty' }
|
|
1376
|
-
const groups = new Map()
|
|
1377
|
-
for (const e of entries) {
|
|
1378
|
-
const key = e.project_id ?? null
|
|
1379
|
-
if (!groups.has(key)) groups.set(key, [])
|
|
1380
|
-
groups.get(key).push(e)
|
|
1381
|
-
}
|
|
1382
|
-
const lines = []
|
|
1383
|
-
for (const [key, rows] of groups) {
|
|
1384
|
-
lines.push(`${key === null ? 'COMMON' : key}:`)
|
|
1385
|
-
for (const e of rows) {
|
|
1386
|
-
lines.push(` id=${e.id} [${e.category}] ${e.element} — ${String(e.summary || '').slice(0, 200)}`)
|
|
1387
|
-
}
|
|
1388
|
-
}
|
|
1389
|
-
return { text: lines.join('\n') }
|
|
1390
|
-
}
|
|
1391
|
-
if (op === 'add') {
|
|
1392
|
-
if (!hasProjectIdKey) {
|
|
1393
|
-
return { text: 'core add: project_id required — pass "common" for COMMON pool, or project slug like "owner/repo" for scoped pool', isError: true }
|
|
1394
|
-
}
|
|
1395
|
-
const entry = await addCore(dataDir, args, projectId)
|
|
1396
|
-
return { text: `core added (id=${entry.id}): [${entry.category}] ${entry.element} — ${entry.summary.slice(0, 200)}` }
|
|
1397
|
-
}
|
|
1398
|
-
if (op === 'edit') {
|
|
1399
|
-
const entry = await editCore(dataDir, args.id, args)
|
|
1400
|
-
return { text: `core edited (id=${entry.id}): [${entry.category}] ${entry.element} — ${entry.summary.slice(0, 200)}` }
|
|
1401
|
-
}
|
|
1402
|
-
if (op === 'delete') {
|
|
1403
|
-
const removed = await deleteCore(dataDir, args.id)
|
|
1404
|
-
return { text: `core deleted (id=${removed.id}): [${removed.category}] ${removed.element}` }
|
|
1405
|
-
}
|
|
1406
|
-
} catch (e) {
|
|
1407
|
-
return { text: `core ${op} failed: ${e.message}`, isError: true }
|
|
1408
|
-
}
|
|
1409
|
-
return { text: `core: unhandled op "${op}"`, isError: true }
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
if (action === 'purge') {
|
|
1413
|
-
if (args.confirm !== 'DELETE ALL MEMORY') {
|
|
1414
|
-
return { text: 'purge requires confirm: "DELETE ALL MEMORY"', isError: true }
|
|
1415
|
-
}
|
|
1416
|
-
const preCount = (await db.query(`SELECT COUNT(*) c FROM entries`)).rows[0].c
|
|
1417
|
-
const coreCount = (await db.query(`SELECT COUNT(*) c FROM core_entries`)).rows[0].c
|
|
1418
|
-
try {
|
|
1419
|
-
await db.query(`DELETE FROM entries`)
|
|
1420
|
-
} catch (e) {
|
|
1421
|
-
return { text: `purge failed: ${e.message}`, isError: true }
|
|
1422
|
-
}
|
|
1423
|
-
return { text: `purged generated memory entries (count=${preCount}); user core preserved (core_entries=${coreCount})` }
|
|
1424
|
-
}
|
|
1425
|
-
|
|
1426
|
-
if (action === 'retro_eval_active') {
|
|
1427
|
-
if (args.confirm !== 'REEVAL ACTIVE') {
|
|
1428
|
-
return { text: 'retro_eval_active requires confirm: "REEVAL ACTIVE" (heavy LLM batch op — reviews all active roots through the unified gate)', isError: true }
|
|
1429
|
-
}
|
|
1430
|
-
const RETRO_BATCH = 50
|
|
1431
|
-
const cycle2Config = config?.cycle2 || {}
|
|
1432
|
-
const allActive = (await db.query(
|
|
1433
|
-
`SELECT id, element, category, summary, score, last_seen_at, project_id, status, reviewed_at
|
|
1434
|
-
FROM entries WHERE is_root = 1 AND status = 'active'
|
|
1435
|
-
ORDER BY reviewed_at ASC, id ASC`
|
|
1436
|
-
)).rows
|
|
1437
|
-
const total = allActive.length
|
|
1438
|
-
let archived = 0, kept = 0, updated = 0, merged = 0, errors = 0
|
|
1439
|
-
const nowMs = Date.now()
|
|
1440
|
-
for (let offset = 0; offset < total; offset += RETRO_BATCH) {
|
|
1441
|
-
const batch = allActive.slice(offset, offset + RETRO_BATCH)
|
|
1442
|
-
const batchIds = batch.map(r => Number(r.id))
|
|
1443
|
-
const activeContext = (await db.query(
|
|
1444
|
-
`SELECT id, element, category, summary, score, last_seen_at, project_id, status
|
|
1445
|
-
FROM entries WHERE is_root = 1 AND status = 'active'
|
|
1446
|
-
ORDER BY score DESC, last_seen_at DESC, id ASC LIMIT 200`
|
|
1447
|
-
)).rows
|
|
1448
|
-
let gateResult
|
|
1449
|
-
try {
|
|
1450
|
-
gateResult = await runUnifiedGate(db, batch, activeContext, cycle2Config, { activeCap: 200 })
|
|
1451
|
-
} catch (err) {
|
|
1452
|
-
__mixdogMemoryLog(`[retro_eval_active] runUnifiedGate failed (offset=${offset}): ${err.message}\n`)
|
|
1453
|
-
errors += batch.length
|
|
1454
|
-
continue
|
|
1455
|
-
}
|
|
1456
|
-
if (gateResult?.parseOk === false || gateResult?.actions === null) {
|
|
1457
|
-
errors += batch.length
|
|
1458
|
-
continue
|
|
1459
|
-
}
|
|
1460
|
-
const actions = gateResult?.actions ?? []
|
|
1461
|
-
// Separate explicit `core` summary lines from primary verbs so an
|
|
1462
|
-
// update/merge/active also refreshes the injected core_summary — mirrors
|
|
1463
|
-
// the cycle2 apply path (memory-cycle2.mjs). Without this, retro could
|
|
1464
|
-
// rewrite a root's summary while leaving its core_summary stale.
|
|
1465
|
-
const coreSummaryById = new Map()
|
|
1466
|
-
const primaryActions = []
|
|
1467
|
-
for (const a of actions) {
|
|
1468
|
-
if (a?.action === 'core') {
|
|
1469
|
-
const cid = Number(a.entry_id)
|
|
1470
|
-
const core = String(a.core_summary ?? '').replace(/\s+/g, ' ').trim().slice(0, CORE_SUMMARY_MAX)
|
|
1471
|
-
if (Number.isFinite(cid) && core) coreSummaryById.set(cid, core)
|
|
1472
|
-
} else {
|
|
1473
|
-
primaryActions.push(a)
|
|
1474
|
-
}
|
|
1475
|
-
}
|
|
1476
|
-
const allowed = new Set(batchIds)
|
|
1477
|
-
const rejected = gateResult?.rejected ?? new Set()
|
|
1478
|
-
// Partial-apply contract: rows the gate never returned a verdict for
|
|
1479
|
-
// (missingIds) must NOT be marked reviewed — they are left for a later
|
|
1480
|
-
// run. Exclude both rejected and missing ids from the reviewed set.
|
|
1481
|
-
const missing = new Set((gateResult?.missingIds ?? []).map(Number))
|
|
1482
|
-
const successIds = new Set(batchIds.filter(id => !rejected.has(id) && !missing.has(id)))
|
|
1483
|
-
for (const id of successIds) {
|
|
1484
|
-
try { await db.query(`UPDATE entries SET reviewed_at = $1 WHERE id = $2`, [nowMs, id]) } catch {}
|
|
1485
|
-
}
|
|
1486
|
-
const setCoreSummary = async (entryId, core) => {
|
|
1487
|
-
if (!core) return
|
|
1488
|
-
try { await db.query(`UPDATE entries SET core_summary = $1 WHERE id = $2 AND is_root = 1`, [core, Number(entryId)]) }
|
|
1489
|
-
catch (err) { __mixdogMemoryLog(`[retro_eval_active] core_summary update failed (id=${entryId}): ${err.message}\n`) }
|
|
1490
|
-
}
|
|
1491
|
-
if (!primaryActions.length) { kept += batch.filter(r => successIds.has(Number(r.id))).length; continue }
|
|
1492
|
-
const acted = new Set()
|
|
1493
|
-
const rowById = new Map(batch.map(r => [Number(r.id), r]))
|
|
1494
|
-
for (const act of primaryActions) {
|
|
1495
|
-
try {
|
|
1496
|
-
const eid = Number(act?.entry_id)
|
|
1497
|
-
if (!Number.isFinite(eid) || !allowed.has(eid)) continue
|
|
1498
|
-
acted.add(eid)
|
|
1499
|
-
// Optimistic guard: skip if the row moved since this batch was read.
|
|
1500
|
-
const snap = rowById.get(eid)
|
|
1501
|
-
const expected = snap ? { status: snap.status, reviewedAt: snap.reviewed_at } : null
|
|
1502
|
-
if (act.action === 'archived') {
|
|
1503
|
-
if (await applySimpleStatus(db, eid, 'archived', expected)) archived += 1
|
|
1504
|
-
} else if (act.action === 'active') {
|
|
1505
|
-
// active → active is a keep verdict from the gate.
|
|
1506
|
-
kept += 1
|
|
1507
|
-
await setCoreSummary(eid, coreSummaryById.get(eid))
|
|
1508
|
-
} else if (act.action === 'update') {
|
|
1509
|
-
if (await applyUpdate(db, eid, act.element, act.summary, { expected })) updated += 1
|
|
1510
|
-
await setCoreSummary(eid, coreSummaryById.get(eid))
|
|
1511
|
-
} else if (act.action === 'merge') {
|
|
1512
|
-
const targetId = Number(act?.target_id)
|
|
1513
|
-
const sourceIds = Array.isArray(act?.source_ids) ? act.source_ids : []
|
|
1514
|
-
if (!Number.isFinite(targetId) || !allowed.has(targetId)) {
|
|
1515
|
-
__mixdogMemoryLog(`[retro_eval_active] merge target outside batch (id=${targetId})\n`)
|
|
1516
|
-
acted.delete(eid)
|
|
1517
|
-
continue
|
|
1518
|
-
}
|
|
1519
|
-
const filteredSources = sourceIds.filter(s => allowed.has(Number(s)))
|
|
1520
|
-
if (filteredSources.length !== sourceIds.length) {
|
|
1521
|
-
__mixdogMemoryLog(
|
|
1522
|
-
`[retro_eval_active] merge sources filtered: ${JSON.stringify(sourceIds)} -> ${JSON.stringify(filteredSources)}\n`,
|
|
1523
|
-
)
|
|
1524
|
-
}
|
|
1525
|
-
acted.add(targetId)
|
|
1526
|
-
filteredSources.forEach(s => acted.add(Number(s)))
|
|
1527
|
-
const moved = await applyMerge(db, targetId, filteredSources)
|
|
1528
|
-
if (moved > 0) {
|
|
1529
|
-
merged += moved
|
|
1530
|
-
if (typeof act.element === 'string' || typeof act.summary === 'string') {
|
|
1531
|
-
try {
|
|
1532
|
-
if (await applyUpdate(db, targetId, act.element, act.summary)) updated += 1
|
|
1533
|
-
} catch (err) {
|
|
1534
|
-
__mixdogMemoryLog(`[retro_eval_active] merge target update failed (target=${targetId}): ${err.message}\n`)
|
|
1535
|
-
}
|
|
1536
|
-
}
|
|
1537
|
-
await setCoreSummary(targetId, coreSummaryById.get(targetId) || coreSummaryById.get(eid))
|
|
1538
|
-
}
|
|
1539
|
-
}
|
|
1540
|
-
} catch (err) {
|
|
1541
|
-
__mixdogMemoryLog(`[retro_eval_active] action error (id=${act?.entry_id}): ${err.message}\n`)
|
|
1542
|
-
errors += 1
|
|
1543
|
-
}
|
|
1544
|
-
}
|
|
1545
|
-
// Entries in successIds but not acted-upon (omit / no-op) are kept.
|
|
1546
|
-
kept += batch.filter(r => successIds.has(Number(r.id)) && !acted.has(Number(r.id))).length
|
|
1547
|
-
}
|
|
1548
|
-
return { text: `retro_eval_active: total=${total} archived=${archived} kept=${kept} updated=${updated} merged=${merged} errors=${errors}` }
|
|
1549
|
-
}
|
|
1550
|
-
|
|
1551
|
-
return { text: `unknown memory action: ${action}`, isError: true }
|
|
1552
|
-
}
|
|
1553
|
-
|
|
1554
|
-
async function handleToolCall(name, args, signal) {
|
|
1555
|
-
try {
|
|
1556
|
-
if (name === 'search_memories') {
|
|
1557
|
-
const result = await handleSearch(args || {}, signal)
|
|
1558
|
-
return { ...result, content: [{ type: 'text', text: result.text }], isError: result.isError || false }
|
|
1559
|
-
}
|
|
1560
|
-
if (name === 'recall') {
|
|
1561
|
-
// recall is aiWrapped in the unified build; in standalone mode map it to
|
|
1562
|
-
// search_memories so the advertised tool name actually works. Forward
|
|
1563
|
-
// every advertised arg so id/limit/offset/sort/includeArchived/
|
|
1564
|
-
// includeMembers/includeRaw reach handleSearch instead of being dropped.
|
|
1565
|
-
const a = args || {}
|
|
1566
|
-
const hasQuery = Array.isArray(a.query)
|
|
1567
|
-
? a.query.some((value) => String(value || '').trim())
|
|
1568
|
-
: String(a.query ?? '').trim() !== ''
|
|
1569
|
-
const recallIds = hasQuery
|
|
1570
|
-
? []
|
|
1571
|
-
: (Array.isArray(a.id) ? a.id : [a.id])
|
|
1572
|
-
.map((value) => Number(value))
|
|
1573
|
-
.filter((value) => Number.isInteger(value) && value > 0)
|
|
1574
|
-
const searchArgs = {
|
|
1575
|
-
...(a.query !== undefined ? { query: a.query } : {}),
|
|
1576
|
-
...(recallIds.length > 0 ? { ids: recallIds } : {}),
|
|
1577
|
-
...(a.period ? { period: a.period } : {}),
|
|
1578
|
-
...(a.limit !== undefined ? { limit: a.limit } : {}),
|
|
1579
|
-
...(a.offset !== undefined ? { offset: a.offset } : {}),
|
|
1580
|
-
...(a.sort !== undefined ? { sort: a.sort } : {}),
|
|
1581
|
-
...(a.category !== undefined ? { category: a.category } : {}),
|
|
1582
|
-
...(a.includeArchived !== undefined ? { includeArchived: a.includeArchived } : {}),
|
|
1583
|
-
...(a.includeMembers !== undefined ? { includeMembers: a.includeMembers } : {}),
|
|
1584
|
-
...(a.includeRaw !== undefined ? { includeRaw: a.includeRaw } : {}),
|
|
1585
|
-
...(a.cwd ? { cwd: a.cwd } : {}),
|
|
1586
|
-
...(a.projectScope ? { projectScope: a.projectScope } : {}),
|
|
1587
|
-
...(a.sessionId ? { sessionId: a.sessionId } : {}),
|
|
1588
|
-
...(a.session_id ? { session_id: a.session_id } : {}),
|
|
1589
|
-
...(a.currentSession !== undefined ? { currentSession: a.currentSession } : {}),
|
|
1590
|
-
// Hint only — never a filter. Marks the caller's own session as
|
|
1591
|
-
// "(current)" in the multi-session grouped browse output.
|
|
1592
|
-
...(a.currentSessionId ? { currentSessionId: a.currentSessionId } : {}),
|
|
1593
|
-
}
|
|
1594
|
-
const result = await handleSearch(searchArgs, signal)
|
|
1595
|
-
return { ...result, content: [{ type: 'text', text: result.text }], isError: result.isError || false }
|
|
1596
|
-
}
|
|
1597
|
-
if (name === 'memory') {
|
|
1598
|
-
const result = await handleMemoryAction(args || {}, signal)
|
|
1599
|
-
return { ...result, content: [{ type: 'text', text: result.text }], isError: result.isError || false }
|
|
1600
|
-
}
|
|
1601
|
-
return { content: [{ type: 'text', text: `unknown tool: ${name}` }], isError: true }
|
|
1602
|
-
} catch (err) {
|
|
1603
|
-
const msg = err instanceof Error ? err.message : String(err)
|
|
1604
|
-
return { content: [{ type: 'text', text: `${name} failed: ${msg}` }], isError: true }
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
695
|
+
// ── Memory action + tool-call handlers (extracted to
|
|
696
|
+
// lib/memory-action-handlers.mjs). The facade keeps db/scheduler ownership and
|
|
697
|
+
// injects live getters plus the query/ingest/cycle primitives.
|
|
698
|
+
const _actionHandlers = createMemoryActionHandlers({
|
|
699
|
+
getDb: () => db,
|
|
700
|
+
dataDir: DATA_DIR,
|
|
701
|
+
log: __mixdogMemoryLog,
|
|
702
|
+
readMainConfig,
|
|
703
|
+
getCycleLastRun,
|
|
704
|
+
ingestSessionMessages,
|
|
705
|
+
entryStats,
|
|
706
|
+
handleSearch,
|
|
707
|
+
dumpSessionRootChunks,
|
|
708
|
+
awaitCycle1Run: _awaitCycle1Run,
|
|
709
|
+
startCycle1Run: _startCycle1Run,
|
|
710
|
+
finalizeCycle2Run: _finalizeCycle2Run,
|
|
711
|
+
finalizeCycle3Run: _finalizeCycle3Run,
|
|
712
|
+
getSchedulerCycle1InFlight: () => _cycleScheduler.getCycle1InFlight(),
|
|
713
|
+
getCycle2CallLlm,
|
|
714
|
+
getCycle3CallLlm,
|
|
715
|
+
ingestTranscriptFile,
|
|
716
|
+
cwdFromTranscriptPath,
|
|
717
|
+
})
|
|
718
|
+
const { handleMemoryAction, handleToolCall } = _actionHandlers
|
|
1607
719
|
|
|
1608
720
|
const mcp = new Server(
|
|
1609
721
|
{ name: 'mixdog-memory', version: PLUGIN_VERSION },
|
|
@@ -1612,715 +724,36 @@ const mcp = new Server(
|
|
|
1612
724
|
mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOL_DEFS }))
|
|
1613
725
|
mcp.setRequestHandler(CallToolRequestSchema, (req) => handleToolCall(req.params.name, req.params.arguments ?? {}))
|
|
1614
726
|
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
)
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
}
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
const generatedScopeClause = projectId !== null
|
|
1643
|
-
? `project_id IS NULL OR project_id = $1`
|
|
1644
|
-
: `project_id IS NULL`
|
|
1645
|
-
const dbRows = (await db.query(`
|
|
1646
|
-
SELECT core_summary
|
|
1647
|
-
FROM entries
|
|
1648
|
-
WHERE is_root = 1
|
|
1649
|
-
AND status = 'active'
|
|
1650
|
-
AND core_summary IS NOT NULL
|
|
1651
|
-
AND (${generatedScopeClause})
|
|
1652
|
-
ORDER BY score DESC, last_seen_at DESC
|
|
1653
|
-
`, projectId !== null ? [projectId] : [])).rows
|
|
1654
|
-
const commonRows = (await db.query(
|
|
1655
|
-
`SELECT summary FROM core_entries WHERE project_id IS NULL AND (status IS NULL OR status = 'active') ORDER BY id ASC`
|
|
1656
|
-
)).rows
|
|
1657
|
-
const scopedRows = projectId !== null
|
|
1658
|
-
? (await db.query(
|
|
1659
|
-
`SELECT summary FROM core_entries WHERE project_id = $1 AND (status IS NULL OR status = 'active') ORDER BY id ASC`,
|
|
1660
|
-
[projectId]
|
|
1661
|
-
)).rows
|
|
1662
|
-
: []
|
|
1663
|
-
return {
|
|
1664
|
-
projectId,
|
|
1665
|
-
dbLines: dbRows.map(r => String(r.core_summary || '').trim()).filter(Boolean),
|
|
1666
|
-
userLines: [
|
|
1667
|
-
...commonRows.map(r => String(r.summary || '').trim()).filter(Boolean),
|
|
1668
|
-
...scopedRows.map(r => String(r.summary || '').trim()).filter(Boolean),
|
|
1669
|
-
],
|
|
1670
|
-
}
|
|
1671
|
-
}
|
|
1672
|
-
|
|
1673
|
-
// Whole-action backfill mutex. memory-cycle1's _cycle1InFlight only protects
|
|
1674
|
-
// cycle1; ingest workers (memory-ops-policy.mjs) and cycle2 can still overlap
|
|
1675
|
-
// if a second backfill kicks in (e.g. setup-server timeout + retry). Track the
|
|
1676
|
-
// in-flight promise here and reject overlaps with 409.
|
|
1677
|
-
let _backfillInFlight = null
|
|
1678
|
-
|
|
1679
|
-
// Owner-side /api/tool in-flight controllers keyed by caller-supplied
|
|
1680
|
-
// X-Mixdog-Call-Id. /api/cancel aborts the matching AbortSignal so the
|
|
1681
|
-
// upstream handleToolCall actually stops when the fork-proxy parent cancels.
|
|
1682
|
-
const _ownerInFlightHttpCalls = new Map()
|
|
1683
|
-
|
|
1684
|
-
const httpServer = http.createServer(async (req, res) => {
|
|
1685
|
-
touchDaemonIdleTimer(`${req.method || 'HTTP'} ${req.url || '/'}`)
|
|
1686
|
-
if (req.method === 'POST' && req.url === '/session-reset') {
|
|
1687
|
-
_bootTimestamp = Date.now()
|
|
1688
|
-
sendJson(res, { ok: true, bootTimestamp: _bootTimestamp })
|
|
1689
|
-
return
|
|
1690
|
-
}
|
|
1691
|
-
if (req.method === 'POST' && req.url === '/rebind') {
|
|
1692
|
-
_bootTimestamp = Date.now()
|
|
1693
|
-
sendJson(res, { ok: true })
|
|
1694
|
-
return
|
|
1695
|
-
}
|
|
1696
|
-
|
|
1697
|
-
if (req.method === 'GET' && req.url === '/health') {
|
|
1698
|
-
if (!_initialized) {
|
|
1699
|
-
sendJson(res, { status: 'starting' }, 503)
|
|
1700
|
-
return
|
|
1701
|
-
}
|
|
1702
|
-
try {
|
|
1703
|
-
const stats = await entryStats()
|
|
1704
|
-
sendJson(res, {
|
|
1705
|
-
status: 'ok',
|
|
1706
|
-
worker_pid: process.pid,
|
|
1707
|
-
server_pid: Number(process.env.MIXDOG_SERVER_PID) || null,
|
|
1708
|
-
owner_lead_pid: Number(process.env.MIXDOG_OWNER_LEAD_PID) || null,
|
|
1709
|
-
code_fingerprint: BOOT_PROMOTION_CODE_FINGERPRINT,
|
|
1710
|
-
bootstrap: await isBootstrapComplete(db),
|
|
1711
|
-
entries: stats.total,
|
|
1712
|
-
roots: stats.roots,
|
|
1713
|
-
active_roots: stats.active_roots,
|
|
1714
|
-
archived_roots: stats.archived_roots,
|
|
1715
|
-
unchunked_leaves: stats.unchunked_leaves,
|
|
1716
|
-
cycle2_pending_roots: stats.cycle2_pending_roots,
|
|
1717
|
-
core_entries: stats.core_entries,
|
|
1718
|
-
core_embed_null: stats.core_embed_null,
|
|
1719
|
-
active_core_summaries: stats.active_core_summaries,
|
|
1720
|
-
active_core_summary_missing: stats.active_core_summary_missing,
|
|
1721
|
-
mv_hot_active_populated: stats.mv_hot_active_populated,
|
|
1722
|
-
cycle_running: _cycleScheduler.getCycleRunning(),
|
|
1723
|
-
cycle_health: _cycleScheduler.getCycleHealth(),
|
|
1724
|
-
cycle_backlog: _cycleScheduler.getCycleBacklogSnapshot(),
|
|
1725
|
-
})
|
|
1726
|
-
} catch (e) { sendError(res, e.message) }
|
|
1727
|
-
return
|
|
1728
|
-
}
|
|
1729
|
-
|
|
1730
|
-
if (!await awaitRuntimeReadyForHttp(res)) return
|
|
1731
|
-
|
|
1732
|
-
if (req.method === 'GET' && req.url === '/admin/entries/active') {
|
|
1733
|
-
try {
|
|
1734
|
-
const { rows } = await db.query(`
|
|
1735
|
-
SELECT id, element, category, summary, score, last_seen_at
|
|
1736
|
-
FROM entries
|
|
1737
|
-
WHERE is_root = 1 AND status = 'active'
|
|
1738
|
-
ORDER BY score DESC
|
|
1739
|
-
`)
|
|
1740
|
-
sendJson(res, { ok: true, items: rows })
|
|
1741
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1742
|
-
return
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
if (req.method === 'GET' && req.url === '/admin/core/entries') {
|
|
1746
|
-
try {
|
|
1747
|
-
const rows = await listCore(DATA_DIR, '*')
|
|
1748
|
-
sendJson(res, { ok: true, items: rows })
|
|
1749
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1750
|
-
return
|
|
1751
|
-
}
|
|
1752
|
-
|
|
1753
|
-
if (req.method === 'POST' && req.url === '/admin/core/entries') {
|
|
1754
|
-
if (!isLocalOrigin(req)) {
|
|
1755
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1756
|
-
return
|
|
1757
|
-
}
|
|
1758
|
-
try {
|
|
1759
|
-
const body = await readBody(req)
|
|
1760
|
-
const projectId = normalizeCoreProjectId(body.project_id)
|
|
1761
|
-
const entry = await addCore(DATA_DIR, body, projectId)
|
|
1762
|
-
sendJson(res, { ok: true, item: entry })
|
|
1763
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1764
|
-
return
|
|
1765
|
-
}
|
|
1766
|
-
|
|
1767
|
-
if (req.method === 'POST' && req.url === '/admin/core/entries/delete') {
|
|
1768
|
-
if (!isLocalOrigin(req)) {
|
|
1769
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1770
|
-
return
|
|
1771
|
-
}
|
|
1772
|
-
try {
|
|
1773
|
-
const body = await readBody(req)
|
|
1774
|
-
const removed = await deleteCore(DATA_DIR, body.id)
|
|
1775
|
-
sendJson(res, { ok: true, item: removed })
|
|
1776
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1777
|
-
return
|
|
1778
|
-
}
|
|
1779
|
-
|
|
1780
|
-
if (req.method === 'POST' && req.url === '/admin/entries/status') {
|
|
1781
|
-
if (!isLocalOrigin(req)) {
|
|
1782
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1783
|
-
return
|
|
1784
|
-
}
|
|
1785
|
-
try {
|
|
1786
|
-
const body = await readBody(req)
|
|
1787
|
-
const id = Number(body.id)
|
|
1788
|
-
const status = String(body.status ?? '').trim().toLowerCase()
|
|
1789
|
-
const VALID = ['pending', 'active', 'archived']
|
|
1790
|
-
if (!Number.isInteger(id) || id <= 0 || !VALID.includes(status)) {
|
|
1791
|
-
sendJson(res, { ok: false, error: 'valid id and status required' }, 400)
|
|
1792
|
-
return
|
|
1793
|
-
}
|
|
1794
|
-
const result = await db.query(
|
|
1795
|
-
`UPDATE entries SET status = $1 WHERE id = $2 AND is_root = 1`,
|
|
1796
|
-
[status, id]
|
|
1797
|
-
)
|
|
1798
|
-
sendJson(res, { ok: true, changes: Number(result.rowCount ?? result.affectedRows ?? 0) })
|
|
1799
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1800
|
-
return
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
|
-
if (req.method === 'POST' && req.url === '/admin/entries/add') {
|
|
1804
|
-
if (!isLocalOrigin(req)) {
|
|
1805
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1806
|
-
return
|
|
1807
|
-
}
|
|
1808
|
-
try {
|
|
1809
|
-
const body = await readBody(req)
|
|
1810
|
-
const result = await handleMemoryAction({
|
|
1811
|
-
action: 'manage',
|
|
1812
|
-
op: 'add',
|
|
1813
|
-
element: body.element,
|
|
1814
|
-
summary: body.summary,
|
|
1815
|
-
category: body.category,
|
|
1816
|
-
cwd: body.cwd,
|
|
1817
|
-
})
|
|
1818
|
-
if (result.isError) {
|
|
1819
|
-
sendJson(res, { ok: false, error: result.text }, 400)
|
|
1820
|
-
return
|
|
1821
|
-
}
|
|
1822
|
-
const idMatch = String(result.text || '').match(/id=(\d+)/)
|
|
1823
|
-
const newId = idMatch ? Number(idMatch[1]) : null
|
|
1824
|
-
sendJson(res, { ok: true, id: newId, text: result.text })
|
|
1825
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1826
|
-
return
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
|
-
if (req.method === 'POST' && req.url === '/admin/backfill') {
|
|
1830
|
-
if (!isLocalOrigin(req)) {
|
|
1831
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1832
|
-
return
|
|
1833
|
-
}
|
|
1834
|
-
let body
|
|
1835
|
-
try { body = await readBody(req) }
|
|
1836
|
-
catch (e) { sendJson(res, { ok: false, error: e.message }, Number(e?.statusCode) || 500); return }
|
|
1837
|
-
try {
|
|
1838
|
-
const result = await handleMemoryAction({
|
|
1839
|
-
action: 'backfill',
|
|
1840
|
-
window: body.window,
|
|
1841
|
-
scope: body.scope,
|
|
1842
|
-
limit: body.limit,
|
|
1843
|
-
})
|
|
1844
|
-
if (result.isError) {
|
|
1845
|
-
// 'backfill already in progress' → 409, other failures → 500
|
|
1846
|
-
const status = result.text === 'backfill already in progress' ? 409 : 500
|
|
1847
|
-
sendJson(res, { ok: false, error: result.text }, status)
|
|
1848
|
-
return
|
|
1849
|
-
}
|
|
1850
|
-
sendJson(res, { ok: true, text: result.text })
|
|
1851
|
-
} catch (e) {
|
|
1852
|
-
sendJson(res, { ok: false, error: e.message }, 500)
|
|
1853
|
-
}
|
|
1854
|
-
return
|
|
1855
|
-
}
|
|
1856
|
-
|
|
1857
|
-
if (req.method === 'POST' && req.url === '/admin/purge') {
|
|
1858
|
-
if (!isLocalOrigin(req)) {
|
|
1859
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1860
|
-
return
|
|
1861
|
-
}
|
|
1862
|
-
try {
|
|
1863
|
-
const body = await readBody(req)
|
|
1864
|
-
if (body?.confirm !== 'DELETE ALL MEMORY') {
|
|
1865
|
-
sendJson(res, { ok: false, error: 'confirm must be exactly "DELETE ALL MEMORY"' }, 400)
|
|
1866
|
-
return
|
|
1867
|
-
}
|
|
1868
|
-
const { rows: countRows } = await db.query(`SELECT COUNT(*) AS c FROM entries`)
|
|
1869
|
-
const preCount = Number(countRows[0].c)
|
|
1870
|
-
const { rows: coreCountRows } = await db.query(`SELECT COUNT(*) AS c FROM core_entries`)
|
|
1871
|
-
const coreCount = Number(coreCountRows[0].c)
|
|
1872
|
-
await db.transaction(async (tx) => {
|
|
1873
|
-
await tx.query(`DELETE FROM entries`)
|
|
1874
|
-
})
|
|
1875
|
-
sendJson(res, { ok: true, deleted: preCount, core_preserved: coreCount })
|
|
1876
|
-
} catch (e) { sendJson(res, { ok: false, error: e.message }, 500) }
|
|
1877
|
-
return
|
|
1878
|
-
}
|
|
1879
|
-
|
|
1880
|
-
if (req.method === 'POST' && req.url === '/admin/trace-record') {
|
|
1881
|
-
if (!isLocalOrigin(req)) {
|
|
1882
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1883
|
-
return
|
|
1884
|
-
}
|
|
1885
|
-
let body
|
|
1886
|
-
try { body = await readBody(req) }
|
|
1887
|
-
catch (e) { sendJson(res, { ok: false, error: e.message }, 400); return }
|
|
1888
|
-
if (!Array.isArray(body?.events)) {
|
|
1889
|
-
sendJson(res, { ok: false, error: 'body.events must be an array' }, 400)
|
|
1890
|
-
return
|
|
1891
|
-
}
|
|
1892
|
-
if (body.events.length > 500) {
|
|
1893
|
-
sendJson(res, { ok: false, error: 'too many events (max 500)' }, 413)
|
|
1894
|
-
return
|
|
1895
|
-
}
|
|
1896
|
-
if (!_traceDb) {
|
|
1897
|
-
try {
|
|
1898
|
-
_traceDb = await openTraceDatabase(DATA_DIR)
|
|
1899
|
-
registerTraceExitDrain(_traceDb)
|
|
1900
|
-
} catch (e) {
|
|
1901
|
-
sendJson(res, { ok: false, error: `trace DB unavailable: ${e.message}` }, 503)
|
|
1902
|
-
return
|
|
1903
|
-
}
|
|
1904
|
-
}
|
|
1905
|
-
try {
|
|
1906
|
-
// Enqueue for async batched flush (100ms / 500-row window).
|
|
1907
|
-
enqueueTraceEvents(_traceDb, body.events)
|
|
1908
|
-
// Use `queued` — events are async; `inserted` would imply durability.
|
|
1909
|
-
sendJson(res, { ok: true, queued: body.events.length })
|
|
1910
|
-
// Fire-and-forget into focused agent analytic tables.
|
|
1911
|
-
insertAgentCalls(_traceDb, body.events).catch(e =>
|
|
1912
|
-
__mixdogMemoryLog(`[trace] insertAgentCalls error: ${e?.message}\n`)
|
|
1913
|
-
)
|
|
1914
|
-
} catch (e) {
|
|
1915
|
-
sendJson(res, { ok: false, error: e.message }, 500)
|
|
1916
|
-
}
|
|
1917
|
-
return
|
|
1918
|
-
}
|
|
1919
|
-
|
|
1920
|
-
if (req.method === 'POST' && req.url === '/session-start/core-memory') {
|
|
1921
|
-
try {
|
|
1922
|
-
const body = await readBody(req)
|
|
1923
|
-
const { projectId, dbLines, userLines } = await buildSessionCoreMemoryPayload(body.cwd)
|
|
1924
|
-
sendJson(res, { ok: true, projectId, dbLines, userLines })
|
|
1925
|
-
} catch (e) { sendError(res, e.message) }
|
|
1926
|
-
return
|
|
1927
|
-
}
|
|
1928
|
-
|
|
1929
|
-
if (req.method === 'POST' && req.url === '/admin/shutdown') {
|
|
1930
|
-
if (!isLocalOrigin(req)) {
|
|
1931
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1932
|
-
return
|
|
1933
|
-
}
|
|
1934
|
-
sendJson(res, { shutting_down: true }, 202)
|
|
1935
|
-
setImmediate(() => {
|
|
1936
|
-
const watchdog = setTimeout(() => {
|
|
1937
|
-
__mixdogMemoryLog('[shutdown] watchdog fired — forcing exit after 8s\n')
|
|
1938
|
-
process.exit(1)
|
|
1939
|
-
}, 8000)
|
|
1940
|
-
watchdog.unref?.()
|
|
1941
|
-
stop()
|
|
1942
|
-
.then(() => { clearTimeout(watchdog); process.exit(0) })
|
|
1943
|
-
.catch(e => {
|
|
1944
|
-
__mixdogMemoryLog(`[shutdown] error ${e.message}\n`)
|
|
1945
|
-
clearTimeout(watchdog)
|
|
1946
|
-
process.exit(1)
|
|
1947
|
-
})
|
|
1948
|
-
})
|
|
1949
|
-
return
|
|
1950
|
-
}
|
|
1951
|
-
|
|
1952
|
-
// DEV-ONLY cycle1 chunking bench. Gated by env MIXDOG_DEV_BENCH=1 so
|
|
1953
|
-
// production is untouched (route returns 404 when unset). Mirrors cycle1's
|
|
1954
|
-
// exact fetch query + per-session windowing, then runs each window through
|
|
1955
|
-
// buildCycle1ChunkPrompt + callAgentDispatch + parseCycle1LineFormat. STRICT
|
|
1956
|
-
// read-only — no UPDATE, no transaction, no commit.
|
|
1957
|
-
if (req.method === 'POST' && req.url === '/dev/cycle1-bench') {
|
|
1958
|
-
// Gate: env MIXDOG_DEV_BENCH=1 OR a runtime flag file, so it can be
|
|
1959
|
-
// toggled without restarting the host agent (env only reaches the worker
|
|
1960
|
-
// on a full CC restart, not via dev-sync full-restart).
|
|
1961
|
-
const _devBenchOn = process.env.MIXDOG_DEV_BENCH === '1'
|
|
1962
|
-
|| (DATA_DIR && fs.existsSync(path.join(DATA_DIR, '.dev-bench-enabled')))
|
|
1963
|
-
if (!_devBenchOn) {
|
|
1964
|
-
sendJson(res, { error: 'not found' }, 404)
|
|
1965
|
-
return
|
|
1966
|
-
}
|
|
1967
|
-
if (!isLocalOrigin(req)) {
|
|
1968
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
1969
|
-
return
|
|
1970
|
-
}
|
|
1971
|
-
try {
|
|
1972
|
-
const body = await readBody(req)
|
|
1973
|
-
const sets = Math.max(1, Number(body?.sets ?? 5))
|
|
1974
|
-
const repeat = Math.max(1, Number(body?.repeat ?? 1))
|
|
1975
|
-
// Optional variant matrix. Each variant: {name, rules}. rules=null → default prompt.
|
|
1976
|
-
const rawVariants = Array.isArray(body?.variants) ? body.variants : null
|
|
1977
|
-
const variants = rawVariants && rawVariants.length > 0
|
|
1978
|
-
? rawVariants.map((v, i) => ({
|
|
1979
|
-
name: typeof v?.name === 'string' && v.name ? v.name : `variant-${i + 1}`,
|
|
1980
|
-
rules: Array.isArray(v?.rules) ? v.rules : null,
|
|
1981
|
-
}))
|
|
1982
|
-
: null
|
|
1983
|
-
|
|
1984
|
-
// Lazy-load LLM + chunking helpers so production boot pays nothing.
|
|
1985
|
-
// Use the same in-process agent dispatch adapter as real cycle1 — the legacy
|
|
1986
|
-
// agent-ipc callAgentDispatch() path is dead in the detached standalone
|
|
1987
|
-
// memory daemon (no connected IPC), so the dev bench must mirror prod.
|
|
1988
|
-
const [{ buildCycle1ChunkPrompt, parseCycle1LineFormat }, { resolveMaintenancePreset }] = await Promise.all([
|
|
1989
|
-
import('./lib/memory-cycle1.mjs'),
|
|
1990
|
-
import('../shared/llm/index.mjs'),
|
|
1991
|
-
])
|
|
1992
|
-
const benchCallLlm = getCycle1CallLlm()
|
|
1993
|
-
|
|
1994
|
-
const CYCLE1_MIN_BATCH = 3
|
|
1995
|
-
const CYCLE1_SESSION_CAP = 10
|
|
1996
|
-
const BATCH_SIZE = 100
|
|
1997
|
-
const TIMEOUT_MS = 180_000
|
|
1998
|
-
const fetchLimit = CYCLE1_SESSION_CAP * BATCH_SIZE
|
|
1999
|
-
|
|
2000
|
-
const fetchResult = await db.query(
|
|
2001
|
-
`SELECT id, ts, role, content, session_id, source_ref, project_id
|
|
2002
|
-
FROM entries
|
|
2003
|
-
WHERE chunk_root IS NULL AND session_id IS NOT NULL
|
|
2004
|
-
ORDER BY ts DESC, id DESC
|
|
2005
|
-
LIMIT $1`,
|
|
2006
|
-
[fetchLimit],
|
|
2007
|
-
)
|
|
2008
|
-
const rowsDesc = fetchResult.rows
|
|
2009
|
-
|
|
2010
|
-
if (rowsDesc.length < CYCLE1_MIN_BATCH) {
|
|
2011
|
-
sendJson(res, {
|
|
2012
|
-
ok: true,
|
|
2013
|
-
sets, repeat,
|
|
2014
|
-
windowsAvailable: 0,
|
|
2015
|
-
note: `not enough pending rows (need >= ${CYCLE1_MIN_BATCH}, got ${rowsDesc.length})`,
|
|
2016
|
-
results: [],
|
|
2017
|
-
})
|
|
2018
|
-
return
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
// Partition by session_id — same as memory-cycle1.mjs _runCycle1Impl L207-233.
|
|
2022
|
-
const sessionMap = new Map()
|
|
2023
|
-
for (const row of rowsDesc.slice().reverse()) {
|
|
2024
|
-
const sid = row.session_id
|
|
2025
|
-
if (!sessionMap.has(sid)) sessionMap.set(sid, [])
|
|
2026
|
-
sessionMap.get(sid).push(row)
|
|
2027
|
-
}
|
|
2028
|
-
const windows = []
|
|
2029
|
-
for (const [sid, sessionRows] of sessionMap) {
|
|
2030
|
-
if (sessionRows.length < CYCLE1_MIN_BATCH) continue
|
|
2031
|
-
const windowCount = Math.max(1, Math.ceil(sessionRows.length / BATCH_SIZE))
|
|
2032
|
-
const baseSize = Math.floor(sessionRows.length / windowCount)
|
|
2033
|
-
const remainder = sessionRows.length % windowCount
|
|
2034
|
-
let _offset = 0
|
|
2035
|
-
for (let i = 0; i < windowCount; i++) {
|
|
2036
|
-
const size = baseSize + (i < remainder ? 1 : 0)
|
|
2037
|
-
windows.push({ sid, rows: sessionRows.slice(_offset, _offset + size) })
|
|
2038
|
-
_offset += size
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
const chosen = windows.slice(0, sets)
|
|
2042
|
-
|
|
2043
|
-
const preset = resolveMaintenancePreset('memory')
|
|
2044
|
-
|
|
2045
|
-
function summariseChunks(chunks, totalEntries) {
|
|
2046
|
-
const usedIdx = new Set()
|
|
2047
|
-
for (const c of chunks) for (const i of (c._idxList || [])) usedIdx.add(i)
|
|
2048
|
-
const omitted = []
|
|
2049
|
-
for (let i = 1; i <= totalEntries; i++) if (!usedIdx.has(i)) omitted.push(i)
|
|
2050
|
-
return { covered: usedIdx.size, omitted }
|
|
2051
|
-
}
|
|
2052
|
-
|
|
2053
|
-
// When variants are absent, fall back to a single implicit baseline so the
|
|
2054
|
-
// pre-variant call shape (single rows × repeat) keeps producing the same
|
|
2055
|
-
// {runs:[…]} payload the trigger already knows how to print.
|
|
2056
|
-
const variantList = variants ?? [{ name: 'baseline', rules: null }]
|
|
2057
|
-
|
|
2058
|
-
async function runOnce(rows, customRules) {
|
|
2059
|
-
const userMessage = buildCycle1ChunkPrompt(rows, customRules)
|
|
2060
|
-
const t0 = Date.now()
|
|
2061
|
-
let raw, error
|
|
2062
|
-
try {
|
|
2063
|
-
raw = await benchCallLlm({
|
|
2064
|
-
preset,
|
|
2065
|
-
timeout: TIMEOUT_MS,
|
|
2066
|
-
}, userMessage)
|
|
2067
|
-
} catch (e) {
|
|
2068
|
-
error = e?.message ?? String(e)
|
|
2069
|
-
}
|
|
2070
|
-
const llmMs = Date.now() - t0
|
|
2071
|
-
if (error) return { ok: false, llmMs, error }
|
|
2072
|
-
const parsed = parseCycle1LineFormat(raw)
|
|
2073
|
-
const chunks = Array.isArray(parsed?.chunks) ? parsed.chunks : []
|
|
2074
|
-
const { covered, omitted } = summariseChunks(chunks, rows.length)
|
|
2075
|
-
const ratio = chunks.length > 0
|
|
2076
|
-
? parseFloat((rows.length / chunks.length).toFixed(2))
|
|
2077
|
-
: null
|
|
2078
|
-
return {
|
|
2079
|
-
ok: true,
|
|
2080
|
-
llmMs,
|
|
2081
|
-
entries: rows.length,
|
|
2082
|
-
chunks: chunks.length,
|
|
2083
|
-
ratio,
|
|
2084
|
-
covered,
|
|
2085
|
-
omitted,
|
|
2086
|
-
chunkList: chunks.map(c => ({
|
|
2087
|
-
idx: c._idxList,
|
|
2088
|
-
element: c.element,
|
|
2089
|
-
category: c.category,
|
|
2090
|
-
summary: c.summary,
|
|
2091
|
-
})),
|
|
2092
|
-
}
|
|
2093
|
-
}
|
|
2094
|
-
|
|
2095
|
-
const results = []
|
|
2096
|
-
for (let s = 0; s < chosen.length; s++) {
|
|
2097
|
-
const { sid, rows } = chosen[s]
|
|
2098
|
-
const sidShort = String(sid).slice(0, 8)
|
|
2099
|
-
if (variants) {
|
|
2100
|
-
// Variant mode: same rows, one run per variant per repeat.
|
|
2101
|
-
const variantResults = []
|
|
2102
|
-
for (const v of variantList) {
|
|
2103
|
-
const runs = []
|
|
2104
|
-
for (let r = 0; r < repeat; r++) {
|
|
2105
|
-
const run = await runOnce(rows, v.rules)
|
|
2106
|
-
runs.push({ repIdx: r + 1, ...run })
|
|
2107
|
-
}
|
|
2108
|
-
variantResults.push({ name: v.name, runs })
|
|
2109
|
-
}
|
|
2110
|
-
results.push({
|
|
2111
|
-
setIdx: s + 1,
|
|
2112
|
-
sessionIdShort: sidShort,
|
|
2113
|
-
entries: rows.length,
|
|
2114
|
-
variants: variantResults,
|
|
2115
|
-
})
|
|
2116
|
-
} else {
|
|
2117
|
-
// Legacy single-baseline payload shape.
|
|
2118
|
-
const runs = []
|
|
2119
|
-
for (let r = 0; r < repeat; r++) {
|
|
2120
|
-
const run = await runOnce(rows, null)
|
|
2121
|
-
runs.push({ repIdx: r + 1, ...run })
|
|
2122
|
-
}
|
|
2123
|
-
results.push({
|
|
2124
|
-
setIdx: s + 1,
|
|
2125
|
-
sessionIdShort: sidShort,
|
|
2126
|
-
entries: rows.length,
|
|
2127
|
-
runs,
|
|
2128
|
-
})
|
|
2129
|
-
}
|
|
2130
|
-
}
|
|
2131
|
-
sendJson(res, {
|
|
2132
|
-
ok: true,
|
|
2133
|
-
sets, repeat,
|
|
2134
|
-
windowsAvailable: windows.length,
|
|
2135
|
-
variants: variants ? variantList.map(v => v.name) : null,
|
|
2136
|
-
results,
|
|
2137
|
-
})
|
|
2138
|
-
} catch (e) {
|
|
2139
|
-
sendError(res, e?.message || String(e))
|
|
2140
|
-
}
|
|
2141
|
-
return
|
|
2142
|
-
}
|
|
2143
|
-
|
|
2144
|
-
if (req.method === 'POST' && req.url === '/api/tool') {
|
|
2145
|
-
if (!isLocalOrigin(req)) {
|
|
2146
|
-
sendJson(res, { content: [{ type: 'text', text: 'forbidden: cross-origin' }], isError: true }, 403)
|
|
2147
|
-
return
|
|
2148
|
-
}
|
|
2149
|
-
// Owner-side cancel plumbing: the fork-proxy worker forwards parent
|
|
2150
|
-
// 'cancel' IPC by issuing POST /api/cancel with the same callId. Track
|
|
2151
|
-
// each in-flight /api/tool by its caller-supplied X-Mixdog-Call-Id so
|
|
2152
|
-
// the cancel endpoint can abort the AbortSignal threaded into
|
|
2153
|
-
// handleToolCall. Without this the proxy-side fetch aborts but the
|
|
2154
|
-
// owner keeps running the upstream tool to completion.
|
|
2155
|
-
const callId = String(req.headers['x-mixdog-call-id'] || '').trim() || null
|
|
2156
|
-
const ac = new AbortController()
|
|
2157
|
-
// Abort only on a genuine mid-flight client disconnect. The req 'close'
|
|
2158
|
-
// event fires on every normal request once the request body is consumed
|
|
2159
|
-
// (before handleToolCall resolves), so gating on it would mark normal
|
|
2160
|
-
// completions as aborted. Use the response side instead: when the
|
|
2161
|
-
// socket closes, res.writableFinished is true iff the response was
|
|
2162
|
-
// fully written — a real client disconnect closes the socket before
|
|
2163
|
-
// the response finishes, leaving writableFinished===false.
|
|
2164
|
-
res.on('close', () => {
|
|
2165
|
-
if (res.writableFinished) return
|
|
2166
|
-
try { ac.abort() } catch {}
|
|
2167
|
-
})
|
|
2168
|
-
if (callId) _ownerInFlightHttpCalls.set(callId, ac)
|
|
2169
|
-
try {
|
|
2170
|
-
const body = await readBody(req)
|
|
2171
|
-
const result = await handleToolCall(body.name, body.arguments ?? {}, ac.signal)
|
|
2172
|
-
sendJson(res, result)
|
|
2173
|
-
} catch (e) {
|
|
2174
|
-
sendJson(res, { content: [{ type: 'text', text: `api/tool error: ${e.message}` }], isError: true }, Number(e?.statusCode) || 500)
|
|
2175
|
-
} finally {
|
|
2176
|
-
if (callId) _ownerInFlightHttpCalls.delete(callId)
|
|
2177
|
-
}
|
|
2178
|
-
return
|
|
2179
|
-
}
|
|
2180
|
-
|
|
2181
|
-
if (req.method === 'POST' && req.url === '/api/cancel') {
|
|
2182
|
-
if (!isLocalOrigin(req)) {
|
|
2183
|
-
sendJson(res, { ok: false, error: 'forbidden: cross-origin' }, 403)
|
|
2184
|
-
return
|
|
2185
|
-
}
|
|
2186
|
-
try {
|
|
2187
|
-
const body = await readBody(req)
|
|
2188
|
-
const id = String(body.callId || '').trim()
|
|
2189
|
-
if (!id) { sendJson(res, { ok: false, error: 'callId required' }, 400); return }
|
|
2190
|
-
const ac = _ownerInFlightHttpCalls.get(id)
|
|
2191
|
-
if (ac) {
|
|
2192
|
-
try { ac.abort() } catch {}
|
|
2193
|
-
_ownerInFlightHttpCalls.delete(id)
|
|
2194
|
-
sendJson(res, { ok: true, cancelled: true })
|
|
2195
|
-
} else {
|
|
2196
|
-
sendJson(res, { ok: true, cancelled: false })
|
|
2197
|
-
}
|
|
2198
|
-
} catch (e) {
|
|
2199
|
-
sendJson(res, { ok: false, error: e.message }, Number(e?.statusCode) || 500)
|
|
2200
|
-
}
|
|
2201
|
-
return
|
|
2202
|
-
}
|
|
2203
|
-
|
|
2204
|
-
if (req.url === '/mcp') {
|
|
2205
|
-
if (!isLocalOrigin(req)) {
|
|
2206
|
-
sendJson(res, { error: 'forbidden: cross-origin' }, 403)
|
|
2207
|
-
return
|
|
2208
|
-
}
|
|
2209
|
-
try {
|
|
2210
|
-
if (req.method === 'POST') {
|
|
2211
|
-
const httpMcp = createHttpMcpServer()
|
|
2212
|
-
const httpTransport = new StreamableHTTPServerTransport({
|
|
2213
|
-
sessionIdGenerator: undefined,
|
|
2214
|
-
enableJsonResponse: true,
|
|
2215
|
-
})
|
|
2216
|
-
res.on('close', () => {
|
|
2217
|
-
httpTransport.close()
|
|
2218
|
-
void httpMcp.close()
|
|
2219
|
-
})
|
|
2220
|
-
await httpMcp.connect(httpTransport)
|
|
2221
|
-
const body = await readBody(req)
|
|
2222
|
-
await httpTransport.handleRequest(req, res, body)
|
|
2223
|
-
} else {
|
|
2224
|
-
sendJson(res, { error: 'Method not allowed' }, 405)
|
|
2225
|
-
}
|
|
2226
|
-
} catch (e) {
|
|
2227
|
-
__mixdogMemoryLog(`[memory-service] /mcp error: ${e.stack || e.message}\n`)
|
|
2228
|
-
if (!res.headersSent) sendError(res, e.message, Number(e?.statusCode) || 500)
|
|
2229
|
-
}
|
|
2230
|
-
return
|
|
2231
|
-
}
|
|
2232
|
-
|
|
2233
|
-
if (req.method !== 'POST') {
|
|
2234
|
-
sendJson(res, { error: 'Method not allowed' }, 405)
|
|
2235
|
-
return
|
|
2236
|
-
}
|
|
2237
|
-
|
|
2238
|
-
// Tail block handles /entry and /ingest-transcript — both mutate the DB,
|
|
2239
|
-
// so apply the same cross-origin guard as /admin/* routes.
|
|
2240
|
-
if (!isLocalOrigin(req)) {
|
|
2241
|
-
sendError(res, 'forbidden: cross-origin', 403)
|
|
2242
|
-
return
|
|
2243
|
-
}
|
|
2244
|
-
|
|
2245
|
-
let body
|
|
2246
|
-
try { body = await readBody(req) }
|
|
2247
|
-
catch (e) { sendError(res, e.message, Number(e?.statusCode) || 500); return }
|
|
2248
|
-
|
|
2249
|
-
try {
|
|
2250
|
-
if (req.url === '/entry') {
|
|
2251
|
-
const role = String(body.role ?? 'user')
|
|
2252
|
-
const content = String(body.content ?? '')
|
|
2253
|
-
const sourceRef = String(body.sourceRef ?? `manual:${Date.now()}-${process.pid}`)
|
|
2254
|
-
const sessionId = body.sessionId ?? null
|
|
2255
|
-
const tsMs = parseTsToMs(body.ts ?? Date.now())
|
|
2256
|
-
if (!content) { sendJson(res, { error: 'content required' }, 400); return }
|
|
2257
|
-
// Run the same scrubber used by ingestTranscriptFile so noise markers
|
|
2258
|
-
// like "[Request interrupted by user]" and whitespace-only payloads
|
|
2259
|
-
// are rejected before they reach the entries table. Match the
|
|
2260
|
-
// existing 400 / { error } convention for invalid payloads.
|
|
2261
|
-
const cleaned = cleanMemoryText(content)
|
|
2262
|
-
if (!cleaned || !cleaned.trim()) {
|
|
2263
|
-
sendJson(res, { error: 'empty after clean' }, 400)
|
|
2264
|
-
return
|
|
2265
|
-
}
|
|
2266
|
-
const entryProjectId = resolveProjectScope(typeof body.cwd === 'string' && body.cwd ? body.cwd : null)
|
|
2267
|
-
try {
|
|
2268
|
-
const result = await db.query(`
|
|
2269
|
-
INSERT INTO entries(ts, role, content, source_ref, session_id, project_id)
|
|
2270
|
-
VALUES ($1, $2, $3, $4, $5, $6)
|
|
2271
|
-
ON CONFLICT DO NOTHING
|
|
2272
|
-
RETURNING id
|
|
2273
|
-
`, [tsMs, role, cleaned, sourceRef, sessionId, entryProjectId])
|
|
2274
|
-
const insertedId = result.rows[0]?.id ?? null
|
|
2275
|
-
sendJson(res, { ok: true, id: insertedId !== null ? Number(insertedId) : null, changes: Number(result.rowCount ?? result.affectedRows ?? 0) })
|
|
2276
|
-
} catch (e) {
|
|
2277
|
-
sendJson(res, { error: e.message }, 500)
|
|
2278
|
-
}
|
|
2279
|
-
return
|
|
2280
|
-
}
|
|
2281
|
-
|
|
2282
|
-
if (req.url === '/ingest-transcript') {
|
|
2283
|
-
const filePath = body.filePath
|
|
2284
|
-
if (!filePath) { sendJson(res, { error: 'filePath required' }, 400); return }
|
|
2285
|
-
try {
|
|
2286
|
-
const n = await ingestTranscriptFile(filePath, { cwd: body.cwd })
|
|
2287
|
-
sendJson(res, { ok: true, ingested: n })
|
|
2288
|
-
} catch (e) {
|
|
2289
|
-
sendJson(res, { error: e.message }, 500)
|
|
2290
|
-
}
|
|
2291
|
-
return
|
|
2292
|
-
}
|
|
2293
|
-
|
|
2294
|
-
if (req.url === '/transcript/ingest-sync') {
|
|
2295
|
-
const filePath = body.path
|
|
2296
|
-
if (!filePath || typeof filePath !== 'string') {
|
|
2297
|
-
sendJson(res, { error: 'path required' }, 400)
|
|
2298
|
-
return
|
|
2299
|
-
}
|
|
2300
|
-
try {
|
|
2301
|
-
let stat
|
|
2302
|
-
try { stat = await fs.promises.stat(filePath) } catch {
|
|
2303
|
-
sendJson(res, { ok: true, complete: true, fileSize: 0, offsetBytes: 0 })
|
|
2304
|
-
return
|
|
2305
|
-
}
|
|
2306
|
-
const fileSize = stat.size
|
|
2307
|
-
await ingestTranscriptFile(filePath, { cwd: body.cwd })
|
|
2308
|
-
const off = _transcriptIngest.getOffset(filePath)
|
|
2309
|
-
const offsetBytes = off && Number.isFinite(off.bytes) ? off.bytes : 0
|
|
2310
|
-
const complete = offsetBytes >= fileSize
|
|
2311
|
-
sendJson(res, { ok: true, offsetBytes, fileSize, complete })
|
|
2312
|
-
} catch (e) {
|
|
2313
|
-
sendJson(res, { error: e.message }, 500)
|
|
2314
|
-
}
|
|
2315
|
-
return
|
|
2316
|
-
}
|
|
2317
|
-
|
|
2318
|
-
sendJson(res, { error: 'Not found' }, 404)
|
|
2319
|
-
} catch (e) {
|
|
2320
|
-
__mixdogMemoryLog(`[memory-service] ${req.url} error: ${e.stack || e.message}\n`)
|
|
2321
|
-
sendError(res, e.message)
|
|
2322
|
-
}
|
|
727
|
+
// ── HTTP request router (extracted to lib/http-router.mjs). The facade owns
|
|
728
|
+
// the http.Server + listen/stop lifecycle; the router builds the request
|
|
729
|
+
// handler and buildSessionCoreMemoryPayload from injected live state.
|
|
730
|
+
const _httpRouter = createHttpRouter({
|
|
731
|
+
getDb: () => db,
|
|
732
|
+
dataDir: DATA_DIR,
|
|
733
|
+
log: __mixdogMemoryLog,
|
|
734
|
+
pluginVersion: PLUGIN_VERSION,
|
|
735
|
+
bootPromotionCodeFingerprint: BOOT_PROMOTION_CODE_FINGERPRINT,
|
|
736
|
+
touchDaemonIdleTimer,
|
|
737
|
+
entryStats,
|
|
738
|
+
cycleScheduler: _cycleScheduler,
|
|
739
|
+
getInitialized: () => _initialized,
|
|
740
|
+
getInitPromise: () => _initPromise,
|
|
741
|
+
setBootTimestamp: (v) => { _bootTimestamp = v },
|
|
742
|
+
handleMemoryAction,
|
|
743
|
+
handleToolCall,
|
|
744
|
+
stop,
|
|
745
|
+
registerClient,
|
|
746
|
+
deregisterClient,
|
|
747
|
+
getDraining: () => _stopPromise != null,
|
|
748
|
+
getCycle1CallLlm,
|
|
749
|
+
getTraceDb: () => _traceDb,
|
|
750
|
+
setTraceDb: (v) => { _traceDb = v },
|
|
751
|
+
ingestTranscriptFile,
|
|
752
|
+
getTranscriptOffset: (fp) => _transcriptIngest.getOffset(fp),
|
|
753
|
+
parseTsToMs,
|
|
2323
754
|
})
|
|
755
|
+
const buildSessionCoreMemoryPayload = _httpRouter.buildSessionCoreMemoryPayload
|
|
756
|
+
const httpServer = http.createServer(_httpRouter.requestHandler)
|
|
2324
757
|
|
|
2325
758
|
export { TOOL_DEFS, handleToolCall, buildSessionCoreMemoryPayload }
|
|
2326
759
|
export { MEMORY_INSTRUCTIONS_TEXT as instructions }
|
|
@@ -2372,6 +805,11 @@ export async function stop() {
|
|
|
2372
805
|
try { clearTimeout(_idleShutdownTimer) } catch {}
|
|
2373
806
|
_idleShutdownTimer = null
|
|
2374
807
|
}
|
|
808
|
+
cancelClientGrace()
|
|
809
|
+
if (_clientSweepTimer) {
|
|
810
|
+
try { clearInterval(_clientSweepTimer) } catch {}
|
|
811
|
+
_clientSweepTimer = null
|
|
812
|
+
}
|
|
2375
813
|
await stopLlmWorker()
|
|
2376
814
|
resetHttpListenErrorHandler()
|
|
2377
815
|
if (_httpBoundPort != null || _httpReadyPromise) {
|