mixdog 0.9.19 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +100 -34
- package/package.json +3 -2
- package/scripts/build-tui.mjs +6 -0
- package/scripts/hook-bus-test.mjs +8 -0
- package/scripts/log-writer-guard-smoke.mjs +131 -0
- package/scripts/reactive-compact-persist-smoke.mjs +22 -6
- package/scripts/recall-bench-cases.json +0 -1
- package/scripts/recall-quality-cases.json +1 -2
- package/scripts/session-ingest-compaction-smoke.mjs +241 -0
- package/scripts/tool-smoke.mjs +150 -45
- package/src/defaults/skills/setup/SKILL.md +327 -0
- package/src/help.mjs +2 -5
- package/src/lib/mixdog-debug.cjs +13 -0
- package/src/mixdog-session-runtime.mjs +7 -3328
- package/src/runtime/agent/orchestrator/context/collect.mjs +33 -9
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +34 -2
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +67 -10
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +3 -0
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +40 -3
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +73 -3
- package/src/runtime/agent/orchestrator/session/agent-loop.mjs +663 -0
- package/src/runtime/agent/orchestrator/session/compact/engine.mjs +6 -0
- package/src/runtime/agent/orchestrator/session/eager-dispatch.mjs +153 -0
- package/src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs +49 -0
- package/src/runtime/agent/orchestrator/session/loop/stored-tool-args.mjs +9 -1
- package/src/runtime/agent/orchestrator/session/loop.mjs +8 -1860
- package/src/runtime/agent/orchestrator/session/manager/agent-runtime-singleton.mjs +29 -0
- package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +686 -0
- package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +60 -12
- package/src/runtime/agent/orchestrator/session/manager/env-utils.mjs +8 -0
- package/src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs +159 -0
- package/src/runtime/agent/orchestrator/session/manager/message-sanitize.mjs +143 -0
- package/src/runtime/agent/orchestrator/session/manager/prefetch-bridge.mjs +268 -0
- package/src/runtime/agent/orchestrator/session/manager/provider-cache-key.mjs +22 -0
- package/src/runtime/agent/orchestrator/session/manager/runtime-loaders.mjs +26 -0
- package/src/runtime/agent/orchestrator/session/manager/session-close.mjs +124 -0
- package/src/runtime/agent/orchestrator/session/manager/session-crud.mjs +258 -0
- package/src/runtime/agent/orchestrator/session/manager/session-errors.mjs +20 -0
- package/src/runtime/agent/orchestrator/session/manager/session-id.mjs +9 -0
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +475 -0
- package/src/runtime/agent/orchestrator/session/manager/session-lock.mjs +23 -0
- package/src/runtime/agent/orchestrator/session/manager.mjs +88 -2285
- package/src/runtime/agent/orchestrator/session/pre-send-compact.mjs +440 -0
- package/src/runtime/agent/orchestrator/session/send-with-recovery.mjs +153 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +4 -1
- package/src/runtime/agent/orchestrator/session/tool-batch.mjs +642 -0
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +23 -3
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-paths.mjs +108 -2
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +15 -3
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +7 -0
- package/src/runtime/agent/orchestrator/tools/builtin/shell-runtime.mjs +24 -2
- package/src/runtime/agent/orchestrator/tools/patch/orchestrator.mjs +2 -2
- package/src/runtime/agent/orchestrator/tools/patch/parsing.mjs +72 -8
- package/src/runtime/agent/orchestrator/tools/shell-policy-danger-target.mjs +5 -1
- package/src/runtime/channels/index.mjs +6 -2183
- package/src/runtime/channels/lib/inbound-handler.mjs +328 -0
- package/src/runtime/channels/lib/interaction-handlers.mjs +260 -0
- package/src/runtime/channels/lib/network-retry.mjs +23 -0
- package/src/runtime/channels/lib/owned-runtime.mjs +627 -0
- package/src/runtime/channels/lib/runtime-paths.mjs +20 -9
- package/src/runtime/channels/lib/transcript-binding.mjs +336 -0
- package/src/runtime/channels/lib/voice-runtime-fetcher.mjs +39 -2
- package/src/runtime/channels/lib/worker-bootstrap.mjs +97 -0
- package/src/runtime/channels/lib/worker-ipc.mjs +201 -0
- package/src/runtime/channels/lib/worker-main.mjs +777 -0
- package/src/runtime/memory/index.mjs +163 -1725
- package/src/runtime/memory/lib/embedding-provider.mjs +4 -2
- package/src/runtime/memory/lib/embedding-worker.mjs +47 -6
- package/src/runtime/memory/lib/http-router.mjs +811 -0
- package/src/runtime/memory/lib/memory-action-handlers.mjs +901 -0
- package/src/runtime/memory/lib/memory-cycle2-gate.mjs +6 -0
- package/src/runtime/memory/lib/memory-cycle2-mutations.mjs +46 -9
- package/src/runtime/memory/lib/memory-cycle2.mjs +87 -11
- package/src/runtime/memory/lib/memory-embed.mjs +28 -7
- package/src/runtime/memory/lib/memory-recall-store.mjs +4 -4
- package/src/runtime/memory/lib/memory.mjs +39 -0
- package/src/runtime/memory/lib/query-handlers.mjs +2 -2
- package/src/runtime/memory/lib/session-ingest-runtime.mjs +401 -0
- package/src/runtime/shared/atomic-file.mjs +138 -80
- package/src/runtime/shared/child-guardian.mjs +61 -3
- package/src/session-runtime/boot-profile.mjs +36 -0
- package/src/session-runtime/channel-config-api.mjs +70 -0
- package/src/session-runtime/context-status.mjs +181 -0
- package/src/session-runtime/env.mjs +17 -0
- package/src/session-runtime/lifecycle-api.mjs +242 -0
- package/src/session-runtime/model-route-api.mjs +198 -0
- package/src/session-runtime/provider-auth-api.mjs +135 -0
- package/src/session-runtime/resource-api.mjs +282 -0
- package/src/session-runtime/runtime-core.mjs +2104 -0
- package/src/session-runtime/session-turn-api.mjs +274 -0
- package/src/session-runtime/tool-catalog.mjs +18 -264
- package/src/session-runtime/tool-defs.mjs +2 -2
- package/src/session-runtime/workflow-agents-api.mjs +238 -0
- package/src/standalone/agent-tool.mjs +2 -2
- package/src/standalone/channel-worker.mjs +67 -7
- package/src/standalone/folder-dialog.mjs +4 -1
- package/src/standalone/memory-runtime-proxy.mjs +154 -17
- package/src/standalone/seeds.mjs +28 -1
- package/src/tui/App.jsx +40 -28
- package/src/tui/app/core-memory-picker.mjs +1 -1
- package/src/tui/app/doctor.mjs +175 -0
- package/src/tui/app/slash-commands.mjs +1 -0
- package/src/tui/app/slash-dispatch.mjs +9 -0
- package/src/tui/app/use-mouse-input.mjs +6 -0
- package/src/tui/app/use-transcript-scroll.mjs +77 -3
- package/src/tui/dist/index.mjs +2851 -2162
- package/src/tui/engine/context-state.mjs +145 -0
- package/src/tui/engine/prompt-history.mjs +27 -0
- package/src/tui/engine/session-api-ext.mjs +478 -0
- package/src/tui/engine/session-api.mjs +564 -0
- package/src/tui/engine/session-flow.mjs +485 -0
- package/src/tui/engine/turn.mjs +1078 -0
- package/src/tui/engine.mjs +68 -2620
- package/src/tui/index.jsx +7 -0
- package/vendor/ink/build/ink.js +16 -1
- package/vendor/ink/build/output.js +30 -4
- package/vendor/ink/build/render.js +5 -0
- package/src/workflows/sequential/WORKFLOW.md +0 -51
|
@@ -11,6 +11,12 @@ import { __mixdogMemoryLog, throwIfAborted, resourceDir } from './memory-cycle2-
|
|
|
11
11
|
|
|
12
12
|
export const CYCLE2_ACTIVE_TARGET_CAP = 100
|
|
13
13
|
|
|
14
|
+
// Minimum number of active (promoted) rows cycle2 must preserve. When the
|
|
15
|
+
// active pool is at or below this floor, active recheck/demotion is skipped
|
|
16
|
+
// and no archiving path may push active below it — prevents draining the
|
|
17
|
+
// pool to zero. Configurable via config.active_floor.
|
|
18
|
+
export const CYCLE2_ACTIVE_MIN_FLOOR = 20
|
|
19
|
+
|
|
14
20
|
// Status-based verb whitelist. 3-tier policy: pending → active/archived,
|
|
15
21
|
// active → active/archived/update/merge.
|
|
16
22
|
const STATUS_ALLOWED_VERBS = {
|
|
@@ -86,11 +86,15 @@ export async function applyBatchStatusVerdicts(db, batch, nowMs) {
|
|
|
86
86
|
)
|
|
87
87
|
let promoted = 0
|
|
88
88
|
let archived = 0
|
|
89
|
+
let archived_active = 0
|
|
89
90
|
for (const r of (res.rows ?? [])) {
|
|
90
91
|
if (r.was_pending && r.new_status === 'active') promoted += 1
|
|
91
|
-
else if (r.new_status === 'archived')
|
|
92
|
+
else if (r.new_status === 'archived') {
|
|
93
|
+
archived += 1
|
|
94
|
+
if (!r.was_pending) archived_active += 1
|
|
95
|
+
}
|
|
92
96
|
}
|
|
93
|
-
return { promoted, archived }
|
|
97
|
+
return { promoted, archived, archived_active }
|
|
94
98
|
}
|
|
95
99
|
|
|
96
100
|
// Generic status update for archived/active terminal transitions.
|
|
@@ -186,24 +190,49 @@ export async function applyMerge(db, targetId, sourceIds, options = {}) {
|
|
|
186
190
|
)
|
|
187
191
|
continue
|
|
188
192
|
}
|
|
193
|
+
// Floor guard: archiving an active source row shrinks the active pool, so
|
|
194
|
+
// it must reserve from the shared floor budget. Over-budget merges are
|
|
195
|
+
// skipped (source stays active); pending sources never reserve.
|
|
196
|
+
const srcActive = srcRow.status === 'active'
|
|
197
|
+
if (srcActive && options.floorGuard && !options.floorGuard.reserve()) {
|
|
198
|
+
__mixdogMemoryLog(`[cycle2] merge source archive skipped (floor guard): target=${targetId} src=${sid}\n`)
|
|
199
|
+
continue
|
|
200
|
+
}
|
|
201
|
+
let archivedPrevStatus = null
|
|
189
202
|
try {
|
|
190
|
-
//
|
|
191
|
-
//
|
|
203
|
+
// Archive is the guarded mutation unit: the reservation is only kept if
|
|
204
|
+
// this transaction commits. Embedding cleanup is deliberately OUTSIDE
|
|
205
|
+
// this try — a post-commit cleanup failure must NOT refund the budget,
|
|
206
|
+
// since the active row is already archived.
|
|
192
207
|
await db.transaction(async (tx) => {
|
|
193
208
|
await tx.query(
|
|
194
209
|
`UPDATE entries SET chunk_root = $1, project_id = $2 WHERE chunk_root = $3 AND id != $4 AND is_root = 0`,
|
|
195
210
|
[targetId, target.project_id, sid, sid],
|
|
196
211
|
)
|
|
197
|
-
|
|
198
|
-
|
|
212
|
+
// Capture prev status in the same statement so a stale snapshot (the
|
|
213
|
+
// row was already archived by a concurrent write) can refund below.
|
|
214
|
+
const ar = await tx.query(
|
|
215
|
+
`WITH pre AS (SELECT id, status AS prev FROM entries WHERE id = $1 AND is_root = 1)
|
|
216
|
+
UPDATE entries SET status = 'archived'
|
|
217
|
+
FROM pre WHERE entries.id = pre.id AND entries.is_root = 1
|
|
218
|
+
RETURNING pre.prev AS prev_status`,
|
|
199
219
|
[sid],
|
|
200
220
|
)
|
|
221
|
+
archivedPrevStatus = ar.rows?.[0]?.prev_status ?? null
|
|
201
222
|
})
|
|
202
|
-
await deleteRootEmbedding(db, sid)
|
|
203
|
-
moved += 1
|
|
204
223
|
} catch (err) {
|
|
224
|
+
if (srcActive && options.floorGuard) options.floorGuard.refund()
|
|
205
225
|
__mixdogMemoryLog(`[cycle2] merge failed (target=${targetId} src=${sid}): ${err.message}\n`)
|
|
226
|
+
continue
|
|
206
227
|
}
|
|
228
|
+
// Archive committed. If the source was no longer active at UPDATE time,
|
|
229
|
+
// the reservation did not cover a real active→archived transition — refund.
|
|
230
|
+
if (srcActive && options.floorGuard && archivedPrevStatus !== 'active') options.floorGuard.refund()
|
|
231
|
+
// Archive committed — best-effort embedding cleanup only. The next abort
|
|
232
|
+
// checkpoint is before the next source.
|
|
233
|
+
moved += 1
|
|
234
|
+
try { await deleteRootEmbedding(db, sid) }
|
|
235
|
+
catch (err) { __mixdogMemoryLog(`[cycle2] merge embedding cleanup failed (target=${targetId} src=${sid}): ${err.message}\n`) }
|
|
207
236
|
}
|
|
208
237
|
return moved
|
|
209
238
|
}
|
|
@@ -245,6 +274,7 @@ async function _llmJudgePair(summaryA, summaryB, siblingContext = [], options =
|
|
|
245
274
|
|
|
246
275
|
export async function runPhaseMerge(db, options = {}) {
|
|
247
276
|
const signal = options?.signal
|
|
277
|
+
const floorGuard = options?.floorGuard
|
|
248
278
|
throwIfAborted(signal)
|
|
249
279
|
// PG-side lateral nearest-neighbor via HNSW index — replaces JS O(n²) double loop.
|
|
250
280
|
const pairRes = await db.query(
|
|
@@ -296,7 +326,9 @@ export async function runPhaseMerge(db, options = {}) {
|
|
|
296
326
|
if (mergedIds.has(a.id) || mergedIds.has(b.id)) return
|
|
297
327
|
const keeper = _pickKeeper(a, b)
|
|
298
328
|
const loser = keeper.id === a.id ? b : a
|
|
299
|
-
|
|
329
|
+
// phase_merge only pairs active rows, so the loser archive is an active
|
|
330
|
+
// demotion — thread the floor budget through applyMerge.
|
|
331
|
+
const moved = await applyMerge(db, keeper.id, [loser.id], { signal, floorGuard })
|
|
300
332
|
if (moved > 0) {
|
|
301
333
|
merged += moved
|
|
302
334
|
mergedIds.add(loser.id)
|
|
@@ -372,6 +404,9 @@ export async function runPhaseMerge(db, options = {}) {
|
|
|
372
404
|
)
|
|
373
405
|
throwIfAborted(signal)
|
|
374
406
|
if (!verdictMerge) continue
|
|
407
|
+
// Core-overlap archive is an active demotion — reserve floor budget and
|
|
408
|
+
// refund if the guarded UPDATE turns out not to fire.
|
|
409
|
+
if (floorGuard && !floorGuard.reserve()) continue
|
|
375
410
|
// Archiving one overlap and deleting its embedding is one mutation unit;
|
|
376
411
|
// cancellation resumes at the next row boundary.
|
|
377
412
|
const r = await db.query(
|
|
@@ -387,6 +422,8 @@ export async function runPhaseMerge(db, options = {}) {
|
|
|
387
422
|
if (Number(r.rowCount ?? r.affectedRows ?? 0) > 0) {
|
|
388
423
|
coreOverlap++
|
|
389
424
|
await deleteRootEmbedding(db, Number(row.entry_id))
|
|
425
|
+
} else if (floorGuard) {
|
|
426
|
+
floorGuard.refund()
|
|
390
427
|
}
|
|
391
428
|
}
|
|
392
429
|
throwIfAborted(signal)
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// memory-cycle2-mutations.mjs — status/update/merge writers + runPhaseMerge
|
|
7
7
|
// memory-cycle2-gate.mjs — prompt/parse/validate + runUnifiedGate + cascade
|
|
8
8
|
import { flushEmbeddingDirty } from './memory-embed.mjs'
|
|
9
|
+
import { refreshHotActive } from './memory.mjs'
|
|
9
10
|
import { backfillCoreEmbeddings, nominateCoreCandidates, CORE_SUMMARY_MAX } from './core-memory-store.mjs'
|
|
10
11
|
import { markCycleRequest, consumeCycleRequests, resolveCoalesceMaxDrains, scheduleCoalescedCycleRetry, makeCycleRequestSignature, resolveCoalesceMaxRetries } from './memory-cycle-requests.mjs'
|
|
11
12
|
import { __mixdogMemoryLog, throwIfAborted } from './memory-cycle2-shared.mjs'
|
|
@@ -13,7 +14,7 @@ import {
|
|
|
13
14
|
applyBatchStatusVerdicts, clampPendingPromotions, applySimpleStatus, applyUpdate, applyMerge, runPhaseMerge,
|
|
14
15
|
} from './memory-cycle2-mutations.mjs'
|
|
15
16
|
import {
|
|
16
|
-
CYCLE2_ACTIVE_TARGET_CAP, loadCurrentRulesDigest, runUnifiedGate, sonnetCascade,
|
|
17
|
+
CYCLE2_ACTIVE_TARGET_CAP, CYCLE2_ACTIVE_MIN_FLOOR, loadCurrentRulesDigest, runUnifiedGate, sonnetCascade,
|
|
17
18
|
NON_ARCHIVE_VERBS, requiredCoreIdForAction,
|
|
18
19
|
} from './memory-cycle2-gate.mjs'
|
|
19
20
|
|
|
@@ -48,6 +49,7 @@ function mergeCycle2Results(a, b) {
|
|
|
48
49
|
archived: Number(a.archived || 0) + Number(b.archived || 0),
|
|
49
50
|
merged: Number(a.merged || 0) + Number(b.merged || 0),
|
|
50
51
|
updated: Number(a.updated || 0) + Number(b.updated || 0),
|
|
52
|
+
floor_protected: Number(a.floor_protected || 0) + Number(b.floor_protected || 0),
|
|
51
53
|
kept: Number(a.kept || 0) + Number(b.kept || 0),
|
|
52
54
|
rejected_verb: Number(a.rejected_verb || 0) + Number(b.rejected_verb || 0),
|
|
53
55
|
merge_rejected: Number(a.merge_rejected || 0) + Number(b.merge_rejected || 0),
|
|
@@ -199,6 +201,9 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
199
201
|
const activeTargetCap = Number.isFinite(Number(config.active_target_cap))
|
|
200
202
|
? Math.max(1, Number(config.active_target_cap))
|
|
201
203
|
: CYCLE2_ACTIVE_TARGET_CAP
|
|
204
|
+
const activeFloor = Number.isFinite(Number(config.active_floor))
|
|
205
|
+
? Math.max(0, Number(config.active_floor))
|
|
206
|
+
: CYCLE2_ACTIVE_MIN_FLOOR
|
|
202
207
|
const nowMs = Date.now()
|
|
203
208
|
|
|
204
209
|
const stats = {
|
|
@@ -211,6 +216,7 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
211
216
|
rescore: { updated: 0 },
|
|
212
217
|
phase_merge: { merged: 0, llm_calls: 0, tier1_pairs: 0, tier2_pairs: 0, core_overlap: 0 },
|
|
213
218
|
cascade: { evaluated: 0, dropped: 0 },
|
|
219
|
+
floor_protected: 0,
|
|
214
220
|
}
|
|
215
221
|
|
|
216
222
|
if (dataDir) {
|
|
@@ -231,6 +237,20 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
231
237
|
const activeCount = Number(activeCountRes.rows[0]?.c ?? 0)
|
|
232
238
|
const reviewActiveRows = activeCount > activeTargetCap
|
|
233
239
|
|
|
240
|
+
// Shared floor-demotion budget: the number of active rows that may be
|
|
241
|
+
// archived this cycle before the active pool would breach the minimum
|
|
242
|
+
// floor. EVERY archiving path that removes an active row must reserve()
|
|
243
|
+
// from this single budget (status demotions, cascade drops, applyMerge
|
|
244
|
+
// sources, and phase_merge) so no path can drain active below the floor.
|
|
245
|
+
// Reservations are refunded when the underlying write turns out not to
|
|
246
|
+
// demote (optimistic-guard miss / merge failure).
|
|
247
|
+
const floorGuard = {
|
|
248
|
+
remaining: Math.max(0, activeCount - activeFloor),
|
|
249
|
+
protected: 0,
|
|
250
|
+
reserve() { if (this.remaining <= 0) { this.protected += 1; return false } this.remaining -= 1; return true },
|
|
251
|
+
refund(n = 1) { this.remaining += Math.max(0, n) },
|
|
252
|
+
}
|
|
253
|
+
|
|
234
254
|
// Rolling active re-review quota. Under cap, the unified selection below
|
|
235
255
|
// pulls only pending rows, so an already-promoted entry that later drifts
|
|
236
256
|
// stale or turns out to restate a rule file never gets re-judged — the
|
|
@@ -244,7 +264,9 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
244
264
|
// restatements. Embedding dedup is skipped on purpose: rule restatements
|
|
245
265
|
// are often cross-language paraphrases whose cosine never clears the merge
|
|
246
266
|
// threshold, but the LLM gate catches the semantic overlap.
|
|
247
|
-
|
|
267
|
+
// Floor guard: when the active pool is at/below the minimum floor, skip the
|
|
268
|
+
// rolling active recheck entirely so demotions cannot drain it further.
|
|
269
|
+
const activeRecheckQuota = (reviewActiveRows || activeCount <= activeFloor)
|
|
248
270
|
? 0
|
|
249
271
|
: Math.max(0, Math.min(Number(config.active_recheck_quota ?? 8), batchSize - 1))
|
|
250
272
|
const pendingLimit = batchSize - activeRecheckQuota
|
|
@@ -460,7 +482,7 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
460
482
|
)
|
|
461
483
|
continue
|
|
462
484
|
}
|
|
463
|
-
const moved = await applyMerge(db, targetId, sourceIds, { signal })
|
|
485
|
+
const moved = await applyMerge(db, targetId, sourceIds, { signal, floorGuard })
|
|
464
486
|
throwIfAborted(signal)
|
|
465
487
|
if (moved > 0) {
|
|
466
488
|
stats.merged += moved
|
|
@@ -489,23 +511,62 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
489
511
|
// Status verdicts are applied as one SQL batch; checkpoint before the
|
|
490
512
|
// batch and then again at the next cycle2 unit boundary.
|
|
491
513
|
throwIfAborted(signal)
|
|
492
|
-
|
|
514
|
+
// Reserve floor budget for each active→archived demotion; over-budget
|
|
515
|
+
// demotions are dropped (the row stays active, still reviewed_at-bumped
|
|
516
|
+
// via reviewedIds). Pending→archived never reserves (was never active).
|
|
517
|
+
const guardedBatch = []
|
|
518
|
+
let reservedDemotions = 0
|
|
493
519
|
for (const item of statusBatch) {
|
|
494
|
-
if (item.new_status === 'archived' && !item.was_pending)
|
|
520
|
+
if (item.new_status === 'archived' && !item.was_pending) {
|
|
521
|
+
if (!floorGuard.reserve()) continue
|
|
522
|
+
reservedDemotions += 1
|
|
523
|
+
}
|
|
524
|
+
guardedBatch.push(item)
|
|
495
525
|
}
|
|
496
|
-
const activeCountForClamp = Math.max(0, activeCount -
|
|
497
|
-
const clampRes = clampPendingPromotions(
|
|
526
|
+
const activeCountForClamp = Math.max(0, activeCount - reservedDemotions)
|
|
527
|
+
const clampRes = clampPendingPromotions(guardedBatch, rowsById, activeCountForClamp, activeTargetCap)
|
|
498
528
|
stats.promotion_clamped = clampRes.clamped
|
|
499
529
|
const batchRes = await applyBatchStatusVerdicts(db, clampRes.batch, nowMs)
|
|
500
530
|
stats.promoted += batchRes.promoted
|
|
501
531
|
stats.archived += batchRes.archived
|
|
532
|
+
// Spend on CONFIRMED demotions only: refund reservations the optimistic
|
|
533
|
+
// guard skipped (concurrent write moved the row).
|
|
534
|
+
const confirmedActive = Number(batchRes.archived_active || 0)
|
|
535
|
+
if (reservedDemotions > confirmedActive) floorGuard.refund(reservedDemotions - confirmedActive)
|
|
502
536
|
}
|
|
503
537
|
|
|
504
538
|
if (cascadeDropArchiveIds.length > 0) {
|
|
505
539
|
throwIfAborted(signal)
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
540
|
+
// Floor guard: a cascade drop of an active row reduces the pool, so
|
|
541
|
+
// reserve budget; pending rows dropped here never were active. The
|
|
542
|
+
// snapshot status can be stale (statusBatch above / concurrent writes),
|
|
543
|
+
// so reserve optimistically then refund against the ACTUAL number of
|
|
544
|
+
// active→archived transitions the UPDATE performed (prev_status read in
|
|
545
|
+
// the same statement).
|
|
546
|
+
let reservedActive = 0
|
|
547
|
+
const toDrop = []
|
|
548
|
+
for (const id of cascadeDropArchiveIds) {
|
|
549
|
+
const row = rowsById.get(Number(id))
|
|
550
|
+
if (row && row.status === 'active') {
|
|
551
|
+
if (!floorGuard.reserve()) continue
|
|
552
|
+
reservedActive += 1
|
|
553
|
+
}
|
|
554
|
+
toDrop.push(id)
|
|
555
|
+
}
|
|
556
|
+
if (toDrop.length > 0) {
|
|
557
|
+
const r = await db.query(`
|
|
558
|
+
WITH pre AS (SELECT id, status AS prev FROM entries WHERE id = ANY($1::bigint[]) AND is_root = 1)
|
|
559
|
+
UPDATE entries SET status = 'archived'
|
|
560
|
+
FROM pre
|
|
561
|
+
WHERE entries.id = pre.id AND entries.is_root = 1
|
|
562
|
+
RETURNING entries.id, pre.prev AS prev_status
|
|
563
|
+
`, [toDrop])
|
|
564
|
+
const dropped = r.rows ?? []
|
|
565
|
+
stats.cascade.dropped += dropped.length
|
|
566
|
+
stats.archived += dropped.length
|
|
567
|
+
const actualArchivedActive = dropped.filter(x => x.prev_status === 'active').length
|
|
568
|
+
if (reservedActive > actualArchivedActive) floorGuard.refund(reservedActive - actualArchivedActive)
|
|
569
|
+
}
|
|
509
570
|
}
|
|
510
571
|
if (reviewedIds.length > 0) {
|
|
511
572
|
throwIfAborted(signal)
|
|
@@ -570,9 +631,12 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
570
631
|
}
|
|
571
632
|
|
|
572
633
|
// phase_merge: cosine dedup over active entries.
|
|
573
|
-
const phaseMergeStats = await runPhaseMerge(db, { ...options, signal })
|
|
634
|
+
const phaseMergeStats = await runPhaseMerge(db, { ...options, signal, floorGuard })
|
|
574
635
|
throwIfAborted(signal)
|
|
575
636
|
stats.phase_merge = phaseMergeStats
|
|
637
|
+
// Surface how many active-row demotions were withheld to hold the floor
|
|
638
|
+
// across every archiving path this cycle (status/cascade/merge/phase_merge).
|
|
639
|
+
stats.floor_protected = floorGuard.protected
|
|
576
640
|
|
|
577
641
|
// Core-candidate nomination (proposal mode): flag strong durable active
|
|
578
642
|
// roots as core-memory candidates for user approval. Runs AFTER phase_merge
|
|
@@ -621,6 +685,18 @@ async function _runCycle2Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
621
685
|
__mixdogMemoryLog(`[cycle2] chronic sweep failed: ${err.message}\n`)
|
|
622
686
|
}
|
|
623
687
|
|
|
688
|
+
// Refresh the hot-active materialized view now that every promotion/archival/
|
|
689
|
+
// dedup mutation for this cycle has landed, so the recall hot path reads a
|
|
690
|
+
// current active set. refreshHotActive is itself best-effort (logs + swallows);
|
|
691
|
+
// the guard here also keeps a refresh failure from failing the cycle.
|
|
692
|
+
try {
|
|
693
|
+
throwIfAborted(signal)
|
|
694
|
+
await refreshHotActive(db)
|
|
695
|
+
} catch (err) {
|
|
696
|
+
if (signal?.aborted) throw signal.reason ?? err
|
|
697
|
+
__mixdogMemoryLog(`[cycle2] mv_hot_active refresh failed: ${err.message}\n`)
|
|
698
|
+
}
|
|
699
|
+
|
|
624
700
|
__mixdogMemoryLog(
|
|
625
701
|
`[cycle2] rescore=${stats.rescore.updated}` +
|
|
626
702
|
` core_backfill=${stats.core_embedding_backfill}` +
|
|
@@ -4,7 +4,7 @@ function __mixdogMemoryLog(...args) {
|
|
|
4
4
|
return __mixdogMemoryStderrWrite(...args);
|
|
5
5
|
}
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { embedTexts, getEmbeddingModelId } from './embedding-provider.mjs'
|
|
8
8
|
import { embeddingToSql } from './memory.mjs'
|
|
9
9
|
import { createHash } from 'crypto'
|
|
10
10
|
|
|
@@ -127,13 +127,19 @@ export async function cachedEmbedTextBatch(db, texts, options = {}) {
|
|
|
127
127
|
: Array.from(r.vector),
|
|
128
128
|
]))
|
|
129
129
|
|
|
130
|
-
// Embed misses
|
|
130
|
+
// Embed misses through the real batch path (provider embedTexts → worker
|
|
131
|
+
// embed-batch): one batched ONNX run per BATCH_SIZE chunk instead of N
|
|
132
|
+
// single-embed calls fanned out via Promise.all that the worker then
|
|
133
|
+
// re-serializes one at a time. Vector format/cache-key semantics unchanged.
|
|
131
134
|
const misses = entries.filter(e => !hitMap.has(e.hash.toString('hex')))
|
|
132
135
|
if (misses.length > 0) {
|
|
133
136
|
throwIfAborted(signal)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
+
for (let i = 0; i < misses.length; i += BATCH_SIZE) {
|
|
138
|
+
throwIfAborted(signal)
|
|
139
|
+
const chunk = misses.slice(i, i + BATCH_SIZE)
|
|
140
|
+
const vectors = await embedTexts(chunk.map(e => e.text))
|
|
141
|
+
for (let j = 0; j < chunk.length; j++) chunk[j].vector = vectors[j]
|
|
142
|
+
}
|
|
137
143
|
throwIfAborted(signal)
|
|
138
144
|
// Bulk INSERT misses
|
|
139
145
|
await db.query(
|
|
@@ -279,6 +285,15 @@ export async function flushEmbeddingDirty(db, options = {}) {
|
|
|
279
285
|
// dense matching, not staleness.
|
|
280
286
|
export async function flushRawEmbeddings(db, options = {}) {
|
|
281
287
|
const { limit = 200, signal } = options ?? {}
|
|
288
|
+
// Optional id allow-list: restrict the SKIP LOCKED claim to a specific set of
|
|
289
|
+
// rows (e.g. exactly the rows a single ingest_session call inserted) so a
|
|
290
|
+
// caller can flush ONLY its own rows instead of inheriting the whole raw
|
|
291
|
+
// backlog. An explicit but empty list means "nothing to do for this call"
|
|
292
|
+
// and must NOT fall through to an unscoped backlog sweep.
|
|
293
|
+
const idFilter = Array.isArray(options?.ids)
|
|
294
|
+
? options.ids.map(Number).filter(Number.isFinite)
|
|
295
|
+
: null
|
|
296
|
+
if (idFilter && idFilter.length === 0) return { attempted: 0, embedded: 0 }
|
|
282
297
|
throwIfAborted(signal)
|
|
283
298
|
await ensureEmbCacheTable(db)
|
|
284
299
|
throwIfAborted(signal)
|
|
@@ -295,16 +310,22 @@ export async function flushRawEmbeddings(db, options = {}) {
|
|
|
295
310
|
try {
|
|
296
311
|
throwIfAborted(signal)
|
|
297
312
|
await client.query('BEGIN')
|
|
313
|
+
const params = [batchCap, cursor]
|
|
314
|
+
let idClause = ''
|
|
315
|
+
if (idFilter) {
|
|
316
|
+
params.push(idFilter)
|
|
317
|
+
idClause = ` AND id = ANY($${params.length}::bigint[])`
|
|
318
|
+
}
|
|
298
319
|
const res = await client.query(
|
|
299
320
|
`SELECT id FROM memory.entries
|
|
300
321
|
WHERE is_root = 0 AND chunk_root IS NULL AND embedding IS NULL
|
|
301
322
|
AND NULLIF(btrim(session_id), '') IS NOT NULL
|
|
302
323
|
AND content IS NOT NULL
|
|
303
|
-
AND id > $2
|
|
324
|
+
AND id > $2${idClause}
|
|
304
325
|
ORDER BY id
|
|
305
326
|
LIMIT $1
|
|
306
327
|
FOR UPDATE SKIP LOCKED`,
|
|
307
|
-
|
|
328
|
+
params,
|
|
308
329
|
)
|
|
309
330
|
ids = res.rows.map(r => Number(r.id))
|
|
310
331
|
throwIfAborted(signal)
|
|
@@ -674,13 +674,13 @@ LEFT JOIN exact x ON x.id = c.id`
|
|
|
674
674
|
// Short keyword/identifier lookups: pass real lexical legs (FTS/exact)
|
|
675
675
|
// freely, but a trigram-only fuzzy hit riding a sub-floor dense leg is
|
|
676
676
|
// filler — require the strong semantic floor so an unrelated 2-token query
|
|
677
|
-
// (e.g.
|
|
677
|
+
// (e.g. a project identifier plus a non-English word for "balance"
|
|
678
678
|
// against another project) returns empty rather
|
|
679
679
|
// than accidental trigram neighbours.
|
|
680
680
|
if (queryTokenCount <= SHORT_QUERY_TOKEN_MAX) {
|
|
681
681
|
// A real FTS (sparse) hit is a topical match — keep it. But an exact/trgm
|
|
682
682
|
// hit alone can be INCIDENTAL: the term appears only in un-rendered body
|
|
683
|
-
// text (e.g. a
|
|
683
|
+
// text (e.g. a project path buried in a codeGraph note),
|
|
684
684
|
// so the row renders a line that never mentions the query at all. For a
|
|
685
685
|
// short keyword query where every rendered line is expected to be about
|
|
686
686
|
// the term, require the token to actually land in the display fields
|
|
@@ -731,8 +731,8 @@ LEFT JOIN exact x ON x.id = c.id`
|
|
|
731
731
|
const memberRootSeen = new Set()
|
|
732
732
|
// Matched member ids grouped by their chunk root. A member-hit root is a
|
|
733
733
|
// grouping artifact surfaced because a SPECIFIC turn matched; rendering its
|
|
734
|
-
// full sibling set floods precision-sensitive queries (
|
|
735
|
-
//
|
|
734
|
+
// full sibling set floods precision-sensitive queries (the negative-keyword
|
|
735
|
+
// bench case) with turns that never mention the term. Attach only the
|
|
736
736
|
// matched turns for such roots (see membersByRoot filtering below); roots
|
|
737
737
|
// matched on their own row keep full chunk expansion for context.
|
|
738
738
|
const matchedMembersByRoot = new Map()
|
|
@@ -296,6 +296,45 @@ async function ensureHotActiveSearchObjects(db) {
|
|
|
296
296
|
await db.exec(`CREATE INDEX IF NOT EXISTS mv_hot_active_score ON mv_hot_active(score DESC)`)
|
|
297
297
|
}
|
|
298
298
|
|
|
299
|
+
// Refresh mv_hot_active. Owned by cycle2 — call after promotion/archival/dedup
|
|
300
|
+
// mutations land so the recall hot path reads the current active set. The MV is
|
|
301
|
+
// created WITH NO DATA (memory.mjs:243), and an unpopulated MV CANNOT be
|
|
302
|
+
// refreshed CONCURRENTLY, so the first refresh is non-concurrent; once populated
|
|
303
|
+
// we refresh CONCURRENTLY (the mv_hot_active_id UNIQUE index — created at
|
|
304
|
+
// :251/:293 — makes this legal) to avoid an AccessExclusive lock on the recall
|
|
305
|
+
// read path. Best-effort: every failure is logged and swallowed so a refresh
|
|
306
|
+
// error never crashes the cycle. Returns true when the view was refreshed.
|
|
307
|
+
export async function refreshHotActive(db) {
|
|
308
|
+
try {
|
|
309
|
+
const r = await db.query(
|
|
310
|
+
`SELECT relispopulated FROM pg_class WHERE relname = 'mv_hot_active' LIMIT 1`,
|
|
311
|
+
)
|
|
312
|
+
if (!r.rows.length) {
|
|
313
|
+
__mixdogMemoryLog('[memory] refreshHotActive: mv_hot_active not found; skipping\n')
|
|
314
|
+
return false
|
|
315
|
+
}
|
|
316
|
+
if (Boolean(r.rows[0].relispopulated)) {
|
|
317
|
+
try {
|
|
318
|
+
await db.exec(`REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hot_active`)
|
|
319
|
+
return true
|
|
320
|
+
} catch (err) {
|
|
321
|
+
// CONCURRENTLY needs a populated MV + unique index; if either is not
|
|
322
|
+
// true (e.g. MV was reset since the catalog read), fall back to a plain
|
|
323
|
+
// refresh so the view still ends up fresh (brief AccessExclusive lock).
|
|
324
|
+
__mixdogMemoryLog(`[memory] refreshHotActive CONCURRENTLY failed, retrying non-concurrent: ${err?.message || err}\n`)
|
|
325
|
+
await db.exec(`REFRESH MATERIALIZED VIEW mv_hot_active`)
|
|
326
|
+
return true
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
// First refresh of a WITH-NO-DATA view — must be non-concurrent.
|
|
330
|
+
await db.exec(`REFRESH MATERIALIZED VIEW mv_hot_active`)
|
|
331
|
+
return true
|
|
332
|
+
} catch (err) {
|
|
333
|
+
__mixdogMemoryLog(`[memory] refreshHotActive failed: ${err?.message || err}\n`)
|
|
334
|
+
return false
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
299
338
|
async function resetEmbeddingColumnsForDims(db, dimCount) {
|
|
300
339
|
const entriesDims = await getEmbeddingColumnDims(db, 'entries')
|
|
301
340
|
const coreDims = await getEmbeddingColumnDims(db, 'core_entries')
|
|
@@ -487,7 +487,7 @@ export function createQueryHandlers({
|
|
|
487
487
|
if (signal?.aborted) throw signal.reason ?? new Error('aborted')
|
|
488
488
|
let queryVector = null
|
|
489
489
|
if (isEmbeddingModelReady()) {
|
|
490
|
-
queryVector = await embedText(query)
|
|
490
|
+
queryVector = await embedText(query, { priority: true })
|
|
491
491
|
} else if (embeddingWarmupCanStart()) {
|
|
492
492
|
// Cold model: WAIT for warmup (bounded) instead of silently degrading
|
|
493
493
|
// to lexical-only. Raw rows are already embedded at ingest time, so
|
|
@@ -499,7 +499,7 @@ export function createQueryHandlers({
|
|
|
499
499
|
let timer
|
|
500
500
|
try {
|
|
501
501
|
queryVector = await Promise.race([
|
|
502
|
-
warmupEmbeddingProvider().then(() => embedText(query)),
|
|
502
|
+
warmupEmbeddingProvider().then(() => embedText(query, { priority: true })),
|
|
503
503
|
new Promise((resolve) => { timer = setTimeout(() => resolve(null), COLD_RECALL_WARMUP_WAIT_MS) }),
|
|
504
504
|
])
|
|
505
505
|
} catch (err) {
|