mixdog 0.9.16 → 0.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/scripts/atomic-lock-tryonce-test.mjs +66 -0
- package/scripts/provider-toolcall-test.mjs +79 -2
- package/src/mixdog-session-runtime.mjs +12 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +33 -1
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +9 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +49 -0
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +14 -0
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +7 -0
- package/src/runtime/agent/orchestrator/session/loop.mjs +8 -0
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +32 -18
- package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +142 -3
- package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +108 -30
- package/src/runtime/agent/orchestrator/session/store.mjs +5 -0
- package/src/runtime/agent/orchestrator/stall-policy.mjs +22 -12
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +3 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +15 -15
- package/src/runtime/agent/orchestrator/tools/builtin/snapshot-store.mjs +10 -2
- package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +8 -2
- package/src/runtime/channels/backends/discord-attachments.mjs +2 -2
- package/src/runtime/channels/backends/discord-gateway.mjs +12 -2
- package/src/runtime/channels/backends/discord.mjs +97 -7
- package/src/runtime/channels/index.mjs +150 -23
- package/src/runtime/channels/lib/crash-log.mjs +21 -3
- package/src/runtime/channels/lib/output-forwarder.mjs +118 -7
- package/src/runtime/channels/lib/runtime-paths.mjs +21 -19
- package/src/runtime/channels/lib/voice-transcription.mjs +4 -2
- package/src/runtime/memory/index.mjs +37 -0
- package/src/runtime/memory/lib/embedding-warmup.mjs +3 -0
- package/src/runtime/memory/lib/ko-morph.mjs +1 -0
- package/src/runtime/shared/atomic-file.mjs +110 -0
- package/src/runtime/shared/transcript-writer.mjs +46 -4
- package/src/session-runtime/provider-models.mjs +47 -8
- package/src/standalone/channel-worker.mjs +14 -2
- package/src/tui/app/transcript-window.mjs +137 -6
- package/src/tui/app/use-transcript-window.mjs +67 -10
- package/src/tui/components/StatusLine.jsx +1 -1
- package/src/tui/dist/index.mjs +436 -93
- package/src/tui/engine/tui-steering-persist.mjs +66 -35
- package/src/tui/engine.mjs +66 -14
- package/src/tui/index.jsx +97 -6
- package/src/ui/statusline-segments.mjs +54 -36
- package/src/ui/statusline.mjs +141 -95
|
@@ -28,8 +28,24 @@ import {
|
|
|
28
28
|
buildToolLine
|
|
29
29
|
} from "./tool-format.mjs";
|
|
30
30
|
|
|
31
|
+
// Per-item send watchdog. drainQueue awaits ONE backend send at a time; a hung
|
|
32
|
+
// send would otherwise never settle, so drainQueue never returns and `sending`
|
|
33
|
+
// stays true forever — wedging ALL outbound behind the queue head. The watchdog
|
|
34
|
+
// does NOT abandon the send (racing a still-running send against a retry would
|
|
35
|
+
// double-deliver): it only pokes the backend to reset its wedged client so the
|
|
36
|
+
// AWAITED send fails fast and `sending` releases on real settlement. Sized above
|
|
37
|
+
// the backend's worst-case per-attempt retries.
|
|
38
|
+
const SEND_ITEM_TIMEOUT_MS = 120_000;
|
|
39
|
+
// After the watchdog resets the backend, wait at most this long for the
|
|
40
|
+
// abandoned send to actually settle before releasing the queue.
|
|
41
|
+
const SEND_SETTLE_CAP_MS = 15_000;
|
|
42
|
+
const TAIL_RECOVERY_BYTES = 256 * 1024;
|
|
43
|
+
|
|
31
44
|
class OutputForwarder {
|
|
32
45
|
ownerGetter = null;
|
|
46
|
+
// Generation of the in-flight supervised send. Bumped when a send is
|
|
47
|
+
// abandoned (settlement cap) so a late/zombie completion can't double-advance.
|
|
48
|
+
_activeSendGen = 0;
|
|
33
49
|
setOwnerGetter(fn) {
|
|
34
50
|
this.ownerGetter = fn;
|
|
35
51
|
}
|
|
@@ -111,20 +127,35 @@ class OutputForwarder {
|
|
|
111
127
|
try {
|
|
112
128
|
const stat = existsSync(this.transcriptPath) ? statSync(this.transcriptPath) : null;
|
|
113
129
|
const currentSize = stat?.size ?? 0;
|
|
130
|
+
const persisted = this.statusState?.read?.();
|
|
131
|
+
const sameTranscript = persisted?.transcriptPath &&
|
|
132
|
+
sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
|
|
114
133
|
let fileSize;
|
|
115
134
|
if (options.replayFromStart) {
|
|
116
135
|
fileSize = 0;
|
|
117
136
|
} else if (options.catchUpFromPersisted) {
|
|
118
|
-
const persisted = this.statusState?.read?.();
|
|
119
137
|
const persistedSize = typeof persisted?.lastFileSize === "number" ? persisted.lastFileSize : -1;
|
|
120
|
-
const sameTranscript = persisted?.transcriptPath &&
|
|
121
|
-
sameResolvedPath(persisted.transcriptPath, this.transcriptPath);
|
|
122
138
|
fileSize = (sameTranscript && persistedSize >= 0)
|
|
123
139
|
? Math.min(Math.max(persistedSize, 0), currentSize)
|
|
124
140
|
: currentSize;
|
|
125
141
|
} else {
|
|
126
142
|
fileSize = currentSize;
|
|
127
143
|
}
|
|
144
|
+
if (options.recoverUnsyncedTail && fileSize === currentSize && currentSize > 0) {
|
|
145
|
+
const sentCount = Number(persisted?.sentCount ?? 0);
|
|
146
|
+
const lastSentTime = Number(persisted?.lastSentTime ?? 0);
|
|
147
|
+
const hasSentHash = typeof persisted?.lastSentHash === "string" && persisted.lastSentHash.length > 0;
|
|
148
|
+
const noSendEvidence = (!Number.isFinite(sentCount) || sentCount <= 0)
|
|
149
|
+
&& (!Number.isFinite(lastSentTime) || lastSentTime <= 0)
|
|
150
|
+
&& !hasSentHash;
|
|
151
|
+
if (!sameTranscript || noSendEvidence) {
|
|
152
|
+
const tailOffset = this.findLastAssistantTextOffset(currentSize);
|
|
153
|
+
if (tailOffset >= 0 && tailOffset < currentSize) {
|
|
154
|
+
fileSize = tailOffset;
|
|
155
|
+
dropTrace("context.recoverUnsyncedTail", { path: this.transcriptPath, offset: tailOffset, currentSize, sameTranscript: !!sameTranscript });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
128
159
|
this.lastFileSize = fileSize;
|
|
129
160
|
this.readFileSize = fileSize;
|
|
130
161
|
} catch {
|
|
@@ -132,6 +163,46 @@ class OutputForwarder {
|
|
|
132
163
|
this.readFileSize = 0;
|
|
133
164
|
}
|
|
134
165
|
}
|
|
166
|
+
findLastAssistantTextOffset(fileSize) {
|
|
167
|
+
if (!this.transcriptPath || fileSize <= 0) return -1;
|
|
168
|
+
const bytesToRead = Math.min(fileSize, TAIL_RECOVERY_BYTES);
|
|
169
|
+
const startOffset = fileSize - bytesToRead;
|
|
170
|
+
let fd = null;
|
|
171
|
+
try {
|
|
172
|
+
fd = openSync(this.transcriptPath, "r");
|
|
173
|
+
const buf = Buffer.alloc(bytesToRead);
|
|
174
|
+
readSync(fd, buf, 0, bytesToRead, startOffset);
|
|
175
|
+
const raw = buf.toString("utf8");
|
|
176
|
+
let chunkStart = 0;
|
|
177
|
+
if (startOffset > 0) {
|
|
178
|
+
const firstNewline = raw.indexOf("\n");
|
|
179
|
+
if (firstNewline < 0) return -1;
|
|
180
|
+
chunkStart = firstNewline + 1;
|
|
181
|
+
}
|
|
182
|
+
let cursor = startOffset + chunkStart;
|
|
183
|
+
let lastOffset = -1;
|
|
184
|
+
for (const line of raw.slice(chunkStart).split("\n")) {
|
|
185
|
+
const lineOffset = cursor;
|
|
186
|
+
cursor += Buffer.byteLength(line, "utf8") + 1;
|
|
187
|
+
if (!line.trim()) continue;
|
|
188
|
+
try {
|
|
189
|
+
const entry = JSON.parse(line);
|
|
190
|
+
if (entry.isSidechain || entry.type !== "assistant") continue;
|
|
191
|
+
const parts = entry.message?.content;
|
|
192
|
+
if (!Array.isArray(parts)) continue;
|
|
193
|
+
if (parts.some((c) => c?.type === "text" && typeof c.text === "string" && c.text.trim())) {
|
|
194
|
+
lastOffset = lineOffset;
|
|
195
|
+
}
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return lastOffset;
|
|
200
|
+
} catch {
|
|
201
|
+
return -1;
|
|
202
|
+
} finally {
|
|
203
|
+
if (fd != null) closeSync(fd);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
135
206
|
/** Reset counters for new turn */
|
|
136
207
|
reset() {
|
|
137
208
|
this.sentCount = 0;
|
|
@@ -282,6 +353,7 @@ class OutputForwarder {
|
|
|
282
353
|
}
|
|
283
354
|
async deliverQueueItem(item) {
|
|
284
355
|
const targetChannelId = item.channelId ?? this.channelId;
|
|
356
|
+
const gen = ++this._activeSendGen;
|
|
285
357
|
if (!item.text || !targetChannelId) {
|
|
286
358
|
this.commitReadProgress(item.nextFileSize);
|
|
287
359
|
return;
|
|
@@ -314,10 +386,13 @@ class OutputForwarder {
|
|
|
314
386
|
// re-sending chunks that already landed. The token is opaque here —
|
|
315
387
|
// the forwarder only stores/passes/clears it, never interprets it.
|
|
316
388
|
await this.cb.send(targetChannelId, formatted, item._resumeToken ? { resumeToken: item._resumeToken } : undefined);
|
|
389
|
+
dropTrace("discord.send.ok", null);
|
|
390
|
+
// A zombie completion (send abandoned past the settlement cap, gen bumped)
|
|
391
|
+
// must not mutate ANY item/queue state — including the resume token.
|
|
392
|
+
if (gen !== this._activeSendGen) return;
|
|
317
393
|
// Full success — drop any stale token so a later reuse of this item
|
|
318
394
|
// object can't carry a dead token.
|
|
319
395
|
item._resumeToken = undefined;
|
|
320
|
-
dropTrace("discord.send.ok", null);
|
|
321
396
|
} catch (err) {
|
|
322
397
|
// Persist the opaque resume token (if any) on the item so the requeued
|
|
323
398
|
// retry resumes from the failed chunk.
|
|
@@ -349,7 +424,18 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
349
424
|
}
|
|
350
425
|
/** Forward new assistant text to Discord. Returns true if text was sent. */
|
|
351
426
|
async forwardNewText() {
|
|
352
|
-
if (!this._isOwner())
|
|
427
|
+
if (!this._isOwner()) {
|
|
428
|
+
// Silent no-send is invisible and has masked real ownership-gate bugs
|
|
429
|
+
// (a live seat that never forwards). Surface it, rate-limited to one
|
|
430
|
+
// line per 60s so a persistent non-owner state stays visible without
|
|
431
|
+
// flooding stderr on every poll tick.
|
|
432
|
+
const now = Date.now();
|
|
433
|
+
if (!this._lastOwnerGateLogAt || now - this._lastOwnerGateLogAt >= 60_000) {
|
|
434
|
+
this._lastOwnerGateLogAt = now;
|
|
435
|
+
process.stderr.write(`[output-forwarder] forwardNewText skipped: not seat owner (channelId=${this.channelId ?? "none"})\n`);
|
|
436
|
+
}
|
|
437
|
+
return false;
|
|
438
|
+
}
|
|
353
439
|
if (!this.channelId) return false;
|
|
354
440
|
const { text: newText, nextFileSize } = this.extractNewText();
|
|
355
441
|
if (!newText) {
|
|
@@ -428,6 +514,31 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
428
514
|
_kickDrain() {
|
|
429
515
|
this.drainQueue().catch(err => process.stderr.write(`[output-forwarder] drainQueue rejected: ${err?.message || err}\n`));
|
|
430
516
|
}
|
|
517
|
+
/** Supervise a per-item send. The underlying promise is AWAITED to real
|
|
518
|
+
* settlement so `sending` is never released early (no unsupervised send
|
|
519
|
+
* racing a retry → duplicate). The watchdog's only job is to reset the
|
|
520
|
+
* backend's wedged client so the awaited send fails fast. */
|
|
521
|
+
_sendSupervised(promise, label) {
|
|
522
|
+
const settled = Promise.resolve(promise);
|
|
523
|
+
settled.catch(() => {}); // never leak an unhandled rejection on late settle
|
|
524
|
+
return new Promise((resolve, reject) => {
|
|
525
|
+
let capTimer = null;
|
|
526
|
+
const watchdog = setTimeout(() => {
|
|
527
|
+
process.stderr.write(`[output-forwarder] ${label} watchdog fired after ${SEND_ITEM_TIMEOUT_MS}ms; resetting backend\n`);
|
|
528
|
+
Promise.resolve(this.cb?.resetBackend?.()).catch(() => {});
|
|
529
|
+
// Bound the post-reset settlement wait; past the cap, invalidate the
|
|
530
|
+
// in-flight send (gen bump → zombie can't double-advance) and release.
|
|
531
|
+
capTimer = setTimeout(() => {
|
|
532
|
+
this._activeSendGen++;
|
|
533
|
+
reject(new Error(`${label} abandoned after ${SEND_SETTLE_CAP_MS}ms settlement cap`));
|
|
534
|
+
}, SEND_SETTLE_CAP_MS);
|
|
535
|
+
}, SEND_ITEM_TIMEOUT_MS);
|
|
536
|
+
settled.then(
|
|
537
|
+
(v) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); resolve(v); },
|
|
538
|
+
(e) => { clearTimeout(watchdog); if (capTimer) clearTimeout(capTimer); reject(e); }
|
|
539
|
+
);
|
|
540
|
+
});
|
|
541
|
+
}
|
|
431
542
|
/** Drain both priority lanes sequentially. finalLane drains first when non-empty. */
|
|
432
543
|
async drainQueue() {
|
|
433
544
|
if (this.sending) return;
|
|
@@ -440,9 +551,9 @@ ${_bt.trim()}` : _bt.trim();
|
|
|
440
551
|
const item = lane[0];
|
|
441
552
|
try {
|
|
442
553
|
if (item.type === "text") {
|
|
443
|
-
await this.deliverQueueItem(item);
|
|
554
|
+
await this._sendSupervised(this.deliverQueueItem(item), "deliverQueueItem");
|
|
444
555
|
} else if (item.type === "toolLog") {
|
|
445
|
-
await this.processToolLog(item);
|
|
556
|
+
await this._sendSupervised(this.processToolLog(item), "processToolLog");
|
|
446
557
|
}
|
|
447
558
|
lane.shift();
|
|
448
559
|
} catch (err) {
|
|
@@ -38,17 +38,6 @@ function parsePositivePid(value) {
|
|
|
38
38
|
function getTerminalLeadPid() {
|
|
39
39
|
return parsePositivePid(process.env.MIXDOG_SUPERVISOR_PID) ?? process.pid;
|
|
40
40
|
}
|
|
41
|
-
// Pinned-owner default: ON. The first session to claim ownership auto-pins
|
|
42
|
-
// itself and keeps the seat until its PID dies. Opt out per session with
|
|
43
|
-
// MIXDOG_PIN_OWNER=0 (or false/no/off) when you want the legacy 10 s stale-
|
|
44
|
-
// window takeover behavior — e.g. throwaway shells that should never become
|
|
45
|
-
// the sticky main.
|
|
46
|
-
function isOwnerPinEnabled() {
|
|
47
|
-
const v = process.env.MIXDOG_PIN_OWNER;
|
|
48
|
-
if (v === undefined || v === null || v === "") return true;
|
|
49
|
-
const low = String(v).toLowerCase();
|
|
50
|
-
return !(low === "0" || low === "false" || low === "no" || low === "off");
|
|
51
|
-
}
|
|
52
41
|
function getServerPid() {
|
|
53
42
|
return parsePositivePid(process.env.MIXDOG_SERVER_PID) ?? (process.env.MIXDOG_WORKER_MODE === "1" ? null : process.pid);
|
|
54
43
|
}
|
|
@@ -104,8 +93,10 @@ function touchUiHeartbeat(instanceId) {
|
|
|
104
93
|
if (!curRaw) return undefined;
|
|
105
94
|
if (instanceId && curRaw.instanceId !== instanceId) return undefined;
|
|
106
95
|
return { ...curRaw, ui_heartbeat_at: Date.now(), updatedAt: Date.now() };
|
|
107
|
-
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate' });
|
|
108
|
-
} catch { /* best-effort
|
|
96
|
+
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate', timeoutMs: 0 });
|
|
97
|
+
} catch { /* best-effort try-once (timeoutMs:0): on lock contention we skip
|
|
98
|
+
this tick without ever blocking the render loop on Atomics.wait;
|
|
99
|
+
the next 30s tick catches up. */ }
|
|
109
100
|
}
|
|
110
101
|
function buildRuntimeIdentity() {
|
|
111
102
|
const terminalLeadPid = getTerminalLeadPid();
|
|
@@ -294,12 +285,10 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
294
285
|
if (outBase !== newBase) next.priorTranscriptPath = outgoing;
|
|
295
286
|
}
|
|
296
287
|
}
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
//
|
|
300
|
-
|
|
301
|
-
if (isOwnerPinEnabled()) next.pinned = true;
|
|
302
|
-
else delete next.pinned;
|
|
288
|
+
// Ownership is strict last-wins (no pin): the newest claim always takes
|
|
289
|
+
// the seat. Drop any legacy `pinned` flag left by older versions — it was
|
|
290
|
+
// written but never consumed by any takeover check.
|
|
291
|
+
delete next.pinned;
|
|
303
292
|
// I1: pg_* spreads FIRST so newFields above win on conflict.
|
|
304
293
|
// prev.pg_port='A', meta.httpPort-adjacent pg_port='B' → result.pg_port='B'.
|
|
305
294
|
// memory_port: preserve when the advertising memory worker is still alive
|
|
@@ -371,6 +360,19 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
371
360
|
}
|
|
372
361
|
}
|
|
373
362
|
}
|
|
363
|
+
// Worker-owned seat: when THIS process is the channels worker and it is
|
|
364
|
+
// the seat owner (no distinct TUI/supervisor Lead — ownerLeadPid resolves
|
|
365
|
+
// to our own pid), keep ui_heartbeat_at fresh. A headless worker never
|
|
366
|
+
// runs the TUI render loop, so an inherited heartbeat (written by a prior
|
|
367
|
+
// TUI session and carried forward in prevRest) would otherwise go stale at
|
|
368
|
+
// UI_HEARTBEAT_STALE_MS and falsely flag a live worker-owned seat. A
|
|
369
|
+
// TUI-owned seat has a distinct supervisor (ownerLeadPid !== our pid), so
|
|
370
|
+
// this branch is a no-op there and TUI staleness is preserved.
|
|
371
|
+
const workerOwnsSeat = process.env.MIXDOG_WORKER_MODE === "1"
|
|
372
|
+
&& identity.ownerLeadPid === process.pid;
|
|
373
|
+
if (workerOwnsSeat) {
|
|
374
|
+
next.ui_heartbeat_at = Date.now();
|
|
375
|
+
}
|
|
374
376
|
return { ...preservedExtra, ...next };
|
|
375
377
|
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate' });
|
|
376
378
|
}
|
|
@@ -167,9 +167,11 @@ function createVoiceTranscription({ getConfig, dataDir }) {
|
|
|
167
167
|
if (attachmentId && result) _voiceTranscriptCache.set(attachmentId, result);
|
|
168
168
|
return result;
|
|
169
169
|
} catch (err) {
|
|
170
|
-
|
|
170
|
+
// Propagate ALL real failures so the caller's retry (network-class) and
|
|
171
|
+
// failure marker (index.mjs catch) fire; null is reserved strictly for a
|
|
172
|
+
// legit empty transcript.
|
|
171
173
|
process.stderr.write(`mixdog: voice.transcription failed: ${err}\n`);
|
|
172
|
-
|
|
174
|
+
throw err;
|
|
173
175
|
}
|
|
174
176
|
}
|
|
175
177
|
|
|
@@ -10,6 +10,7 @@ process.on('warning', () => {})
|
|
|
10
10
|
|
|
11
11
|
import http from 'node:http'
|
|
12
12
|
import os from 'node:os'
|
|
13
|
+
import { performance } from 'node:perf_hooks'
|
|
13
14
|
import fs from 'node:fs'
|
|
14
15
|
import path from 'node:path'
|
|
15
16
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
@@ -102,6 +103,17 @@ if (!DATA_DIR) {
|
|
|
102
103
|
process.exit(1)
|
|
103
104
|
}
|
|
104
105
|
__mixdogMemoryLog(`[memory-service] DATA_DIR=${DATA_DIR}\n`)
|
|
106
|
+
const MEMORY_PROFILE_ENABLED = /^(1|true|yes|on)$/i.test(String(process.env.MIXDOG_MEMORY_PROFILE || process.env.MIXDOG_BOOT_PROFILE || ''))
|
|
107
|
+
const MEMORY_PROFILE_START = performance.now()
|
|
108
|
+
function memoryProfile(event, fields = {}) {
|
|
109
|
+
if (!MEMORY_PROFILE_ENABLED) return
|
|
110
|
+
const parts = [`[memory-profile] +${(performance.now() - MEMORY_PROFILE_START).toFixed(1)}ms`, event]
|
|
111
|
+
for (const [key, value] of Object.entries(fields || {})) {
|
|
112
|
+
if (value === undefined || value === null || value === '') continue
|
|
113
|
+
parts.push(`${key}=${String(value).replace(/\s+/g, '_')}`)
|
|
114
|
+
}
|
|
115
|
+
__mixdogMemoryLog(`${parts.join(' ')}\n`)
|
|
116
|
+
}
|
|
105
117
|
|
|
106
118
|
import {
|
|
107
119
|
parsePositivePid,
|
|
@@ -283,6 +295,7 @@ function _ingestIdentityFields(m) {
|
|
|
283
295
|
_ingestIdentityFieldsCache.set(m, fields)
|
|
284
296
|
return fields
|
|
285
297
|
}
|
|
298
|
+
|
|
286
299
|
// Ingest now WAITS for its embedding flush (bounded) so freshly ingested rows
|
|
287
300
|
// are dense-searchable the moment ingest_session returns. On a cold ONNX
|
|
288
301
|
// model the flush can stall on warmup; the race below caps the wait and lets
|
|
@@ -339,6 +352,8 @@ function assertSecondaryPgAttachable() {
|
|
|
339
352
|
}
|
|
340
353
|
|
|
341
354
|
async function _initStore() {
|
|
355
|
+
const initStoreStartedAt = performance.now()
|
|
356
|
+
memoryProfile('init-store:start')
|
|
342
357
|
mainConfig = readMainConfig()
|
|
343
358
|
const embeddingConfig = mainConfig?.embedding
|
|
344
359
|
if (embeddingConfig?.provider || embeddingConfig?.ollamaModel || embeddingConfig?.dtype) {
|
|
@@ -380,17 +395,24 @@ async function _initStore() {
|
|
|
380
395
|
if (dimsResolved) {
|
|
381
396
|
primeEmbeddingDims(dimsResolved)
|
|
382
397
|
assertSecondaryPgAttachable()
|
|
398
|
+
const openStartedAt = performance.now()
|
|
383
399
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
400
|
+
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
384
401
|
_embeddingWarmup.schedule(EMBEDDING_META_PATH, metaKey)
|
|
385
402
|
} else {
|
|
386
403
|
if (!embeddingWarmupCanStart()) {
|
|
387
404
|
throw new Error('memory-service: embedding dims unavailable while warmup is disabled')
|
|
388
405
|
}
|
|
389
406
|
// Cold path: meta missed AND model not registered. Sequential.
|
|
407
|
+
const warmupStartedAt = performance.now()
|
|
408
|
+
memoryProfile('embedding:cold-warmup:start')
|
|
390
409
|
await warmupEmbeddingProvider()
|
|
410
|
+
memoryProfile('embedding:cold-warmup:done', { ms: (performance.now() - warmupStartedAt).toFixed(1) })
|
|
391
411
|
dimsResolved = Number(getEmbeddingDims())
|
|
392
412
|
assertSecondaryPgAttachable()
|
|
413
|
+
const openStartedAt = performance.now()
|
|
393
414
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
415
|
+
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
394
416
|
try {
|
|
395
417
|
writeJsonAtomicSync(EMBEDDING_META_PATH, { ...metaKey, dims: dimsResolved }, { lock: true })
|
|
396
418
|
} catch (e) {
|
|
@@ -429,13 +451,18 @@ async function _initStore() {
|
|
|
429
451
|
if (cyclesCapable || memoryLlmWorkerEnabled()) {
|
|
430
452
|
try {
|
|
431
453
|
const agentCfg = loadAgentConfig()
|
|
454
|
+
const providersStartedAt = performance.now()
|
|
432
455
|
await initProviders(agentCfg.providers || {})
|
|
456
|
+
memoryProfile('providers:init:done', { ms: (performance.now() - providersStartedAt).toFixed(1) })
|
|
433
457
|
} catch (e) {
|
|
434
458
|
process.stderr.write(`[memory-service] initProviders failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`)
|
|
435
459
|
}
|
|
436
460
|
}
|
|
437
461
|
_bootTimestamp = Date.now()
|
|
462
|
+
const offsetsStartedAt = performance.now()
|
|
438
463
|
await loadTranscriptOffsets()
|
|
464
|
+
memoryProfile('transcript-offsets:loaded', { ms: (performance.now() - offsetsStartedAt).toFixed(1) })
|
|
465
|
+
memoryProfile('init-store:done', { ms: (performance.now() - initStoreStartedAt).toFixed(1) })
|
|
439
466
|
}
|
|
440
467
|
|
|
441
468
|
async function getCycleLastRun() {
|
|
@@ -679,11 +706,16 @@ function _stopCycles() {
|
|
|
679
706
|
|
|
680
707
|
async function _initRuntime() {
|
|
681
708
|
if (_initialized) return
|
|
709
|
+
const runtimeStartedAt = performance.now()
|
|
710
|
+
memoryProfile('runtime-init:start')
|
|
682
711
|
await _initStore()
|
|
712
|
+
memoryProfile('runtime-init:init-store-ready', { ms: (performance.now() - runtimeStartedAt).toFixed(1) })
|
|
683
713
|
// Restore the core_entries.id == 1..N invariant once per boot: SERIAL only
|
|
684
714
|
// increments, so deleted rows leave permanent gaps. Fast no-op when already
|
|
685
715
|
// contiguous (or empty). Runs only here — never in cycle2/addCore/deleteCore.
|
|
716
|
+
const compactStartedAt = performance.now()
|
|
686
717
|
await compactCoreIds(DATA_DIR)
|
|
718
|
+
memoryProfile('core-ids:compact:done', { ms: (performance.now() - compactStartedAt).toFixed(1) })
|
|
687
719
|
// Memory module is always-on: the transcript watcher/ingest runs
|
|
688
720
|
// unconditionally except in secondary mode (secondary attaches to a primary's
|
|
689
721
|
// PG and must not double-ingest). The recap toggle only gates whether the
|
|
@@ -697,7 +729,9 @@ async function _initRuntime() {
|
|
|
697
729
|
__mixdogMemoryLog('[memory-service] secondary mode; skipping transcript watcher\n')
|
|
698
730
|
}
|
|
699
731
|
if (!memorySecondaryMode() && !envFlagEnabled('MIXDOG_MEMORY_DISABLE_CYCLES')) {
|
|
732
|
+
const cyclesStartedAt = performance.now()
|
|
700
733
|
_startCycles()
|
|
734
|
+
memoryProfile('cycles:start:done', { ms: (performance.now() - cyclesStartedAt).toFixed(1) })
|
|
701
735
|
} else {
|
|
702
736
|
__mixdogMemoryLog('[memory-service] background cycle tick loop not started (secondary/env-disabled)\n')
|
|
703
737
|
}
|
|
@@ -705,6 +739,7 @@ async function _initRuntime() {
|
|
|
705
739
|
// Boot complete — continue straight into the deferred embedding warmup.
|
|
706
740
|
// Fire-and-forget on the embedding worker thread; never awaited so it does
|
|
707
741
|
// not delay init() returning or the memory-ready signal.
|
|
742
|
+
memoryProfile('embedding:warmup:fire')
|
|
708
743
|
_embeddingWarmup.fireDeferred()
|
|
709
744
|
// Boot-edge Korean morph warmup. Same fire-and-forget style: lazy async init
|
|
710
745
|
// downloads+caches the Kiwi model once under DATA_DIR/kiwi-model/<version>,
|
|
@@ -712,8 +747,10 @@ async function _initRuntime() {
|
|
|
712
747
|
// buildFtsQuery keeps the websearch_to_tsquery fallback (never throws here).
|
|
713
748
|
// Skipped in secondary mode (query path lives on the primary).
|
|
714
749
|
if (!memorySecondaryMode()) {
|
|
750
|
+
memoryProfile('ko-morph:warmup:fire')
|
|
715
751
|
initKoMorph(DATA_DIR, __mixdogMemoryLog).catch(() => {})
|
|
716
752
|
}
|
|
753
|
+
memoryProfile('runtime-init:done', { ms: (performance.now() - runtimeStartedAt).toFixed(1) })
|
|
717
754
|
}
|
|
718
755
|
|
|
719
756
|
function _beginRuntimeInit() {
|
|
@@ -28,8 +28,11 @@ export function createEmbeddingWarmup({
|
|
|
28
28
|
return
|
|
29
29
|
}
|
|
30
30
|
_pending = () => {
|
|
31
|
+
const startedAt = Date.now()
|
|
32
|
+
log('[memory-service] embedding warmup start\n')
|
|
31
33
|
warmup()
|
|
32
34
|
.then(() => {
|
|
35
|
+
log(`[memory-service] embedding warmup ready in ${Date.now() - startedAt}ms\n`)
|
|
33
36
|
const measured = Number(getDims())
|
|
34
37
|
try {
|
|
35
38
|
persistMeta(metaPath, { ...metaKey, dims: measured })
|
|
@@ -130,6 +130,7 @@ export async function init(dataDir, log = () => {}) {
|
|
|
130
130
|
_log = typeof log === 'function' ? log : (() => {})
|
|
131
131
|
_state = 'loading'
|
|
132
132
|
const t0 = Date.now()
|
|
133
|
+
_log(`[memory-service] kiwi morph init start (model ${KIWI_MODEL_VERSION})\n`)
|
|
133
134
|
_initPromise = (async () => {
|
|
134
135
|
// Resolve the WASM path from the installed package without hard-importing
|
|
135
136
|
// (keeps the whole feature optional if kiwi-nlp isn't installed).
|
|
@@ -63,6 +63,16 @@ export function withFileLockSync(lockPath, fn, opts = {}) {
|
|
|
63
63
|
} catch (err) {
|
|
64
64
|
lastErr = err;
|
|
65
65
|
if (!LOCK_WAIT_CODES.has(err?.code)) throw err;
|
|
66
|
+
// try-once semantics: timeoutMs:0 makes a single openSync attempt and
|
|
67
|
+
// never sleeps — on contention it throws ELOCKCONTENDED immediately so
|
|
68
|
+
// the main loop is never blocked by Atomics.wait. No stale-reclaim,
|
|
69
|
+
// since reclaim would require waiting on the guard.
|
|
70
|
+
if (timeoutMs <= 0) {
|
|
71
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
72
|
+
contErr.code = 'ELOCKCONTENDED';
|
|
73
|
+
contErr.cause = err;
|
|
74
|
+
throw contErr;
|
|
75
|
+
}
|
|
66
76
|
try {
|
|
67
77
|
const st = statSync(lockPath);
|
|
68
78
|
if (Date.now() - st.mtimeMs > staleMs) {
|
|
@@ -246,6 +256,86 @@ function _releaseReclaimGuard(reclaim) {
|
|
|
246
256
|
}
|
|
247
257
|
}
|
|
248
258
|
|
|
259
|
+
// ── Async lock variant (non-blocking wait) ──────────────────────────
|
|
260
|
+
// Identical lock protocol to withFileLockSync (same lockPath, same pid
|
|
261
|
+
// stamp, same stale/ownership/reclaim rules) so a sync holder and an
|
|
262
|
+
// async holder are mutually exclusive against one another — they contend
|
|
263
|
+
// for the SAME openSync('wx') on the SAME path. The ONLY difference is
|
|
264
|
+
// the wait: instead of Atomics.wait blocking the event loop, we yield via
|
|
265
|
+
// setTimeout backoff. Intended for UI-process consistency-required writes
|
|
266
|
+
// that must not stall the render loop.
|
|
267
|
+
function _asyncSleep(ms) {
|
|
268
|
+
return new Promise((resolve) => { setTimeout(resolve, Math.max(1, Number(ms) || 1)); });
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export async function withFileLock(lockPath, fn, opts = {}) {
|
|
272
|
+
const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : DEFAULT_LOCK_TIMEOUT_MS;
|
|
273
|
+
const staleMs = Number.isFinite(opts.staleMs) ? opts.staleMs : 30000;
|
|
274
|
+
const deadline = Date.now() + timeoutMs;
|
|
275
|
+
mkdirSync(dirname(lockPath), { recursive: true });
|
|
276
|
+
let attempt = 0;
|
|
277
|
+
let lastErr = null;
|
|
278
|
+
while (true) {
|
|
279
|
+
let fd;
|
|
280
|
+
try {
|
|
281
|
+
fd = openSync(lockPath, 'wx');
|
|
282
|
+
} catch (err) {
|
|
283
|
+
lastErr = err;
|
|
284
|
+
if (!LOCK_WAIT_CODES.has(err?.code)) throw err;
|
|
285
|
+
if (timeoutMs <= 0) {
|
|
286
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
287
|
+
contErr.code = 'ELOCKCONTENDED';
|
|
288
|
+
contErr.cause = err;
|
|
289
|
+
throw contErr;
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
const st = statSync(lockPath);
|
|
293
|
+
if (Date.now() - st.mtimeMs > staleMs) {
|
|
294
|
+
const deadPid = _readLockOwnerPid(lockPath);
|
|
295
|
+
if (deadPid !== null && _pidIsDead(deadPid)) {
|
|
296
|
+
const reclaim = _tryAcquireReclaimGuard(lockPath, staleMs);
|
|
297
|
+
if (reclaim !== null) {
|
|
298
|
+
let reclaimed = false;
|
|
299
|
+
try {
|
|
300
|
+
const currentSt = statSync(lockPath);
|
|
301
|
+
if (Date.now() - currentSt.mtimeMs > staleMs) {
|
|
302
|
+
const currentPid = _readLockOwnerPid(lockPath);
|
|
303
|
+
if (currentPid === deadPid && _pidIsDead(currentPid)) {
|
|
304
|
+
try { unlinkSync(lockPath); reclaimed = true; } catch {}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} finally {
|
|
308
|
+
_releaseReclaimGuard(reclaim);
|
|
309
|
+
}
|
|
310
|
+
if (reclaimed) continue;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
} catch {}
|
|
315
|
+
if (Date.now() >= deadline) break;
|
|
316
|
+
const base = DEFAULT_BACKOFFS_MS[Math.min(attempt, DEFAULT_BACKOFFS_MS.length - 1)];
|
|
317
|
+
const jitter = Math.floor(Math.random() * Math.min(75, Math.max(1, base)));
|
|
318
|
+
await _asyncSleep(Math.min(Math.max(1, deadline - Date.now()), base + jitter));
|
|
319
|
+
attempt += 1;
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
try { writeFileSync(fd, `${process.pid} ${Date.now()}\n`, 'utf8'); } catch {}
|
|
323
|
+
try {
|
|
324
|
+
if (opts.secret === true) _enforceOwnerOnlyAclWin32(lockPath);
|
|
325
|
+
return await fn();
|
|
326
|
+
} finally {
|
|
327
|
+
try { closeSync(fd); } catch {}
|
|
328
|
+
try {
|
|
329
|
+
if (_lockOwnedByPid(lockPath, process.pid)) unlinkSync(lockPath);
|
|
330
|
+
} catch {}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath}`);
|
|
334
|
+
timeoutErr.code = 'ELOCKTIMEOUT';
|
|
335
|
+
timeoutErr.cause = lastErr;
|
|
336
|
+
throw timeoutErr;
|
|
337
|
+
}
|
|
338
|
+
|
|
249
339
|
// ── Windows owner-only ACL enforcement (fail-closed) ─────────────────
|
|
250
340
|
// POSIX 0o600/0o700 mode bits are advisory-at-best on Windows: NTFS
|
|
251
341
|
// ignores them and the file inherits the parent ACL, which in shared or
|
|
@@ -459,3 +549,23 @@ export function updateJsonAtomicSync(filePath, mutator, opts = {}) {
|
|
|
459
549
|
return next;
|
|
460
550
|
}, opts);
|
|
461
551
|
}
|
|
552
|
+
|
|
553
|
+
// Async read-modify-write. Same lock path (`${filePath}.lock`) and protocol
|
|
554
|
+
// as updateJsonAtomicSync, so it is mutually exclusive with the sync variant.
|
|
555
|
+
// The critical section itself is synchronous (readFileSync + writeJsonAtomicSync);
|
|
556
|
+
// only the lock WAIT is async, so the event loop is never blocked on contention.
|
|
557
|
+
export async function updateJsonAtomic(filePath, mutator, opts = {}) {
|
|
558
|
+
const { lock: _lock, ...writeOpts } = opts;
|
|
559
|
+
return withFileLock(`${filePath}.lock`, () => {
|
|
560
|
+
let cur = null;
|
|
561
|
+
try {
|
|
562
|
+
cur = JSON.parse(readFileSync(filePath, 'utf8'));
|
|
563
|
+
} catch {
|
|
564
|
+
cur = null;
|
|
565
|
+
}
|
|
566
|
+
const next = mutator(cur);
|
|
567
|
+
if (next === undefined) return cur;
|
|
568
|
+
writeJsonAtomicSync(filePath, next, { ...writeOpts, lock: false });
|
|
569
|
+
return next;
|
|
570
|
+
}, opts);
|
|
571
|
+
}
|
|
@@ -15,6 +15,13 @@
|
|
|
15
15
|
* cannot spam the terminal.
|
|
16
16
|
*/
|
|
17
17
|
import { existsSync, mkdirSync, renameSync, statSync, writeFileSync } from 'node:fs';
|
|
18
|
+
|
|
19
|
+
// After this many appended bytes since the last rotation stat, re-check
|
|
20
|
+
// disk size via statSync. Bounds the sync-fs cost per append to roughly
|
|
21
|
+
// once every ~64KB of transcript growth instead of on every single line,
|
|
22
|
+
// while still catching rotation promptly (a JSONL line is typically well
|
|
23
|
+
// under 1KB, so this is on the order of dozens of appends between stats).
|
|
24
|
+
const ROTATE_CHECK_BYTE_STRIDE = 64 * 1024;
|
|
18
25
|
import { dirname, join, resolve } from 'node:path';
|
|
19
26
|
import { appendBuffered, drainPathSync, hasInFlightWrite } from './buffered-appender.mjs';
|
|
20
27
|
|
|
@@ -62,10 +69,28 @@ export function createTranscriptWriter({ mixdogHome, sessionId, cwd, pid } = {})
|
|
|
62
69
|
}
|
|
63
70
|
}
|
|
64
71
|
|
|
65
|
-
|
|
72
|
+
// Local byte counter tracking growth since the last real statSync, so
|
|
73
|
+
// rotateIfNeeded's fs work is O(1) per append instead of an existsSync+
|
|
74
|
+
// statSync pair on every single JSONL line (this fires on every assistant
|
|
75
|
+
// chunk / tool call, i.e. per token-stream event, not per turn). We only
|
|
76
|
+
// hit disk again once the locally-tracked size estimate could plausibly
|
|
77
|
+
// have crossed the rotate threshold, or a full check hasn't happened yet.
|
|
78
|
+
let sizeChecked = false;
|
|
79
|
+
let lastKnownSize = 0;
|
|
80
|
+
let bytesSinceCheck = 0;
|
|
81
|
+
|
|
82
|
+
function statAndMaybeRotate() {
|
|
66
83
|
try {
|
|
67
|
-
if (!existsSync(transcriptPath))
|
|
84
|
+
if (!existsSync(transcriptPath)) {
|
|
85
|
+
sizeChecked = true;
|
|
86
|
+
lastKnownSize = 0;
|
|
87
|
+
bytesSinceCheck = 0;
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
68
90
|
const { size } = statSync(transcriptPath);
|
|
91
|
+
sizeChecked = true;
|
|
92
|
+
lastKnownSize = size;
|
|
93
|
+
bytesSinceCheck = 0;
|
|
69
94
|
if (size < TRANSCRIPT_ROTATE_BYTES) return;
|
|
70
95
|
// An async appendFile may be in flight for this path; renaming out
|
|
71
96
|
// from under it races the write on Windows. Skip this round and
|
|
@@ -76,17 +101,34 @@ export function createTranscriptWriter({ mixdogHome, sessionId, cwd, pid } = {})
|
|
|
76
101
|
// file post-rename doesn't inherit stale in-memory chunks.
|
|
77
102
|
drainPathSync(transcriptPath);
|
|
78
103
|
const rotatedPath = `${transcriptPath}.1`;
|
|
79
|
-
try {
|
|
104
|
+
try {
|
|
105
|
+
renameSync(transcriptPath, rotatedPath);
|
|
106
|
+
lastKnownSize = 0;
|
|
107
|
+
} catch (err) { logOnce(err); }
|
|
80
108
|
} catch (err) {
|
|
81
109
|
logOnce(err);
|
|
82
110
|
}
|
|
83
111
|
}
|
|
84
112
|
|
|
113
|
+
function rotateIfNeeded() {
|
|
114
|
+
// Real stat only when: no check has happened yet, the locally-tracked
|
|
115
|
+
// growth alone could have crossed the rotate threshold, or the last
|
|
116
|
+
// known size plus tracked growth is already at/over the threshold
|
|
117
|
+
// (catches a file that was already large when this process attached).
|
|
118
|
+
if (!sizeChecked
|
|
119
|
+
|| bytesSinceCheck >= ROTATE_CHECK_BYTE_STRIDE
|
|
120
|
+
|| lastKnownSize + bytesSinceCheck >= TRANSCRIPT_ROTATE_BYTES) {
|
|
121
|
+
statAndMaybeRotate();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
85
125
|
function appendLine(entry) {
|
|
86
126
|
ensureProjectDir();
|
|
87
127
|
rotateIfNeeded();
|
|
88
128
|
try {
|
|
89
|
-
|
|
129
|
+
const line = `${JSON.stringify(entry)}\n`;
|
|
130
|
+
appendBuffered(transcriptPath, line);
|
|
131
|
+
bytesSinceCheck += Buffer.byteLength(line);
|
|
90
132
|
} catch (err) {
|
|
91
133
|
logOnce(err);
|
|
92
134
|
}
|