mixdog 0.9.16 → 0.9.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/scripts/atomic-lock-tryonce-test.mjs +66 -0
- package/scripts/bench/cache-probe-tasks.json +1 -1
- package/scripts/bench/lead-review-tasks-r3.json +1 -1
- package/scripts/bench/r4-mixed-tasks.json +1 -1
- package/scripts/bench/review-tasks.json +1 -1
- package/scripts/build-runtime-windows.ps1 +242 -242
- package/scripts/provider-toolcall-test.mjs +79 -2
- package/scripts/recall-usecase-cases.json +1 -1
- package/scripts/smoke-runtime-negative.ps1 +106 -106
- package/scripts/tool-efficiency-diag.mjs +1 -1
- package/src/mixdog-session-runtime.mjs +12 -0
- package/src/rules/lead/02-channels.md +3 -3
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +33 -1
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +9 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +49 -0
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +14 -0
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +7 -0
- package/src/runtime/agent/orchestrator/session/loop.mjs +8 -0
- package/src/runtime/agent/orchestrator/session/manager/pending-messages.mjs +32 -18
- package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +142 -3
- package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +108 -30
- package/src/runtime/agent/orchestrator/session/store.mjs +5 -0
- package/src/runtime/agent/orchestrator/stall-policy.mjs +22 -12
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +3 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +15 -15
- package/src/runtime/agent/orchestrator/tools/builtin/snapshot-store.mjs +10 -2
- package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +8 -2
- package/src/runtime/channels/lib/runtime-paths.mjs +8 -19
- package/src/runtime/memory/index.mjs +37 -0
- package/src/runtime/memory/lib/embedding-warmup.mjs +3 -0
- package/src/runtime/memory/lib/ko-morph.mjs +1 -0
- package/src/runtime/shared/atomic-file.mjs +110 -0
- package/src/runtime/shared/transcript-writer.mjs +46 -4
- package/src/session-runtime/provider-models.mjs +47 -8
- package/src/tui/app/transcript-window.mjs +115 -6
- package/src/tui/app/use-transcript-window.mjs +58 -7
- package/src/tui/components/StatusLine.jsx +1 -1
- package/src/tui/dist/index.mjs +373 -81
- package/src/tui/engine/tui-steering-persist.mjs +66 -35
- package/src/tui/engine.mjs +6 -4
- package/src/tui/index.jsx +97 -6
- package/src/ui/statusline-segments.mjs +54 -36
- package/src/ui/statusline.mjs +141 -95
|
@@ -38,17 +38,6 @@ function parsePositivePid(value) {
|
|
|
38
38
|
function getTerminalLeadPid() {
|
|
39
39
|
return parsePositivePid(process.env.MIXDOG_SUPERVISOR_PID) ?? process.pid;
|
|
40
40
|
}
|
|
41
|
-
// Pinned-owner default: ON. The first session to claim ownership auto-pins
|
|
42
|
-
// itself and keeps the seat until its PID dies. Opt out per session with
|
|
43
|
-
// MIXDOG_PIN_OWNER=0 (or false/no/off) when you want the legacy 10 s stale-
|
|
44
|
-
// window takeover behavior — e.g. throwaway shells that should never become
|
|
45
|
-
// the sticky main.
|
|
46
|
-
function isOwnerPinEnabled() {
|
|
47
|
-
const v = process.env.MIXDOG_PIN_OWNER;
|
|
48
|
-
if (v === undefined || v === null || v === "") return true;
|
|
49
|
-
const low = String(v).toLowerCase();
|
|
50
|
-
return !(low === "0" || low === "false" || low === "no" || low === "off");
|
|
51
|
-
}
|
|
52
41
|
function getServerPid() {
|
|
53
42
|
return parsePositivePid(process.env.MIXDOG_SERVER_PID) ?? (process.env.MIXDOG_WORKER_MODE === "1" ? null : process.pid);
|
|
54
43
|
}
|
|
@@ -104,8 +93,10 @@ function touchUiHeartbeat(instanceId) {
|
|
|
104
93
|
if (!curRaw) return undefined;
|
|
105
94
|
if (instanceId && curRaw.instanceId !== instanceId) return undefined;
|
|
106
95
|
return { ...curRaw, ui_heartbeat_at: Date.now(), updatedAt: Date.now() };
|
|
107
|
-
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate' });
|
|
108
|
-
} catch { /* best-effort
|
|
96
|
+
}, { compact: true, fsync: false, fsyncDir: false, renameFallback: 'truncate', timeoutMs: 0 });
|
|
97
|
+
} catch { /* best-effort try-once (timeoutMs:0): on lock contention we skip
|
|
98
|
+
this tick without ever blocking the render loop on Atomics.wait;
|
|
99
|
+
the next 30s tick catches up. */ }
|
|
109
100
|
}
|
|
110
101
|
function buildRuntimeIdentity() {
|
|
111
102
|
const terminalLeadPid = getTerminalLeadPid();
|
|
@@ -294,12 +285,10 @@ function refreshActiveInstance(instanceId, meta, options) {
|
|
|
294
285
|
if (outBase !== newBase) next.priorTranscriptPath = outgoing;
|
|
295
286
|
}
|
|
296
287
|
}
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
//
|
|
300
|
-
|
|
301
|
-
if (isOwnerPinEnabled()) next.pinned = true;
|
|
302
|
-
else delete next.pinned;
|
|
288
|
+
// Ownership is strict last-wins (no pin): the newest claim always takes
|
|
289
|
+
// the seat. Drop any legacy `pinned` flag left by older versions — it was
|
|
290
|
+
// written but never consumed by any takeover check.
|
|
291
|
+
delete next.pinned;
|
|
303
292
|
// I1: pg_* spreads FIRST so newFields above win on conflict.
|
|
304
293
|
// prev.pg_port='A', meta.httpPort-adjacent pg_port='B' → result.pg_port='B'.
|
|
305
294
|
// memory_port: preserve when the advertising memory worker is still alive
|
|
@@ -10,6 +10,7 @@ process.on('warning', () => {})
|
|
|
10
10
|
|
|
11
11
|
import http from 'node:http'
|
|
12
12
|
import os from 'node:os'
|
|
13
|
+
import { performance } from 'node:perf_hooks'
|
|
13
14
|
import fs from 'node:fs'
|
|
14
15
|
import path from 'node:path'
|
|
15
16
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
@@ -102,6 +103,17 @@ if (!DATA_DIR) {
|
|
|
102
103
|
process.exit(1)
|
|
103
104
|
}
|
|
104
105
|
__mixdogMemoryLog(`[memory-service] DATA_DIR=${DATA_DIR}\n`)
|
|
106
|
+
const MEMORY_PROFILE_ENABLED = /^(1|true|yes|on)$/i.test(String(process.env.MIXDOG_MEMORY_PROFILE || process.env.MIXDOG_BOOT_PROFILE || ''))
|
|
107
|
+
const MEMORY_PROFILE_START = performance.now()
|
|
108
|
+
function memoryProfile(event, fields = {}) {
|
|
109
|
+
if (!MEMORY_PROFILE_ENABLED) return
|
|
110
|
+
const parts = [`[memory-profile] +${(performance.now() - MEMORY_PROFILE_START).toFixed(1)}ms`, event]
|
|
111
|
+
for (const [key, value] of Object.entries(fields || {})) {
|
|
112
|
+
if (value === undefined || value === null || value === '') continue
|
|
113
|
+
parts.push(`${key}=${String(value).replace(/\s+/g, '_')}`)
|
|
114
|
+
}
|
|
115
|
+
__mixdogMemoryLog(`${parts.join(' ')}\n`)
|
|
116
|
+
}
|
|
105
117
|
|
|
106
118
|
import {
|
|
107
119
|
parsePositivePid,
|
|
@@ -283,6 +295,7 @@ function _ingestIdentityFields(m) {
|
|
|
283
295
|
_ingestIdentityFieldsCache.set(m, fields)
|
|
284
296
|
return fields
|
|
285
297
|
}
|
|
298
|
+
|
|
286
299
|
// Ingest now WAITS for its embedding flush (bounded) so freshly ingested rows
|
|
287
300
|
// are dense-searchable the moment ingest_session returns. On a cold ONNX
|
|
288
301
|
// model the flush can stall on warmup; the race below caps the wait and lets
|
|
@@ -339,6 +352,8 @@ function assertSecondaryPgAttachable() {
|
|
|
339
352
|
}
|
|
340
353
|
|
|
341
354
|
async function _initStore() {
|
|
355
|
+
const initStoreStartedAt = performance.now()
|
|
356
|
+
memoryProfile('init-store:start')
|
|
342
357
|
mainConfig = readMainConfig()
|
|
343
358
|
const embeddingConfig = mainConfig?.embedding
|
|
344
359
|
if (embeddingConfig?.provider || embeddingConfig?.ollamaModel || embeddingConfig?.dtype) {
|
|
@@ -380,17 +395,24 @@ async function _initStore() {
|
|
|
380
395
|
if (dimsResolved) {
|
|
381
396
|
primeEmbeddingDims(dimsResolved)
|
|
382
397
|
assertSecondaryPgAttachable()
|
|
398
|
+
const openStartedAt = performance.now()
|
|
383
399
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
400
|
+
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
384
401
|
_embeddingWarmup.schedule(EMBEDDING_META_PATH, metaKey)
|
|
385
402
|
} else {
|
|
386
403
|
if (!embeddingWarmupCanStart()) {
|
|
387
404
|
throw new Error('memory-service: embedding dims unavailable while warmup is disabled')
|
|
388
405
|
}
|
|
389
406
|
// Cold path: meta missed AND model not registered. Sequential.
|
|
407
|
+
const warmupStartedAt = performance.now()
|
|
408
|
+
memoryProfile('embedding:cold-warmup:start')
|
|
390
409
|
await warmupEmbeddingProvider()
|
|
410
|
+
memoryProfile('embedding:cold-warmup:done', { ms: (performance.now() - warmupStartedAt).toFixed(1) })
|
|
391
411
|
dimsResolved = Number(getEmbeddingDims())
|
|
392
412
|
assertSecondaryPgAttachable()
|
|
413
|
+
const openStartedAt = performance.now()
|
|
393
414
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
415
|
+
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
394
416
|
try {
|
|
395
417
|
writeJsonAtomicSync(EMBEDDING_META_PATH, { ...metaKey, dims: dimsResolved }, { lock: true })
|
|
396
418
|
} catch (e) {
|
|
@@ -429,13 +451,18 @@ async function _initStore() {
|
|
|
429
451
|
if (cyclesCapable || memoryLlmWorkerEnabled()) {
|
|
430
452
|
try {
|
|
431
453
|
const agentCfg = loadAgentConfig()
|
|
454
|
+
const providersStartedAt = performance.now()
|
|
432
455
|
await initProviders(agentCfg.providers || {})
|
|
456
|
+
memoryProfile('providers:init:done', { ms: (performance.now() - providersStartedAt).toFixed(1) })
|
|
433
457
|
} catch (e) {
|
|
434
458
|
process.stderr.write(`[memory-service] initProviders failed (non-fatal): ${e instanceof Error ? e.message : String(e)}\n`)
|
|
435
459
|
}
|
|
436
460
|
}
|
|
437
461
|
_bootTimestamp = Date.now()
|
|
462
|
+
const offsetsStartedAt = performance.now()
|
|
438
463
|
await loadTranscriptOffsets()
|
|
464
|
+
memoryProfile('transcript-offsets:loaded', { ms: (performance.now() - offsetsStartedAt).toFixed(1) })
|
|
465
|
+
memoryProfile('init-store:done', { ms: (performance.now() - initStoreStartedAt).toFixed(1) })
|
|
439
466
|
}
|
|
440
467
|
|
|
441
468
|
async function getCycleLastRun() {
|
|
@@ -679,11 +706,16 @@ function _stopCycles() {
|
|
|
679
706
|
|
|
680
707
|
async function _initRuntime() {
|
|
681
708
|
if (_initialized) return
|
|
709
|
+
const runtimeStartedAt = performance.now()
|
|
710
|
+
memoryProfile('runtime-init:start')
|
|
682
711
|
await _initStore()
|
|
712
|
+
memoryProfile('runtime-init:init-store-ready', { ms: (performance.now() - runtimeStartedAt).toFixed(1) })
|
|
683
713
|
// Restore the core_entries.id == 1..N invariant once per boot: SERIAL only
|
|
684
714
|
// increments, so deleted rows leave permanent gaps. Fast no-op when already
|
|
685
715
|
// contiguous (or empty). Runs only here — never in cycle2/addCore/deleteCore.
|
|
716
|
+
const compactStartedAt = performance.now()
|
|
686
717
|
await compactCoreIds(DATA_DIR)
|
|
718
|
+
memoryProfile('core-ids:compact:done', { ms: (performance.now() - compactStartedAt).toFixed(1) })
|
|
687
719
|
// Memory module is always-on: the transcript watcher/ingest runs
|
|
688
720
|
// unconditionally except in secondary mode (secondary attaches to a primary's
|
|
689
721
|
// PG and must not double-ingest). The recap toggle only gates whether the
|
|
@@ -697,7 +729,9 @@ async function _initRuntime() {
|
|
|
697
729
|
__mixdogMemoryLog('[memory-service] secondary mode; skipping transcript watcher\n')
|
|
698
730
|
}
|
|
699
731
|
if (!memorySecondaryMode() && !envFlagEnabled('MIXDOG_MEMORY_DISABLE_CYCLES')) {
|
|
732
|
+
const cyclesStartedAt = performance.now()
|
|
700
733
|
_startCycles()
|
|
734
|
+
memoryProfile('cycles:start:done', { ms: (performance.now() - cyclesStartedAt).toFixed(1) })
|
|
701
735
|
} else {
|
|
702
736
|
__mixdogMemoryLog('[memory-service] background cycle tick loop not started (secondary/env-disabled)\n')
|
|
703
737
|
}
|
|
@@ -705,6 +739,7 @@ async function _initRuntime() {
|
|
|
705
739
|
// Boot complete — continue straight into the deferred embedding warmup.
|
|
706
740
|
// Fire-and-forget on the embedding worker thread; never awaited so it does
|
|
707
741
|
// not delay init() returning or the memory-ready signal.
|
|
742
|
+
memoryProfile('embedding:warmup:fire')
|
|
708
743
|
_embeddingWarmup.fireDeferred()
|
|
709
744
|
// Boot-edge Korean morph warmup. Same fire-and-forget style: lazy async init
|
|
710
745
|
// downloads+caches the Kiwi model once under DATA_DIR/kiwi-model/<version>,
|
|
@@ -712,8 +747,10 @@ async function _initRuntime() {
|
|
|
712
747
|
// buildFtsQuery keeps the websearch_to_tsquery fallback (never throws here).
|
|
713
748
|
// Skipped in secondary mode (query path lives on the primary).
|
|
714
749
|
if (!memorySecondaryMode()) {
|
|
750
|
+
memoryProfile('ko-morph:warmup:fire')
|
|
715
751
|
initKoMorph(DATA_DIR, __mixdogMemoryLog).catch(() => {})
|
|
716
752
|
}
|
|
753
|
+
memoryProfile('runtime-init:done', { ms: (performance.now() - runtimeStartedAt).toFixed(1) })
|
|
717
754
|
}
|
|
718
755
|
|
|
719
756
|
function _beginRuntimeInit() {
|
|
@@ -28,8 +28,11 @@ export function createEmbeddingWarmup({
|
|
|
28
28
|
return
|
|
29
29
|
}
|
|
30
30
|
_pending = () => {
|
|
31
|
+
const startedAt = Date.now()
|
|
32
|
+
log('[memory-service] embedding warmup start\n')
|
|
31
33
|
warmup()
|
|
32
34
|
.then(() => {
|
|
35
|
+
log(`[memory-service] embedding warmup ready in ${Date.now() - startedAt}ms\n`)
|
|
33
36
|
const measured = Number(getDims())
|
|
34
37
|
try {
|
|
35
38
|
persistMeta(metaPath, { ...metaKey, dims: measured })
|
|
@@ -130,6 +130,7 @@ export async function init(dataDir, log = () => {}) {
|
|
|
130
130
|
_log = typeof log === 'function' ? log : (() => {})
|
|
131
131
|
_state = 'loading'
|
|
132
132
|
const t0 = Date.now()
|
|
133
|
+
_log(`[memory-service] kiwi morph init start (model ${KIWI_MODEL_VERSION})\n`)
|
|
133
134
|
_initPromise = (async () => {
|
|
134
135
|
// Resolve the WASM path from the installed package without hard-importing
|
|
135
136
|
// (keeps the whole feature optional if kiwi-nlp isn't installed).
|
|
@@ -63,6 +63,16 @@ export function withFileLockSync(lockPath, fn, opts = {}) {
|
|
|
63
63
|
} catch (err) {
|
|
64
64
|
lastErr = err;
|
|
65
65
|
if (!LOCK_WAIT_CODES.has(err?.code)) throw err;
|
|
66
|
+
// try-once semantics: timeoutMs:0 makes a single openSync attempt and
|
|
67
|
+
// never sleeps — on contention it throws ELOCKCONTENDED immediately so
|
|
68
|
+
// the main loop is never blocked by Atomics.wait. No stale-reclaim,
|
|
69
|
+
// since reclaim would require waiting on the guard.
|
|
70
|
+
if (timeoutMs <= 0) {
|
|
71
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
72
|
+
contErr.code = 'ELOCKCONTENDED';
|
|
73
|
+
contErr.cause = err;
|
|
74
|
+
throw contErr;
|
|
75
|
+
}
|
|
66
76
|
try {
|
|
67
77
|
const st = statSync(lockPath);
|
|
68
78
|
if (Date.now() - st.mtimeMs > staleMs) {
|
|
@@ -246,6 +256,86 @@ function _releaseReclaimGuard(reclaim) {
|
|
|
246
256
|
}
|
|
247
257
|
}
|
|
248
258
|
|
|
259
|
+
// ── Async lock variant (non-blocking wait) ──────────────────────────
|
|
260
|
+
// Identical lock protocol to withFileLockSync (same lockPath, same pid
|
|
261
|
+
// stamp, same stale/ownership/reclaim rules) so a sync holder and an
|
|
262
|
+
// async holder are mutually exclusive against one another — they contend
|
|
263
|
+
// for the SAME openSync('wx') on the SAME path. The ONLY difference is
|
|
264
|
+
// the wait: instead of Atomics.wait blocking the event loop, we yield via
|
|
265
|
+
// setTimeout backoff. Intended for UI-process consistency-required writes
|
|
266
|
+
// that must not stall the render loop.
|
|
267
|
+
function _asyncSleep(ms) {
|
|
268
|
+
return new Promise((resolve) => { setTimeout(resolve, Math.max(1, Number(ms) || 1)); });
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export async function withFileLock(lockPath, fn, opts = {}) {
|
|
272
|
+
const timeoutMs = Number.isFinite(opts.timeoutMs) ? opts.timeoutMs : DEFAULT_LOCK_TIMEOUT_MS;
|
|
273
|
+
const staleMs = Number.isFinite(opts.staleMs) ? opts.staleMs : 30000;
|
|
274
|
+
const deadline = Date.now() + timeoutMs;
|
|
275
|
+
mkdirSync(dirname(lockPath), { recursive: true });
|
|
276
|
+
let attempt = 0;
|
|
277
|
+
let lastErr = null;
|
|
278
|
+
while (true) {
|
|
279
|
+
let fd;
|
|
280
|
+
try {
|
|
281
|
+
fd = openSync(lockPath, 'wx');
|
|
282
|
+
} catch (err) {
|
|
283
|
+
lastErr = err;
|
|
284
|
+
if (!LOCK_WAIT_CODES.has(err?.code)) throw err;
|
|
285
|
+
if (timeoutMs <= 0) {
|
|
286
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
287
|
+
contErr.code = 'ELOCKCONTENDED';
|
|
288
|
+
contErr.cause = err;
|
|
289
|
+
throw contErr;
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
const st = statSync(lockPath);
|
|
293
|
+
if (Date.now() - st.mtimeMs > staleMs) {
|
|
294
|
+
const deadPid = _readLockOwnerPid(lockPath);
|
|
295
|
+
if (deadPid !== null && _pidIsDead(deadPid)) {
|
|
296
|
+
const reclaim = _tryAcquireReclaimGuard(lockPath, staleMs);
|
|
297
|
+
if (reclaim !== null) {
|
|
298
|
+
let reclaimed = false;
|
|
299
|
+
try {
|
|
300
|
+
const currentSt = statSync(lockPath);
|
|
301
|
+
if (Date.now() - currentSt.mtimeMs > staleMs) {
|
|
302
|
+
const currentPid = _readLockOwnerPid(lockPath);
|
|
303
|
+
if (currentPid === deadPid && _pidIsDead(currentPid)) {
|
|
304
|
+
try { unlinkSync(lockPath); reclaimed = true; } catch {}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} finally {
|
|
308
|
+
_releaseReclaimGuard(reclaim);
|
|
309
|
+
}
|
|
310
|
+
if (reclaimed) continue;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
} catch {}
|
|
315
|
+
if (Date.now() >= deadline) break;
|
|
316
|
+
const base = DEFAULT_BACKOFFS_MS[Math.min(attempt, DEFAULT_BACKOFFS_MS.length - 1)];
|
|
317
|
+
const jitter = Math.floor(Math.random() * Math.min(75, Math.max(1, base)));
|
|
318
|
+
await _asyncSleep(Math.min(Math.max(1, deadline - Date.now()), base + jitter));
|
|
319
|
+
attempt += 1;
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
try { writeFileSync(fd, `${process.pid} ${Date.now()}\n`, 'utf8'); } catch {}
|
|
323
|
+
try {
|
|
324
|
+
if (opts.secret === true) _enforceOwnerOnlyAclWin32(lockPath);
|
|
325
|
+
return await fn();
|
|
326
|
+
} finally {
|
|
327
|
+
try { closeSync(fd); } catch {}
|
|
328
|
+
try {
|
|
329
|
+
if (_lockOwnedByPid(lockPath, process.pid)) unlinkSync(lockPath);
|
|
330
|
+
} catch {}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath}`);
|
|
334
|
+
timeoutErr.code = 'ELOCKTIMEOUT';
|
|
335
|
+
timeoutErr.cause = lastErr;
|
|
336
|
+
throw timeoutErr;
|
|
337
|
+
}
|
|
338
|
+
|
|
249
339
|
// ── Windows owner-only ACL enforcement (fail-closed) ─────────────────
|
|
250
340
|
// POSIX 0o600/0o700 mode bits are advisory-at-best on Windows: NTFS
|
|
251
341
|
// ignores them and the file inherits the parent ACL, which in shared or
|
|
@@ -459,3 +549,23 @@ export function updateJsonAtomicSync(filePath, mutator, opts = {}) {
|
|
|
459
549
|
return next;
|
|
460
550
|
}, opts);
|
|
461
551
|
}
|
|
552
|
+
|
|
553
|
+
// Async read-modify-write. Same lock path (`${filePath}.lock`) and protocol
|
|
554
|
+
// as updateJsonAtomicSync, so it is mutually exclusive with the sync variant.
|
|
555
|
+
// The critical section itself is synchronous (readFileSync + writeJsonAtomicSync);
|
|
556
|
+
// only the lock WAIT is async, so the event loop is never blocked on contention.
|
|
557
|
+
export async function updateJsonAtomic(filePath, mutator, opts = {}) {
|
|
558
|
+
const { lock: _lock, ...writeOpts } = opts;
|
|
559
|
+
return withFileLock(`${filePath}.lock`, () => {
|
|
560
|
+
let cur = null;
|
|
561
|
+
try {
|
|
562
|
+
cur = JSON.parse(readFileSync(filePath, 'utf8'));
|
|
563
|
+
} catch {
|
|
564
|
+
cur = null;
|
|
565
|
+
}
|
|
566
|
+
const next = mutator(cur);
|
|
567
|
+
if (next === undefined) return cur;
|
|
568
|
+
writeJsonAtomicSync(filePath, next, { ...writeOpts, lock: false });
|
|
569
|
+
return next;
|
|
570
|
+
}, opts);
|
|
571
|
+
}
|
|
@@ -15,6 +15,13 @@
|
|
|
15
15
|
* cannot spam the terminal.
|
|
16
16
|
*/
|
|
17
17
|
import { existsSync, mkdirSync, renameSync, statSync, writeFileSync } from 'node:fs';
|
|
18
|
+
|
|
19
|
+
// After this many appended bytes since the last rotation stat, re-check
|
|
20
|
+
// disk size via statSync. Bounds the sync-fs cost per append to roughly
|
|
21
|
+
// once every ~64KB of transcript growth instead of on every single line,
|
|
22
|
+
// while still catching rotation promptly (a JSONL line is typically well
|
|
23
|
+
// under 1KB, so this is on the order of dozens of appends between stats).
|
|
24
|
+
const ROTATE_CHECK_BYTE_STRIDE = 64 * 1024;
|
|
18
25
|
import { dirname, join, resolve } from 'node:path';
|
|
19
26
|
import { appendBuffered, drainPathSync, hasInFlightWrite } from './buffered-appender.mjs';
|
|
20
27
|
|
|
@@ -62,10 +69,28 @@ export function createTranscriptWriter({ mixdogHome, sessionId, cwd, pid } = {})
|
|
|
62
69
|
}
|
|
63
70
|
}
|
|
64
71
|
|
|
65
|
-
|
|
72
|
+
// Local byte counter tracking growth since the last real statSync, so
|
|
73
|
+
// rotateIfNeeded's fs work is O(1) per append instead of an existsSync+
|
|
74
|
+
// statSync pair on every single JSONL line (this fires on every assistant
|
|
75
|
+
// chunk / tool call, i.e. per token-stream event, not per turn). We only
|
|
76
|
+
// hit disk again once the locally-tracked size estimate could plausibly
|
|
77
|
+
// have crossed the rotate threshold, or a full check hasn't happened yet.
|
|
78
|
+
let sizeChecked = false;
|
|
79
|
+
let lastKnownSize = 0;
|
|
80
|
+
let bytesSinceCheck = 0;
|
|
81
|
+
|
|
82
|
+
function statAndMaybeRotate() {
|
|
66
83
|
try {
|
|
67
|
-
if (!existsSync(transcriptPath))
|
|
84
|
+
if (!existsSync(transcriptPath)) {
|
|
85
|
+
sizeChecked = true;
|
|
86
|
+
lastKnownSize = 0;
|
|
87
|
+
bytesSinceCheck = 0;
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
68
90
|
const { size } = statSync(transcriptPath);
|
|
91
|
+
sizeChecked = true;
|
|
92
|
+
lastKnownSize = size;
|
|
93
|
+
bytesSinceCheck = 0;
|
|
69
94
|
if (size < TRANSCRIPT_ROTATE_BYTES) return;
|
|
70
95
|
// An async appendFile may be in flight for this path; renaming out
|
|
71
96
|
// from under it races the write on Windows. Skip this round and
|
|
@@ -76,17 +101,34 @@ export function createTranscriptWriter({ mixdogHome, sessionId, cwd, pid } = {})
|
|
|
76
101
|
// file post-rename doesn't inherit stale in-memory chunks.
|
|
77
102
|
drainPathSync(transcriptPath);
|
|
78
103
|
const rotatedPath = `${transcriptPath}.1`;
|
|
79
|
-
try {
|
|
104
|
+
try {
|
|
105
|
+
renameSync(transcriptPath, rotatedPath);
|
|
106
|
+
lastKnownSize = 0;
|
|
107
|
+
} catch (err) { logOnce(err); }
|
|
80
108
|
} catch (err) {
|
|
81
109
|
logOnce(err);
|
|
82
110
|
}
|
|
83
111
|
}
|
|
84
112
|
|
|
113
|
+
function rotateIfNeeded() {
|
|
114
|
+
// Real stat only when: no check has happened yet, the locally-tracked
|
|
115
|
+
// growth alone could have crossed the rotate threshold, or the last
|
|
116
|
+
// known size plus tracked growth is already at/over the threshold
|
|
117
|
+
// (catches a file that was already large when this process attached).
|
|
118
|
+
if (!sizeChecked
|
|
119
|
+
|| bytesSinceCheck >= ROTATE_CHECK_BYTE_STRIDE
|
|
120
|
+
|| lastKnownSize + bytesSinceCheck >= TRANSCRIPT_ROTATE_BYTES) {
|
|
121
|
+
statAndMaybeRotate();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
85
125
|
function appendLine(entry) {
|
|
86
126
|
ensureProjectDir();
|
|
87
127
|
rotateIfNeeded();
|
|
88
128
|
try {
|
|
89
|
-
|
|
129
|
+
const line = `${JSON.stringify(entry)}\n`;
|
|
130
|
+
appendBuffered(transcriptPath, line);
|
|
131
|
+
bytesSinceCheck += Buffer.byteLength(line);
|
|
90
132
|
} catch (err) {
|
|
91
133
|
logOnce(err);
|
|
92
134
|
}
|
|
@@ -11,6 +11,10 @@ import { fastCapableFor, fastPreferenceFor } from './model-capabilities.mjs';
|
|
|
11
11
|
import { modelSettingsFor } from './config-helpers.mjs';
|
|
12
12
|
import { isSelectableLlmModel } from './model-recency.mjs';
|
|
13
13
|
|
|
14
|
+
const PROVIDER_MODELS_PROFILE_ENABLED = /^(1|true|yes|on)$/i.test(String(
|
|
15
|
+
process.env.MIXDOG_PROVIDER_MODELS_PROFILE || process.env.MIXDOG_BOOT_PROFILE || '',
|
|
16
|
+
));
|
|
17
|
+
|
|
14
18
|
export function createProviderModels({
|
|
15
19
|
caches,
|
|
16
20
|
modelMetaByRoute,
|
|
@@ -32,6 +36,10 @@ export function createProviderModels({
|
|
|
32
36
|
const config = () => getConfig();
|
|
33
37
|
const route = () => getRoute();
|
|
34
38
|
const reg = () => getReg();
|
|
39
|
+
function profile(event, fields = {}) {
|
|
40
|
+
if (!PROVIDER_MODELS_PROFILE_ENABLED) return;
|
|
41
|
+
bootProfile(`provider-models:${event}`, fields);
|
|
42
|
+
}
|
|
35
43
|
|
|
36
44
|
function modelMetaKey(providerId, modelId) {
|
|
37
45
|
return `${clean(providerId)}\n${clean(modelId)}`;
|
|
@@ -152,12 +160,17 @@ export function createProviderModels({
|
|
|
152
160
|
return results;
|
|
153
161
|
}
|
|
154
162
|
|
|
155
|
-
async function loadProviderModelsFresh({ forceRefresh = false } = {}) {
|
|
156
|
-
|
|
163
|
+
async function loadProviderModelsFresh({ forceRefresh = false, loadSecrets = true } = {}) {
|
|
164
|
+
const startedAt = performance.now();
|
|
165
|
+
profile('load:start', { forceRefresh, loadSecrets });
|
|
166
|
+
if (loadSecrets) ensureFullConfig();
|
|
167
|
+
const providersStartedAt = performance.now();
|
|
157
168
|
await ensureProvidersReady(config().providers || {});
|
|
169
|
+
profile('providers-ready', { ms: (performance.now() - providersStartedAt).toFixed(1) });
|
|
158
170
|
const allProviders = [...reg().getAllProviders()];
|
|
159
171
|
const providerResults = await Promise.all(allProviders.map(async ([name, provider]) => {
|
|
160
172
|
if (typeof provider?.listModels !== 'function') return [];
|
|
173
|
+
const providerStartedAt = performance.now();
|
|
161
174
|
try {
|
|
162
175
|
let models = null;
|
|
163
176
|
if (forceRefresh && typeof provider._refreshModelCache === 'function') {
|
|
@@ -173,8 +186,19 @@ export function createProviderModels({
|
|
|
173
186
|
if (!isSelectableLlmModel(m)) continue;
|
|
174
187
|
rows.push(providerModelCacheRow(name, m));
|
|
175
188
|
}
|
|
189
|
+
profile('provider:done', {
|
|
190
|
+
provider: name,
|
|
191
|
+
ms: (performance.now() - providerStartedAt).toFixed(1),
|
|
192
|
+
models: models.length,
|
|
193
|
+
rows: rows.length,
|
|
194
|
+
});
|
|
176
195
|
return rows;
|
|
177
|
-
} catch {
|
|
196
|
+
} catch (error) {
|
|
197
|
+
profile('provider:failed', {
|
|
198
|
+
provider: name,
|
|
199
|
+
ms: (performance.now() - providerStartedAt).toFixed(1),
|
|
200
|
+
error: error?.message || String(error),
|
|
201
|
+
});
|
|
178
202
|
// Ignore per-provider catalog failures so one bad credential or
|
|
179
203
|
// transient /models error does not hide other authenticated models.
|
|
180
204
|
return [];
|
|
@@ -189,9 +213,21 @@ export function createProviderModels({
|
|
|
189
213
|
results.push(row);
|
|
190
214
|
modelMetaByRoute.set(modelMetaKey(row.provider, row.id), row);
|
|
191
215
|
}
|
|
216
|
+
profile('load:done', { ms: (performance.now() - startedAt).toFixed(1), providers: allProviders.length, rows: results.length });
|
|
192
217
|
return results;
|
|
193
218
|
}
|
|
194
219
|
|
|
220
|
+
function shouldAdoptProviderModelCache(models, { loadSecrets = true } = {}) {
|
|
221
|
+
// Background warmup deliberately avoids ensureFullConfig() so it cannot
|
|
222
|
+
// block the TUI on keychain/config reload. That no-secrets path is only a
|
|
223
|
+
// best-effort provider-internal prefetch. Its result may be a partial
|
|
224
|
+
// catalog (for example local/env providers listed while keychain-backed
|
|
225
|
+
// providers failed), so never let it become the authoritative picker cache.
|
|
226
|
+
// Foreground/forced loads still adopt empty lists because they loaded the
|
|
227
|
+
// authoritative config.
|
|
228
|
+
return loadSecrets;
|
|
229
|
+
}
|
|
230
|
+
|
|
195
231
|
async function collectSearchProviderModels({ force = false } = {}) {
|
|
196
232
|
if (!force && Array.isArray(caches.searchProviderModelsCache.models)) {
|
|
197
233
|
return providerModelsFromCacheRows(quickHelpers.searchRowsWithDefault(caches.searchProviderModelsCache.models));
|
|
@@ -223,15 +259,15 @@ export function createProviderModels({
|
|
|
223
259
|
}
|
|
224
260
|
if (force) {
|
|
225
261
|
const seq = ++caches.providerModelsLoadSeq;
|
|
226
|
-
const models = await loadProviderModelsFresh({ forceRefresh: true });
|
|
262
|
+
const models = await loadProviderModelsFresh({ forceRefresh: true, loadSecrets: true });
|
|
227
263
|
if (seq === caches.providerModelsLoadSeq) caches.providerModelsCache = { models, at: Date.now() };
|
|
228
264
|
return providerModelsFromCacheRows(models);
|
|
229
265
|
}
|
|
230
266
|
if (!caches.providerModelsPromise) {
|
|
231
267
|
const seq = ++caches.providerModelsLoadSeq;
|
|
232
|
-
caches.providerModelsPromise = loadProviderModelsFresh()
|
|
268
|
+
caches.providerModelsPromise = loadProviderModelsFresh({ loadSecrets: true })
|
|
233
269
|
.then((models) => {
|
|
234
|
-
if (seq === caches.providerModelsLoadSeq) caches.providerModelsCache = { models, at: Date.now() };
|
|
270
|
+
if (seq === caches.providerModelsLoadSeq && shouldAdoptProviderModelCache(models, { loadSecrets: true })) caches.providerModelsCache = { models, at: Date.now() };
|
|
235
271
|
return models;
|
|
236
272
|
})
|
|
237
273
|
.finally(() => {
|
|
@@ -243,10 +279,13 @@ export function createProviderModels({
|
|
|
243
279
|
|
|
244
280
|
function warmProviderModelCache() {
|
|
245
281
|
if (Array.isArray(caches.providerModelsCache.models) || caches.providerModelsPromise) return caches.providerModelsPromise;
|
|
282
|
+
profile('warm:start');
|
|
246
283
|
const seq = ++caches.providerModelsLoadSeq;
|
|
247
|
-
caches.providerModelsPromise = loadProviderModelsFresh()
|
|
284
|
+
caches.providerModelsPromise = loadProviderModelsFresh({ loadSecrets: false })
|
|
248
285
|
.then((models) => {
|
|
249
|
-
if (seq === caches.providerModelsLoadSeq
|
|
286
|
+
if (seq === caches.providerModelsLoadSeq && shouldAdoptProviderModelCache(models, { loadSecrets: false })) {
|
|
287
|
+
caches.providerModelsCache = { models, at: Date.now() };
|
|
288
|
+
}
|
|
250
289
|
bootProfile('provider-models:warm-ready', { count: models.length });
|
|
251
290
|
return models;
|
|
252
291
|
})
|