mixdog 0.9.21 → 0.9.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -4
- package/package.json +1 -1
- package/scripts/channel-daemon-smoke.mjs +165 -0
- package/scripts/channel-daemon-stub.mjs +69 -0
- package/scripts/statusline-quota-hysteresis-test.mjs +37 -0
- package/scripts/tool-smoke.mjs +30 -17
- package/src/runtime/agent/orchestrator/mcp/child-tree.mjs +187 -0
- package/src/runtime/agent/orchestrator/mcp/client.mjs +40 -3
- package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +1 -1
- package/src/runtime/agent/orchestrator/providers/openai-compat-wire.mjs +5 -3
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +2 -2
- package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +2 -2
- package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs +1 -1
- package/src/runtime/agent/orchestrator/session/loop/completion-guards.mjs +3 -2
- package/src/runtime/agent/orchestrator/session/loop/deferred-call-through.mjs +6 -3
- package/src/runtime/agent/orchestrator/session/loop/tool-helpers.mjs +1 -1
- package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +37 -4
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +7 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +30 -14
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +10 -10
- package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +8 -6
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +14 -1
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-paths.mjs +1 -1
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +24 -11
- package/src/runtime/agent/orchestrator/tools/builtin/shell-runtime.mjs +6 -5
- package/src/runtime/agent/orchestrator/tools/patch/orchestrator.mjs +23 -13
- package/src/runtime/agent/orchestrator/tools/shell-command.mjs +56 -1
- package/src/runtime/channels/lib/owned-runtime.mjs +126 -167
- package/src/runtime/channels/lib/owner-heartbeat.mjs +11 -60
- package/src/runtime/channels/lib/parent-bridge.mjs +16 -1
- package/src/runtime/channels/lib/runtime-paths.mjs +32 -113
- package/src/runtime/channels/lib/seat-lock.mjs +196 -0
- package/src/runtime/channels/lib/session-discovery.mjs +2 -1
- package/src/runtime/channels/lib/tool-dispatch.mjs +24 -8
- package/src/runtime/channels/lib/worker-main.mjs +42 -11
- package/src/runtime/memory/index.mjs +54 -7
- package/src/runtime/memory/lib/embedding-warmup.mjs +7 -0
- package/src/runtime/memory/lib/pg/process.mjs +85 -40
- package/src/runtime/memory/lib/pg/supervisor.mjs +82 -17
- package/src/runtime/shared/atomic-file.mjs +44 -10
- package/src/runtime/shared/tool-primitives.mjs +31 -1
- package/src/runtime/shared/tool-surface.mjs +23 -13
- package/src/session-runtime/config-lifecycle.mjs +48 -7
- package/src/session-runtime/lifecycle-api.mjs +9 -0
- package/src/session-runtime/mcp-glue.mjs +63 -1
- package/src/session-runtime/resource-api.mjs +62 -8
- package/src/session-runtime/runtime-core.mjs +32 -2
- package/src/session-runtime/session-text.mjs +41 -0
- package/src/session-runtime/session-turn-api.mjs +24 -0
- package/src/session-runtime/settings-api.mjs +8 -1
- package/src/session-runtime/tool-catalog.mjs +306 -38
- package/src/session-runtime/tool-defs.mjs +7 -7
- package/src/session-runtime/workflow.mjs +2 -1
- package/src/standalone/channel-daemon-client.mjs +224 -0
- package/src/standalone/channel-daemon-transport.mjs +351 -0
- package/src/standalone/channel-daemon.mjs +139 -0
- package/src/standalone/channel-worker.mjs +213 -4
- package/src/standalone/hook-bus.mjs +71 -3
- package/src/tui/App.jsx +105 -17
- package/src/tui/app/clipboard.mjs +39 -19
- package/src/tui/app/doctor.mjs +57 -0
- package/src/tui/app/extension-pickers.mjs +53 -9
- package/src/tui/app/maintenance-pickers.mjs +0 -5
- package/src/tui/app/slash-dispatch.mjs +4 -4
- package/src/tui/app/text-layout.mjs +11 -0
- package/src/tui/app/use-mouse-input.mjs +235 -51
- package/src/tui/app/use-prompt-handlers.mjs +49 -30
- package/src/tui/app/use-transcript-scroll.mjs +124 -27
- package/src/tui/app/use-transcript-window.mjs +55 -1
- package/src/tui/components/Message.jsx +1 -1
- package/src/tui/components/PromptInput.jsx +3 -1
- package/src/tui/components/QueuedCommands.jsx +21 -10
- package/src/tui/components/ToolExecution.jsx +2 -2
- package/src/tui/dist/index.mjs +653 -310
- package/src/tui/engine/session-api.mjs +17 -7
- package/src/tui/engine/session-flow.mjs +6 -0
- package/src/tui/engine.mjs +8 -0
- package/src/tui/index.jsx +62 -18
- package/src/tui/paste-attachments.mjs +26 -0
- package/src/ui/statusline.mjs +45 -5
- package/src/ui/tool-card.mjs +8 -1
- package/src/vendor/statusline/bin/statusline-route.mjs +23 -7
- package/src/workflows/bench/WORKFLOW.md +46 -0
- package/src/workflows/default/WORKFLOW.md +6 -0
- package/src/workflows/solo/WORKFLOW.md +5 -0
- package/vendor/ink/build/ink.js +23 -1
- package/vendor/ink/build/output.js +154 -71
- package/vendor/ink/build/render-node-to-output.js +44 -2
- package/vendor/ink/build/render.js +4 -0
- package/vendor/ink/build/renderer.js +4 -1
|
@@ -67,6 +67,7 @@ import { createParentBridge } from "./parent-bridge.mjs";
|
|
|
67
67
|
import { createInboundRouting } from "./inbound-routing.mjs";
|
|
68
68
|
import { createToolDispatch } from "./tool-dispatch.mjs";
|
|
69
69
|
import { createOwnerHeartbeat } from "./owner-heartbeat.mjs";
|
|
70
|
+
import { createSeatLock } from "./seat-lock.mjs";
|
|
70
71
|
import { createTranscriptBinding } from "./transcript-binding.mjs";
|
|
71
72
|
import { isNetworkError, retryOnNetwork } from "./network-retry.mjs";
|
|
72
73
|
import { runWorkerIpc } from "./worker-ipc.mjs";
|
|
@@ -138,6 +139,9 @@ let config = loadConfig();
|
|
|
138
139
|
let backend = createBackend(config);
|
|
139
140
|
const INSTANCE_ID = makeInstanceId();
|
|
140
141
|
const TERMINAL_LEAD_PID = getTerminalLeadPid();
|
|
142
|
+
// OS-enforced bridge-seat singleton. Ownership TRUTH is holding this listener;
|
|
143
|
+
// a holder crash auto-releases it and takeover is an explicit release message.
|
|
144
|
+
const seatLock = createSeatLock({ runtimeRoot: RUNTIME_ROOT, instanceId: INSTANCE_ID });
|
|
141
145
|
runWorkerBootstrap({
|
|
142
146
|
instanceId: INSTANCE_ID,
|
|
143
147
|
isWorkerMode: _isWorkerMode,
|
|
@@ -324,13 +328,8 @@ const {
|
|
|
324
328
|
logOwnership,
|
|
325
329
|
currentOwnerState,
|
|
326
330
|
getBridgeOwnershipSnapshot,
|
|
327
|
-
claimBridgeOwnership,
|
|
328
|
-
startOwnerHeartbeat,
|
|
329
|
-
stopOwnerHeartbeat,
|
|
330
331
|
} = createOwnerHeartbeat({
|
|
331
|
-
|
|
332
|
-
readActiveInstance,
|
|
333
|
-
refreshActiveInstance,
|
|
332
|
+
isSeatHeld: () => seatLock.isSeatHeld(),
|
|
334
333
|
});
|
|
335
334
|
// ── Owned-runtime lifecycle ─────────────────────────────────────────────────
|
|
336
335
|
// Extracted -> lib/owned-runtime.mjs. Owns its own start/stop/refresh in-flight
|
|
@@ -364,9 +363,10 @@ const {
|
|
|
364
363
|
statusState,
|
|
365
364
|
logOwnership,
|
|
366
365
|
currentOwnerState,
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
366
|
+
acquireSeat: (opts) => seatLock.acquireSeat(opts),
|
|
367
|
+
closeSeatServer: () => seatLock.closeSeatServer(),
|
|
368
|
+
isSeatHeld: () => seatLock.isSeatHeld(),
|
|
369
|
+
onTakeover: (cb) => seatLock.onTakeover(cb),
|
|
370
370
|
bindPersistedTranscriptIfAny,
|
|
371
371
|
cancelPendingTranscriptRearm,
|
|
372
372
|
schedulePendingTranscriptRearm,
|
|
@@ -634,10 +634,38 @@ const {
|
|
|
634
634
|
setChannelBridgeActive: (v) => { channelBridgeActive = v; },
|
|
635
635
|
writeBridgeState,
|
|
636
636
|
stopServerTyping,
|
|
637
|
-
claimBridgeOwnership,
|
|
638
637
|
notifyRemoteAcquired,
|
|
639
638
|
refreshBridgeOwnership,
|
|
640
639
|
bindPersistedTranscriptIfAny,
|
|
640
|
+
// Lead-pushed repoint: bind the exact transcript the lead just created
|
|
641
|
+
// (auto-acquire / newSession / resume / clear) instead of waiting for the
|
|
642
|
+
// next inbound parent-chain steal. Idempotent + best-effort: same binding
|
|
643
|
+
// path as the inbound steal (rebindTranscriptContext -> applyTranscriptBinding).
|
|
644
|
+
rebindCurrentTranscript: async (transcriptPath) => {
|
|
645
|
+
const cleanPath = typeof transcriptPath === "string" ? transcriptPath.trim() : "";
|
|
646
|
+
if (!cleanPath) return;
|
|
647
|
+
const channelId = statusState.read().channelId || config.channelId;
|
|
648
|
+
if (!channelId) return;
|
|
649
|
+
// Fail-closed: a malformed / not-yet-on-disk path must NEVER fall through
|
|
650
|
+
// to rebindTranscriptContext's discovery loop (which would mutate the
|
|
651
|
+
// binding onto a different session). Only an existing regular file binds;
|
|
652
|
+
// otherwise log + return so the current binding is left untouched.
|
|
653
|
+
let exists = false;
|
|
654
|
+
try { exists = fs.statSync(cleanPath).isFile(); } catch { exists = false; }
|
|
655
|
+
if (!exists) {
|
|
656
|
+
process.stderr.write(`mixdog: rebind_current_transcript: ignoring non-existent path ${cleanPath}\n`);
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
// Idempotent: already bound to this exact transcript => no-op. In
|
|
660
|
+
// particular do NOT re-run recoverUnsyncedTail, which is only meaningful
|
|
661
|
+
// when the binding actually changes.
|
|
662
|
+
if (forwarder.hasBinding() && sameResolvedPath(forwarder.transcriptPath, cleanPath)) return;
|
|
663
|
+
// Binding changed: same path as the inbound steal's applyTranscriptBinding.
|
|
664
|
+
applyTranscriptBinding(channelId, cleanPath, {
|
|
665
|
+
persistStatus: true,
|
|
666
|
+
recoverUnsyncedTail: true,
|
|
667
|
+
});
|
|
668
|
+
},
|
|
641
669
|
stopOwnedRuntime,
|
|
642
670
|
reloadRuntimeConfig,
|
|
643
671
|
},
|
|
@@ -745,7 +773,10 @@ async function stop() {
|
|
|
745
773
|
}
|
|
746
774
|
}
|
|
747
775
|
// ── IPC worker mode ──────────────────────────────────────────────
|
|
748
|
-
|
|
776
|
+
// Skipped under the machine-global daemon (MIXDOG_CHANNEL_DAEMON=1): the
|
|
777
|
+
// daemon entry (src/standalone/channel-daemon.mjs) drives start()/stop() and
|
|
778
|
+
// its own HTTP+SSE transport instead of the parent node-IPC call/notify loop.
|
|
779
|
+
if (_isWorkerMode && process.send && process.env.MIXDOG_CHANNEL_DAEMON !== '1') {
|
|
749
780
|
runWorkerIpc({
|
|
750
781
|
start,
|
|
751
782
|
stop,
|
|
@@ -134,6 +134,10 @@ const RUNTIME_ROOT = process.env.MIXDOG_RUNTIME_ROOT
|
|
|
134
134
|
|
|
135
135
|
let _periodicAdvertiseInstalled = false
|
|
136
136
|
let _periodicAdvertiseTimer = null
|
|
137
|
+
// Single module-level advertise retry chain. A newer advertiseMemoryPort call
|
|
138
|
+
// cancels the older chain so a delayed retry never replays a stale boundPort.
|
|
139
|
+
let _advertiseRetryTimer = null
|
|
140
|
+
let _advertiseGeneration = 0
|
|
137
141
|
// Track the most recently advertised port so the periodic tick re-reads it
|
|
138
142
|
// every interval. Without this the setInterval closure binds the FIRST port
|
|
139
143
|
// (the upstream we proxied to) and keeps re-advertising the dead upstream
|
|
@@ -248,7 +252,14 @@ function deregisterClient(clientPid) {
|
|
|
248
252
|
|
|
249
253
|
function advertiseMemoryPort(boundPort, attempt = 0) {
|
|
250
254
|
if (!Number.isFinite(boundPort) || boundPort <= 0) return
|
|
251
|
-
|
|
255
|
+
// A fresh top-level advertise (attempt 0) supersedes any pending retry chain:
|
|
256
|
+
// last write wins, so a delayed retry never clobbers a newer boundPort.
|
|
257
|
+
if (attempt === 0) {
|
|
258
|
+
_currentAdvertisedPort = boundPort
|
|
259
|
+
_advertiseGeneration++
|
|
260
|
+
if (_advertiseRetryTimer) { try { clearTimeout(_advertiseRetryTimer) } catch {} ; _advertiseRetryTimer = null }
|
|
261
|
+
}
|
|
262
|
+
const generation = _advertiseGeneration
|
|
252
263
|
if (!_periodicAdvertiseInstalled) {
|
|
253
264
|
_periodicAdvertiseInstalled = true
|
|
254
265
|
_periodicAdvertiseTimer = setInterval(() => {
|
|
@@ -284,12 +295,24 @@ function advertiseMemoryPort(boundPort, attempt = 0) {
|
|
|
284
295
|
updatedAt: Date.now(),
|
|
285
296
|
}
|
|
286
297
|
return next
|
|
287
|
-
}, { compact: true, fsyncDir: true, renameFallback: 'truncate' })
|
|
298
|
+
}, { compact: true, fsyncDir: true, renameFallback: 'truncate', timeoutMs: 1000 })
|
|
299
|
+
if (generation === _advertiseGeneration) _advertiseRetryTimer = null
|
|
288
300
|
} catch (e) {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
301
|
+
// Boot path must not serially block on the default 8s lock wait: use a short
|
|
302
|
+
// lock timeout and treat lock contention/timeout as transient so pg_port /
|
|
303
|
+
// memory_port still eventually publish via unref'd, backed-off bg retries.
|
|
304
|
+
const transient =
|
|
305
|
+
e?.code === 'EPERM' || e?.code === 'EBUSY' || e?.code === 'EACCES' ||
|
|
306
|
+
e?.code === 'ELOCKTIMEOUT' || e?.code === 'ELOCKCONTENDED'
|
|
307
|
+
if (transient && attempt < 5 && generation === _advertiseGeneration) {
|
|
308
|
+
const delay = Math.min(2000, 50 * 2 ** attempt)
|
|
309
|
+
// Fire-time generation re-check: even if clearTimeout was missed, a
|
|
310
|
+
// retry from a superseded chain must never republish an old boundPort.
|
|
311
|
+
_advertiseRetryTimer = setTimeout(() => {
|
|
312
|
+
if (generation !== _advertiseGeneration) return
|
|
313
|
+
advertiseMemoryPort(boundPort, attempt + 1)
|
|
314
|
+
}, delay)
|
|
315
|
+
_advertiseRetryTimer.unref?.()
|
|
293
316
|
return
|
|
294
317
|
}
|
|
295
318
|
__mixdogMemoryLog(`[memory-service] active-instance memory_port advertise failed: ${e?.message || e}\n`)
|
|
@@ -454,10 +477,23 @@ async function _initStore() {
|
|
|
454
477
|
if (dimsResolved) {
|
|
455
478
|
primeEmbeddingDims(dimsResolved)
|
|
456
479
|
assertSecondaryPgAttachable()
|
|
480
|
+
// Start embedding + kiwi warmup NOW so they run on their worker threads
|
|
481
|
+
// during the PG ensure I/O-wait window below: openDatabase spawns/awaits
|
|
482
|
+
// Postgres (I/O-bound, not CPU), so overlapping warmup there is safe and
|
|
483
|
+
// lands a cold boot's first recall dense instead of lexical-fallback.
|
|
484
|
+
// Both are fire-and-forget; the boot-complete fireDeferred/initKoMorph edge
|
|
485
|
+
// stays an idempotent no-op fallback. schedule() respects canStart
|
|
486
|
+
// (secondary/env-disabled skip → fireDeferred no-op); kiwi skips secondary.
|
|
487
|
+
_embeddingWarmup.schedule(EMBEDDING_META_PATH, metaKey)
|
|
488
|
+
memoryProfile('embedding:warmup:fire')
|
|
489
|
+
_embeddingWarmup.fireDeferred()
|
|
490
|
+
if (!memorySecondaryMode()) {
|
|
491
|
+
memoryProfile('ko-morph:warmup:fire')
|
|
492
|
+
initKoMorph(DATA_DIR, __mixdogMemoryLog).catch(() => {})
|
|
493
|
+
}
|
|
457
494
|
const openStartedAt = performance.now()
|
|
458
495
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
459
496
|
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
460
|
-
_embeddingWarmup.schedule(EMBEDDING_META_PATH, metaKey)
|
|
461
497
|
} else {
|
|
462
498
|
if (!embeddingWarmupCanStart()) {
|
|
463
499
|
throw new Error('memory-service: embedding dims unavailable while warmup is disabled')
|
|
@@ -469,6 +505,12 @@ async function _initStore() {
|
|
|
469
505
|
memoryProfile('embedding:cold-warmup:done', { ms: (performance.now() - warmupStartedAt).toFixed(1) })
|
|
470
506
|
dimsResolved = Number(getEmbeddingDims())
|
|
471
507
|
assertSecondaryPgAttachable()
|
|
508
|
+
// Embedding is already warm (awaited above); still fire kiwi during the PG
|
|
509
|
+
// ensure I/O-wait so the first recall's FTS path is morph-aware too.
|
|
510
|
+
if (!memorySecondaryMode()) {
|
|
511
|
+
memoryProfile('ko-morph:warmup:fire')
|
|
512
|
+
initKoMorph(DATA_DIR, __mixdogMemoryLog).catch(() => {})
|
|
513
|
+
}
|
|
472
514
|
const openStartedAt = performance.now()
|
|
473
515
|
db = await openDatabase(DATA_DIR, dimsResolved)
|
|
474
516
|
memoryProfile('open-db:done', { ms: (performance.now() - openStartedAt).toFixed(1), dims: dimsResolved })
|
|
@@ -798,6 +840,11 @@ export async function stop() {
|
|
|
798
840
|
try { clearInterval(_periodicAdvertiseTimer) } catch {}
|
|
799
841
|
_periodicAdvertiseTimer = null
|
|
800
842
|
}
|
|
843
|
+
if (_advertiseRetryTimer) {
|
|
844
|
+
try { clearTimeout(_advertiseRetryTimer) } catch {}
|
|
845
|
+
_advertiseRetryTimer = null
|
|
846
|
+
}
|
|
847
|
+
_advertiseGeneration++
|
|
801
848
|
_periodicAdvertiseInstalled = false
|
|
802
849
|
_currentAdvertisedPort = null
|
|
803
850
|
_embeddingWarmup.reset()
|
|
@@ -16,11 +16,16 @@ export function createEmbeddingWarmup({
|
|
|
16
16
|
log = () => {},
|
|
17
17
|
}) {
|
|
18
18
|
let _pending = null
|
|
19
|
+
// Fired latch: an init() retry after the early (PG-wait-edge) fire must not
|
|
20
|
+
// re-schedule and run the warmup twice (double ONNX session create +
|
|
21
|
+
// double embedding-meta persist). reset() clears it for true re-inits.
|
|
22
|
+
let _fired = false
|
|
19
23
|
|
|
20
24
|
// Queue the warmup; fireDeferred() runs it once boot completes. Re-queuing
|
|
21
25
|
// overwrites any prior pending closure (matches the original single-slot
|
|
22
26
|
// module var). MIXDOG_EMBED_WARMUP=0 (checked via canStart) disables it.
|
|
23
27
|
function schedule(metaPath, metaKey) {
|
|
28
|
+
if (_fired) return
|
|
24
29
|
if (!canStart()) {
|
|
25
30
|
// Silent skip here previously made "warmup never ran" boots
|
|
26
31
|
// indistinguishable from failed warmups. One line, boot-edge only.
|
|
@@ -50,11 +55,13 @@ export function createEmbeddingWarmup({
|
|
|
50
55
|
const fire = _pending
|
|
51
56
|
if (!fire) return
|
|
52
57
|
_pending = null
|
|
58
|
+
_fired = true
|
|
53
59
|
fire()
|
|
54
60
|
}
|
|
55
61
|
|
|
56
62
|
function reset() {
|
|
57
63
|
_pending = null
|
|
64
|
+
_fired = false
|
|
58
65
|
}
|
|
59
66
|
|
|
60
67
|
return { schedule, fireDeferred, reset }
|
|
@@ -14,7 +14,7 @@ function __mixdogMemoryLog(...args) {
|
|
|
14
14
|
|
|
15
15
|
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'
|
|
16
16
|
import { join } from 'path'
|
|
17
|
-
import { spawnSync } from 'child_process'
|
|
17
|
+
import { spawn, spawnSync } from 'child_process'
|
|
18
18
|
import { createConnection } from 'net'
|
|
19
19
|
import { createServer } from 'net'
|
|
20
20
|
|
|
@@ -248,55 +248,100 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
|
|
|
248
248
|
|
|
249
249
|
__mixdogMemoryLog(`[pg-process] pg_ctl start -D ${pgdataDir} -p ${port}\n`)
|
|
250
250
|
|
|
251
|
-
const
|
|
252
|
-
|
|
251
|
+
const pgIsReady = pgBin(runtimeDir, 'pg_isready')
|
|
252
|
+
const startArgs = [
|
|
253
|
+
'start',
|
|
253
254
|
'-D', pgdataDir,
|
|
254
255
|
'-l', logFile,
|
|
255
256
|
'-o', `-p ${port} -h 127.0.0.1`,
|
|
256
|
-
]
|
|
257
|
+
]
|
|
257
258
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
259
|
+
// Poll-sleep is intentionally NOT unref'd: while startPg is awaiting readiness
|
|
260
|
+
// it must keep the process alive even if the event loop would otherwise drain.
|
|
261
|
+
const sleep = ms => new Promise(res => setTimeout(res, ms))
|
|
262
|
+
|
|
263
|
+
// Read pid + port from postmaster.pid (line 1 = pid, line 4 = port). Returns
|
|
264
|
+
// null unless the file exists, pid > 0, and its port matches ours.
|
|
265
|
+
function readPostmaster() {
|
|
266
|
+
try {
|
|
267
|
+
const pidFile = join(pgdataDir, 'postmaster.pid')
|
|
268
|
+
if (!existsSync(pidFile)) return null
|
|
269
|
+
const lines = readFileSync(pidFile, 'utf8').split('\n')
|
|
270
|
+
const pid = parseInt(lines[0], 10)
|
|
271
|
+
const pmPort = parseInt(lines[3], 10)
|
|
272
|
+
if (pid > 0 && pmPort === port) return { pid }
|
|
273
|
+
return null
|
|
274
|
+
} catch { return null }
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// pg_isready can succeed a beat before postmaster.pid is fully written; give
|
|
278
|
+
// it a brief window to appear (and confirm the port matches) before trusting.
|
|
279
|
+
async function confirmPid() {
|
|
280
|
+
for (let i = 0; i < 5; i++) {
|
|
281
|
+
const pm = readPostmaster()
|
|
282
|
+
if (pm) return pm.pid
|
|
283
|
+
await sleep(100)
|
|
284
|
+
}
|
|
285
|
+
return null
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// Spawn pg_ctl asynchronously (no -w) and poll readiness ourselves. On
|
|
289
|
+
// AV-throttled boxes pg_ctl can lag ~30s behind actual postmaster readiness,
|
|
290
|
+
// so we return the instant pg_isready succeeds rather than waiting on pg_ctl.
|
|
291
|
+
// Only the long-lived child handle is unref'd; poll timers stay ref'd.
|
|
292
|
+
async function startAndWaitReady() {
|
|
293
|
+
const child = spawn(pgctl, startArgs, { env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true })
|
|
294
|
+
child.unref?.()
|
|
295
|
+
let stdout = '', stderr = '', closed = false, exitCode = null
|
|
296
|
+
child.stdout?.on('data', d => { stdout += d.toString() })
|
|
297
|
+
child.stderr?.on('data', d => { stderr += d.toString() })
|
|
298
|
+
child.stdout?.unref?.(); child.stderr?.unref?.()
|
|
299
|
+
child.on('error', err => { closed = true; exitCode = -1; stderr += (err?.message || String(err)) })
|
|
300
|
+
// Use 'close' (stdio flushed), not 'exit', so captured stderr is complete
|
|
301
|
+
// and the "another server might be running" match below never truncates.
|
|
302
|
+
child.on('close', code => { closed = true; exitCode = code })
|
|
303
|
+
|
|
304
|
+
const deadline = Date.now() + 30_000
|
|
305
|
+
// eslint-disable-next-line no-constant-condition
|
|
306
|
+
while (true) {
|
|
307
|
+
const probe = spawnSync(pgIsReady, ['-h', '127.0.0.1', '-p', String(port)], {
|
|
308
|
+
env, stdio: 'pipe', timeout: 3_000, windowsHide: true,
|
|
309
|
+
})
|
|
310
|
+
if (probe.status === 0) {
|
|
311
|
+
const pid = await confirmPid()
|
|
312
|
+
// pid confirmed with matching port → ready. Otherwise keep polling until
|
|
313
|
+
// the cap (postmaster.pid not yet written or port mismatch).
|
|
314
|
+
if (pid != null) return { ready: true, pid }
|
|
285
315
|
}
|
|
316
|
+
// pg_ctl exited nonzero before PG became reachable — real startup failure.
|
|
317
|
+
if (closed && exitCode !== 0) return { ready: false, exited: true, stdout, stderr }
|
|
318
|
+
if (Date.now() >= deadline) return { ready: false, timeout: true, stdout, stderr }
|
|
319
|
+
await sleep(250)
|
|
286
320
|
}
|
|
287
|
-
throw new Error(`[pg-process] pg_ctl start failed: ${errText}`)
|
|
288
321
|
}
|
|
289
322
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
323
|
+
const r = await startAndWaitReady()
|
|
324
|
+
if (r.ready) return { pid: r.pid, port }
|
|
325
|
+
|
|
326
|
+
const errText = r.stderr || r.stdout || ''
|
|
327
|
+
// "another server might be running" — try status + immediate stop before failing.
|
|
328
|
+
if (r.exited && errText.includes('another server might be running')) {
|
|
329
|
+
__mixdogMemoryLog(`[pg-process] pg_ctl start: "another server might be running" — probing status\n`)
|
|
330
|
+
const statusR = spawnSync(pgctl, ['status', '-D', pgdataDir], { env, stdio: 'pipe', timeout: 3_000, windowsHide: true })
|
|
331
|
+
__mixdogMemoryLog(`[pg-process] pg_ctl status: ${statusR.stdout?.toString() || statusR.stderr?.toString() || 'no output'}\n`)
|
|
332
|
+
const stopR = spawnSync(pgctl, ['stop', '-m', 'immediate', '-D', pgdataDir], { env, stdio: 'pipe', timeout: 3_000, windowsHide: true })
|
|
333
|
+
if (stopR.status === 0) {
|
|
334
|
+
// Retry start once after clearing the stale instance.
|
|
335
|
+
const r2 = await startAndWaitReady()
|
|
336
|
+
if (r2.ready) return { pid: r2.pid, port }
|
|
337
|
+
__mixdogMemoryLog(`[pg-process] retry start after stop also failed: ${r2.stderr || ''}\n`)
|
|
338
|
+
} else {
|
|
339
|
+
__mixdogMemoryLog(`[pg-process] immediate stop failed: ${stopR.stderr?.toString() || ''} — treating as degraded\n`)
|
|
296
340
|
}
|
|
297
|
-
}
|
|
341
|
+
}
|
|
298
342
|
|
|
299
|
-
|
|
343
|
+
const detail = errText || (r.timeout ? '(readiness probe timed out after 30s; no pg_ctl output)' : '(no captured output)')
|
|
344
|
+
throw new Error(`[pg-process] pg_ctl start failed: ${detail}`)
|
|
300
345
|
}
|
|
301
346
|
|
|
302
347
|
// ---------------------------------------------------------------------------
|
|
@@ -182,24 +182,85 @@ const _RUNTIME_ROOT = process.env.MIXDOG_RUNTIME_ROOT
|
|
|
182
182
|
: join(tmpdir(), 'mixdog');
|
|
183
183
|
const _ACTIVE_FILE = join(_RUNTIME_ROOT, 'active-instance.json');
|
|
184
184
|
|
|
185
|
-
|
|
185
|
+
// Lock contention codes thrown by withFileLockSync when the active-instance
|
|
186
|
+
// lock cannot be acquired within timeoutMs (or try-once contention).
|
|
187
|
+
const _LOCK_CONTENTION_CODES = new Set(['ELOCKTIMEOUT', 'ELOCKCONTENDED']);
|
|
188
|
+
|
|
189
|
+
function _applyActiveInstancePatch(fields, timeoutMs) {
|
|
190
|
+
updateJsonAtomicSync(_ACTIVE_FILE, (curRaw) => {
|
|
191
|
+
// Drop stale fields (pid/startedAt) written by older server versions.
|
|
192
|
+
const { pid: _legacyPid, startedAt: _legacyStartedAt, ...cur } = curRaw ?? {};
|
|
193
|
+
// Omit null-valued fields (clean removal when pg is stopped).
|
|
194
|
+
const merged = { ...cur, updatedAt: Date.now() };
|
|
195
|
+
for (const [k, v] of Object.entries(fields)) {
|
|
196
|
+
if (v == null) delete merged[k];
|
|
197
|
+
else merged[k] = v;
|
|
198
|
+
}
|
|
199
|
+
return merged;
|
|
200
|
+
}, { compact: true, fsyncDir: true, renameFallback: 'truncate', timeoutMs });
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Single module-level background-retry slot: a delayed retry must NEVER replay
|
|
204
|
+
// stale `fields` over newer state (e.g. a stale clear deleting a fresh pg_port,
|
|
205
|
+
// or a stale start resurrecting pg_port after shutdown). Every patch bumps the
|
|
206
|
+
// generation and cancels any pending chain, so only the newest fields replay.
|
|
207
|
+
let _patchGeneration = 0;
|
|
208
|
+
let _pendingPatchRetry = null; // { timer, generation }
|
|
209
|
+
|
|
210
|
+
function _cancelPendingPatchRetry() {
|
|
211
|
+
if (_pendingPatchRetry) {
|
|
212
|
+
try { clearTimeout(_pendingPatchRetry.timer); } catch {}
|
|
213
|
+
_pendingPatchRetry = null;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function _tryPatchInBackground(fields, timeoutMs, generation, attempt) {
|
|
218
|
+
// A newer patch has superseded this chain — abandon (last write wins).
|
|
219
|
+
if (generation !== _patchGeneration) return;
|
|
186
220
|
try {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
const { pid: _legacyPid, startedAt: _legacyStartedAt, ...cur } = curRaw ?? {};
|
|
190
|
-
// Omit null-valued fields (clean removal when pg is stopped).
|
|
191
|
-
const merged = { ...cur, updatedAt: Date.now() };
|
|
192
|
-
for (const [k, v] of Object.entries(fields)) {
|
|
193
|
-
if (v == null) delete merged[k];
|
|
194
|
-
else merged[k] = v;
|
|
195
|
-
}
|
|
196
|
-
return merged;
|
|
197
|
-
}, { compact: true, fsyncDir: true, renameFallback: 'truncate' });
|
|
221
|
+
_applyActiveInstancePatch(fields, timeoutMs);
|
|
222
|
+
if (_pendingPatchRetry?.generation === generation) _pendingPatchRetry = null;
|
|
198
223
|
} catch (e) {
|
|
224
|
+
if (_LOCK_CONTENTION_CODES.has(e?.code) && attempt < 5) {
|
|
225
|
+
const delay = Math.min(2000, 100 * 2 ** attempt);
|
|
226
|
+
const timer = setTimeout(
|
|
227
|
+
() => _tryPatchInBackground(fields, timeoutMs, generation, attempt + 1),
|
|
228
|
+
delay,
|
|
229
|
+
);
|
|
230
|
+
timer.unref?.();
|
|
231
|
+
_pendingPatchRetry = { timer, generation };
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
if (_pendingPatchRetry?.generation === generation) _pendingPatchRetry = null;
|
|
199
235
|
__mixdogMemoryLog(`[supervisor-pg] patchActiveInstance failed: ${e?.message}\n`);
|
|
200
236
|
}
|
|
201
237
|
}
|
|
202
238
|
|
|
239
|
+
// Boot-path advertise writes (pg_port et al.) must never serially block init on
|
|
240
|
+
// the 8s default lock wait. Callers pass { timeoutMs, background, syncRetries }:
|
|
241
|
+
// background:true — short lock timeout; on contention schedule unref'd,
|
|
242
|
+
// backed-off, bounded bg retries of the NEWEST fields only.
|
|
243
|
+
// background:false — synchronous best-effort with `syncRetries` extra sync
|
|
244
|
+
// attempts on lock timeout (shutdown clear path).
|
|
245
|
+
function patchActiveInstance(fields, opts = {}) {
|
|
246
|
+
const { timeoutMs = 8000, background = false, syncRetries = 0 } = opts;
|
|
247
|
+
// Supersede any in-flight background retry: last write wins per process.
|
|
248
|
+
_patchGeneration++;
|
|
249
|
+
_cancelPendingPatchRetry();
|
|
250
|
+
if (background) {
|
|
251
|
+
_tryPatchInBackground(fields, timeoutMs, _patchGeneration, 0);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
for (let i = 0; ; i++) {
|
|
255
|
+
try { _applyActiveInstancePatch(fields, timeoutMs); return; }
|
|
256
|
+
catch (e) {
|
|
257
|
+
if (_LOCK_CONTENTION_CODES.has(e?.code) && i < syncRetries) continue;
|
|
258
|
+
__mixdogMemoryLog(`[supervisor-pg] patchActiveInstance failed: ${e?.message}\n`);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
203
264
|
// ── postmaster.pid helpers ───────────────────────────────────────────────────
|
|
204
265
|
|
|
205
266
|
function readPostmasterPid(pgdata) {
|
|
@@ -276,7 +337,7 @@ async function _startFresh(dataDir, pgdata, port, runtimeDir) {
|
|
|
276
337
|
patchActiveInstance({
|
|
277
338
|
pg_port: actualPort, pg_started_at: Date.now(),
|
|
278
339
|
pg_pgdata: pgdata, pg_runtime_dir: runtimeDir,
|
|
279
|
-
});
|
|
340
|
+
}, { timeoutMs: 1000, background: true });
|
|
280
341
|
__mixdogMemoryLog(`[supervisor-pg] ${proc?.attached ? 'attached to' : 'started'} PG port=${actualPort} pgdata=${pgdata}\n`);
|
|
281
342
|
return { host: '127.0.0.1', port: actualPort, runtimeDir, pgdataDir };
|
|
282
343
|
}
|
|
@@ -314,7 +375,7 @@ async function tryReusePgInstance({ pgdata, runtimeDir, healthcheckPg, source =
|
|
|
314
375
|
pg_started_at: ai?.pg_started_at ?? Date.now(),
|
|
315
376
|
pg_pgdata: pgdata,
|
|
316
377
|
pg_runtime_dir: runtimeDir,
|
|
317
|
-
});
|
|
378
|
+
}, { timeoutMs: 1000, background: true });
|
|
318
379
|
return { host: '127.0.0.1', port: pm.port, runtimeDir, pgdataDir: pgdata };
|
|
319
380
|
}
|
|
320
381
|
} catch {}
|
|
@@ -429,7 +490,7 @@ async function _doEnsure(dataDir) {
|
|
|
429
490
|
try { unlinkSync(join(pgdata, 'postmaster.pid')); } catch {}
|
|
430
491
|
}
|
|
431
492
|
// Clear stale pg fields before restart
|
|
432
|
-
patchActiveInstance({ pg_port: null, pg_started_at: null, pg_pgdata: null });
|
|
493
|
+
patchActiveInstance({ pg_port: null, pg_started_at: null, pg_pgdata: null }, { timeoutMs: 1000, background: true });
|
|
433
494
|
}
|
|
434
495
|
|
|
435
496
|
// ── Allocate a fresh port and spawn ───────────────────────────────────
|
|
@@ -490,7 +551,9 @@ export async function stopPgForShutdown() {
|
|
|
490
551
|
} catch (e) {
|
|
491
552
|
__mixdogMemoryLog(`[supervisor-pg] stopPg error on shutdown (no _live): ${e?.message}\n`);
|
|
492
553
|
}
|
|
493
|
-
|
|
554
|
+
// Bounded sync clear (<=4s: 2s + one 2s retry). A residual stale pg_port is
|
|
555
|
+
// tolerated — readers healthcheck-verify the port before reuse.
|
|
556
|
+
patchActiveInstance({ pg_port: null, pg_started_at: null, pg_pgdata: null, pg_runtime_dir: null }, { timeoutMs: 2000, syncRetries: 1 });
|
|
494
557
|
return;
|
|
495
558
|
}
|
|
496
559
|
const snap = _live;
|
|
@@ -503,5 +566,7 @@ export async function stopPgForShutdown() {
|
|
|
503
566
|
} catch (e) {
|
|
504
567
|
__mixdogMemoryLog(`[supervisor-pg] stopPg error on shutdown: ${e?.message}\n`);
|
|
505
568
|
}
|
|
506
|
-
|
|
569
|
+
// Bounded sync clear (<=4s: 2s + one 2s retry). A residual stale pg_port is
|
|
570
|
+
// tolerated — readers healthcheck-verify the port before reuse.
|
|
571
|
+
patchActiveInstance({ pg_port: null, pg_started_at: null, pg_pgdata: null, pg_runtime_dir: null }, { timeoutMs: 2000, syncRetries: 1 });
|
|
507
572
|
}
|
|
@@ -38,6 +38,22 @@ function sleepSync(ms) {
|
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
// Best-effort holder snapshot for lock-failure diagnostics: pid/token/age
|
|
42
|
+
// read from the lock file AT THROW TIME, so ELOCKTIMEOUT/ELOCKCONTENDED
|
|
43
|
+
// messages identify the culprit without needing a live repro. Never throws.
|
|
44
|
+
function _describeLockHolder(lockPath) {
|
|
45
|
+
try {
|
|
46
|
+
const st = statSync(lockPath);
|
|
47
|
+
const owner = _readLockOwner(lockPath);
|
|
48
|
+
const ageMs = Math.max(0, Math.round(Date.now() - st.mtimeMs));
|
|
49
|
+
const live = owner.pid === null ? 'unknown' : (_ownerIsLive(owner) ? 'live' : 'dead');
|
|
50
|
+
const token = owner.token === null ? '?' : String(owner.token).slice(0, 8);
|
|
51
|
+
return `holder pid=${owner.pid ?? '?'} token=${token} age=${ageMs}ms ${live}`;
|
|
52
|
+
} catch {
|
|
53
|
+
return 'holder unknown (lock file unreadable/absent)';
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
41
57
|
export function renameWithRetrySync(src, dst, opts = {}) {
|
|
42
58
|
const backoffs = Array.isArray(opts.backoffs) && opts.backoffs.length > 0
|
|
43
59
|
? opts.backoffs
|
|
@@ -76,7 +92,7 @@ export function withFileLockSync(lockPath, fn, opts = {}) {
|
|
|
76
92
|
// the main loop is never blocked by Atomics.wait. No stale-reclaim,
|
|
77
93
|
// since reclaim would require waiting on the guard.
|
|
78
94
|
if (timeoutMs <= 0) {
|
|
79
|
-
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
95
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath} [${_describeLockHolder(lockPath)}]`);
|
|
80
96
|
contErr.code = 'ELOCKCONTENDED';
|
|
81
97
|
contErr.cause = err;
|
|
82
98
|
throw contErr;
|
|
@@ -118,7 +134,7 @@ export function withFileLockSync(lockPath, fn, opts = {}) {
|
|
|
118
134
|
} catch {}
|
|
119
135
|
}
|
|
120
136
|
}
|
|
121
|
-
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath}`);
|
|
137
|
+
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath} [${_describeLockHolder(lockPath)}]`);
|
|
122
138
|
timeoutErr.code = 'ELOCKTIMEOUT';
|
|
123
139
|
timeoutErr.cause = lastErr;
|
|
124
140
|
throw timeoutErr;
|
|
@@ -278,8 +294,24 @@ function _reclaimGuardMatchesToken(guardPath, token) {
|
|
|
278
294
|
}
|
|
279
295
|
}
|
|
280
296
|
|
|
281
|
-
function
|
|
282
|
-
|
|
297
|
+
function _readGuardContent(guardPath) {
|
|
298
|
+
try { return readFileSync(guardPath, 'utf8'); } catch { return null; }
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function _parseStampPid(raw) {
|
|
302
|
+
if (raw === null) return null;
|
|
303
|
+
const tok = String(raw).trim().split(/\s+/)[0];
|
|
304
|
+
const pid = Number.parseInt(tok, 10);
|
|
305
|
+
return Number.isFinite(pid) && pid > 0 ? pid : null;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
// Unlink the guard only if its FULL content (pid ts token) is still the
|
|
309
|
+
// exact content the stale/dead decision was made against. A regenerated
|
|
310
|
+
// guard — even one whose pid was recycled to the same value (same-pid
|
|
311
|
+
// ABA) — carries a fresh ts+token and therefore never matches, so a
|
|
312
|
+
// fresh guard is never removed by a stale decision.
|
|
313
|
+
function _unlinkReclaimGuardIfContentMatches(guardPath, content) {
|
|
314
|
+
if (_readGuardContent(guardPath) !== content) return false;
|
|
283
315
|
try { unlinkSync(guardPath); return true; } catch { return false; }
|
|
284
316
|
}
|
|
285
317
|
|
|
@@ -289,10 +321,12 @@ function _unlinkUnreadableStaleReclaimGuard(guardPath, staleMs) {
|
|
|
289
321
|
}
|
|
290
322
|
|
|
291
323
|
function _tryClearStaleReclaimGuard(guardPath, staleMs) {
|
|
292
|
-
const
|
|
324
|
+
const guardContent = _readGuardContent(guardPath);
|
|
325
|
+
const guardPid = _parseStampPid(guardContent);
|
|
293
326
|
if (guardPid !== null) {
|
|
294
|
-
// Re-read the guard immediately before unlinking so a
|
|
295
|
-
// guard
|
|
327
|
+
// Re-read the guard's full content immediately before unlinking so a
|
|
328
|
+
// regenerated guard (different owner OR same-pid ABA) is not removed
|
|
329
|
+
// by a stale decision.
|
|
296
330
|
// This is still not a serialized primitive: correctness comes from
|
|
297
331
|
// the lockPath owner-token reverify before unlinking lockPath, not
|
|
298
332
|
// from making guard cleanup itself authoritative.
|
|
@@ -304,7 +338,7 @@ function _tryClearStaleReclaimGuard(guardPath, staleMs) {
|
|
|
304
338
|
let guardStale = false;
|
|
305
339
|
try { guardStale = Date.now() - statSync(guardPath).mtimeMs > staleMs; } catch {}
|
|
306
340
|
if (_pidIsDead(guardPid) || guardStale) {
|
|
307
|
-
|
|
341
|
+
_unlinkReclaimGuardIfContentMatches(guardPath, guardContent);
|
|
308
342
|
}
|
|
309
343
|
return;
|
|
310
344
|
}
|
|
@@ -365,7 +399,7 @@ export async function withFileLock(lockPath, fn, opts = {}) {
|
|
|
365
399
|
lastErr = err;
|
|
366
400
|
if (!LOCK_WAIT_CODES.has(err?.code)) throw err;
|
|
367
401
|
if (timeoutMs <= 0) {
|
|
368
|
-
const contErr = new Error(`atomic lock contended (try-once): ${lockPath}`);
|
|
402
|
+
const contErr = new Error(`atomic lock contended (try-once): ${lockPath} [${_describeLockHolder(lockPath)}]`);
|
|
369
403
|
contErr.code = 'ELOCKCONTENDED';
|
|
370
404
|
contErr.cause = err;
|
|
371
405
|
throw contErr;
|
|
@@ -394,7 +428,7 @@ export async function withFileLock(lockPath, fn, opts = {}) {
|
|
|
394
428
|
} catch {}
|
|
395
429
|
}
|
|
396
430
|
}
|
|
397
|
-
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath}`);
|
|
431
|
+
const timeoutErr = new Error(`atomic lock timeout after ${timeoutMs}ms: ${lockPath} [${_describeLockHolder(lockPath)}]`);
|
|
398
432
|
timeoutErr.code = 'ELOCKTIMEOUT';
|
|
399
433
|
timeoutErr.cause = lastErr;
|
|
400
434
|
throw timeoutErr;
|