mixdog 0.9.51 → 0.9.53

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. package/package.json +5 -3
  2. package/scripts/abort-recovery-test.mjs +17 -1
  3. package/scripts/agent-model-liveness-test.mjs +79 -2
  4. package/scripts/anthropic-admission-retry-integration-test.mjs +119 -0
  5. package/scripts/anthropic-transport-policy-test.mjs +466 -0
  6. package/scripts/atomic-lock-tryonce-test.mjs +60 -1
  7. package/scripts/bench-run.mjs +2 -2
  8. package/scripts/build-tui.mjs +13 -1
  9. package/scripts/channel-daemon-smoke.mjs +630 -10
  10. package/scripts/code-graph-aggregate-cwd-test.mjs +41 -37
  11. package/scripts/code-graph-disk-hit-test.mjs +11 -0
  12. package/scripts/code-graph-root-federation-test.mjs +273 -0
  13. package/scripts/compact-pressure-test.mjs +159 -1
  14. package/scripts/compact-smoke.mjs +80 -0
  15. package/scripts/context-mcp-metering-test.mjs +1350 -19
  16. package/scripts/deferred-tool-loading-test.mjs +17 -0
  17. package/scripts/desktop-session-bridge-test.mjs +704 -0
  18. package/scripts/freevar-smoke.mjs +7 -4
  19. package/scripts/gemini-provider-test.mjs +1053 -0
  20. package/scripts/interrupted-turn-history-test.mjs +371 -0
  21. package/scripts/lifecycle-api-test.mjs +137 -0
  22. package/scripts/max-output-recovery-test.mjs +86 -0
  23. package/scripts/mcp-grace-deferred-test.mjs +89 -13
  24. package/scripts/memory-core-input-test.mjs +10 -0
  25. package/scripts/memory-pg-recovery-test.mjs +59 -0
  26. package/scripts/openai-oauth-ws-1006-retry-test.mjs +387 -6
  27. package/scripts/openai-ws-early-settle-test.mjs +40 -0
  28. package/scripts/parent-abort-link-test.mjs +24 -0
  29. package/scripts/process-lifecycle-test.mjs +447 -0
  30. package/scripts/provider-admission-scheduler-test.mjs +582 -0
  31. package/scripts/provider-contract-test.mjs +525 -0
  32. package/scripts/provider-toolcall-test.mjs +492 -11
  33. package/scripts/reactive-compact-persist-smoke.mjs +59 -0
  34. package/scripts/resource-admission-test.mjs +789 -0
  35. package/scripts/session-orphan-sweep-test.mjs +27 -1
  36. package/scripts/shell-jobs-windows-hide-test.mjs +1 -1
  37. package/scripts/steering-drain-buckets-test.mjs +18 -0
  38. package/scripts/toolcall-args-test.mjs +14 -6
  39. package/scripts/tui-transcript-perf-test.mjs +5 -7
  40. package/src/cli.mjs +15 -2
  41. package/src/lib/keychain-cjs.cjs +36 -23
  42. package/src/runtime/agent/orchestrator/agent-runtime/agent-dispatch.mjs +27 -13
  43. package/src/runtime/agent/orchestrator/agent-runtime/agent-progress-watchdog.mjs +4 -1
  44. package/src/runtime/agent/orchestrator/agent-runtime/cache-strategy.mjs +7 -8
  45. package/src/runtime/agent/orchestrator/agent-trace.mjs +33 -9
  46. package/src/runtime/agent/orchestrator/providers/admission-scheduler.mjs +331 -0
  47. package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +7 -2
  48. package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +114 -308
  49. package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +31 -21
  50. package/src/runtime/agent/orchestrator/providers/anthropic.mjs +136 -286
  51. package/src/runtime/agent/orchestrator/providers/gemini-cache.mjs +42 -5
  52. package/src/runtime/agent/orchestrator/providers/gemini-schema.mjs +554 -42
  53. package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs +67 -18
  54. package/src/runtime/agent/orchestrator/providers/gemini.mjs +84 -150
  55. package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +17 -119
  56. package/src/runtime/agent/orchestrator/providers/lib/anthropic-request-utils.mjs +224 -0
  57. package/src/runtime/agent/orchestrator/providers/lib/env-utils.mjs +6 -0
  58. package/src/runtime/agent/orchestrator/providers/lib/gemini-model-catalog.mjs +119 -0
  59. package/src/runtime/agent/orchestrator/providers/lib/grok-tool-schema.mjs +109 -0
  60. package/src/runtime/agent/orchestrator/providers/lib/openai-tool-args.mjs +70 -0
  61. package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +54 -14
  62. package/src/runtime/agent/orchestrator/providers/openai-compat-presets.mjs +1 -1
  63. package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +28 -74
  64. package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +10 -80
  65. package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +136 -20
  66. package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +94 -33
  67. package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +201 -123
  68. package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +37 -28
  69. package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs +10 -10
  70. package/src/runtime/agent/orchestrator/providers/openai-ws-events.mjs +30 -3
  71. package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs +36 -3
  72. package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs +50 -45
  73. package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +78 -12
  74. package/src/runtime/agent/orchestrator/providers/opencode-go.mjs +44 -5
  75. package/src/runtime/agent/orchestrator/providers/registry.mjs +49 -8
  76. package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs +229 -111
  77. package/src/runtime/agent/orchestrator/session/agent-loop.mjs +130 -63
  78. package/src/runtime/agent/orchestrator/session/compact/engine.mjs +99 -32
  79. package/src/runtime/agent/orchestrator/session/context-compaction-policy.mjs +170 -0
  80. package/src/runtime/agent/orchestrator/session/context-utils.mjs +37 -224
  81. package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs +11 -17
  82. package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +81 -42
  83. package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +10 -1
  84. package/src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs +21 -5
  85. package/src/runtime/agent/orchestrator/session/manager/message-sanitize.mjs +8 -28
  86. package/src/runtime/agent/orchestrator/session/manager/prefetch-bridge.mjs +3 -58
  87. package/src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs +30 -7
  88. package/src/runtime/agent/orchestrator/session/manager/session-close.mjs +2 -0
  89. package/src/runtime/agent/orchestrator/session/manager/session-crud.mjs +10 -1
  90. package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +42 -1
  91. package/src/runtime/agent/orchestrator/session/manager/turn-interruption.mjs +220 -0
  92. package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +57 -16
  93. package/src/runtime/agent/orchestrator/session/pre-send-compact.mjs +20 -4
  94. package/src/runtime/agent/orchestrator/session/save-session-worker.mjs +2 -2
  95. package/src/runtime/agent/orchestrator/session/send-with-recovery.mjs +12 -1
  96. package/src/runtime/agent/orchestrator/session/store/paths-heartbeat.mjs +52 -0
  97. package/src/runtime/agent/orchestrator/session/store/write-guards.mjs +62 -0
  98. package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +17 -0
  99. package/src/runtime/agent/orchestrator/session/store.mjs +353 -108
  100. package/src/runtime/agent/orchestrator/stall-policy.mjs +2 -12
  101. package/src/runtime/agent/orchestrator/tools/bash-session.mjs +42 -4
  102. package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +74 -37
  103. package/src/runtime/agent/orchestrator/tools/builtin/lib/list-helpers.mjs +46 -0
  104. package/src/runtime/agent/orchestrator/tools/builtin/lib/search-grep-chunks.mjs +173 -0
  105. package/src/runtime/agent/orchestrator/tools/builtin/lib/search-input-helpers.mjs +117 -0
  106. package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-job-insights.mjs +199 -0
  107. package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-spawn-helpers.mjs +107 -0
  108. package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +1 -40
  109. package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +19 -277
  110. package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +223 -2
  111. package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +317 -250
  112. package/src/runtime/agent/orchestrator/tools/code-graph/build.mjs +176 -21
  113. package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +14 -0
  114. package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +108 -2
  115. package/src/runtime/agent/orchestrator/tools/code-graph/trusted-roots.mjs +93 -0
  116. package/src/runtime/agent/orchestrator/tools/code-graph-prewarm-worker.mjs +12 -3
  117. package/src/runtime/agent/orchestrator/tools/lib/shell-spawn-retry.mjs +67 -0
  118. package/src/runtime/agent/orchestrator/tools/shell-command.mjs +125 -85
  119. package/src/runtime/channels/backends/discord-gateway.mjs +1 -1
  120. package/src/runtime/channels/backends/discord.mjs +6 -6
  121. package/src/runtime/channels/lib/config.mjs +15 -2
  122. package/src/runtime/channels/lib/memory-client.mjs +20 -3
  123. package/src/runtime/channels/lib/owned-runtime.mjs +19 -12
  124. package/src/runtime/channels/lib/status-snapshot.mjs +9 -0
  125. package/src/runtime/channels/lib/tool-dispatch.mjs +9 -5
  126. package/src/runtime/channels/lib/worker-main.mjs +16 -5
  127. package/src/runtime/memory/index.mjs +46 -202
  128. package/src/runtime/memory/lib/memory-action-handlers.mjs +2 -53
  129. package/src/runtime/memory/lib/memory-daemon-lifecycle.mjs +115 -0
  130. package/src/runtime/memory/lib/memory-port-advertiser.mjs +105 -0
  131. package/src/runtime/memory/lib/pg/adapter.mjs +84 -10
  132. package/src/runtime/memory/lib/pg/process.mjs +91 -47
  133. package/src/runtime/memory/lib/pg/supervisor.mjs +50 -17
  134. package/src/runtime/memory/lib/query-handlers.mjs +2 -122
  135. package/src/runtime/memory/lib/query-maintenance-handlers.mjs +126 -0
  136. package/src/runtime/memory/lib/tool-call-handler.mjs +57 -0
  137. package/src/runtime/search/lib/http-fetch.mjs +1 -1
  138. package/src/runtime/shared/atomic-file.mjs +28 -150
  139. package/src/runtime/shared/config.mjs +58 -13
  140. package/src/runtime/shared/llm/cost.mjs +14 -4
  141. package/src/runtime/shared/memory-snapshot.mjs +236 -0
  142. package/src/runtime/shared/process-lifecycle.mjs +436 -0
  143. package/src/runtime/shared/process-shutdown.mjs +33 -4
  144. package/src/runtime/shared/resource-admission.mjs +364 -0
  145. package/src/runtime/shared/staged-child-result.mjs +19 -0
  146. package/src/session-runtime/channel-config-api.mjs +7 -7
  147. package/src/session-runtime/config-lifecycle.mjs +20 -17
  148. package/src/session-runtime/context-status.mjs +38 -27
  149. package/src/session-runtime/cwd-plugins.mjs +9 -7
  150. package/src/session-runtime/env.mjs +1 -2
  151. package/src/session-runtime/hitch-profile.mjs +45 -0
  152. package/src/session-runtime/lifecycle-api.mjs +60 -13
  153. package/src/session-runtime/mcp-glue.mjs +6 -11
  154. package/src/session-runtime/provider-init-key.mjs +17 -0
  155. package/src/session-runtime/provider-request-tools.mjs +72 -0
  156. package/src/session-runtime/runtime-core.mjs +49 -105
  157. package/src/session-runtime/runtime-paths.mjs +20 -0
  158. package/src/session-runtime/runtime-tool-routing.mjs +55 -0
  159. package/src/session-runtime/session-turn-api.mjs +5 -4
  160. package/src/session-runtime/tool-catalog-data.mjs +51 -0
  161. package/src/session-runtime/tool-catalog.mjs +390 -104
  162. package/src/standalone/agent-tool/spawn-preset.mjs +73 -0
  163. package/src/standalone/agent-tool/worker-rows.mjs +93 -0
  164. package/src/standalone/agent-tool.mjs +29 -200
  165. package/src/standalone/channel-admin.mjs +29 -0
  166. package/src/standalone/channel-daemon-client.mjs +200 -38
  167. package/src/standalone/channel-daemon-transport.mjs +136 -9
  168. package/src/standalone/channel-worker.mjs +79 -12
  169. package/src/tui/App.jsx +11 -5
  170. package/src/tui/dist/index.mjs +275 -343
  171. package/src/tui/engine/session-api-ext.mjs +50 -6
  172. package/src/tui/engine/session-api.mjs +1 -1
  173. package/src/tui/engine/turn.mjs +10 -6
  174. package/src/tui/engine.mjs +13 -4
  175. package/src/tui/index.jsx +5 -1
  176. package/src/tui/lib/voice-setup.mjs +21 -5
  177. package/scripts/_devtools-stub.mjs +0 -1
  178. package/src/runtime/lib/keychain-cjs.cjs +0 -288
@@ -4,8 +4,70 @@ function __mixdogMemoryLog(...args) {
4
4
  return __mixdogMemoryStderrWrite(...args);
5
5
  }
6
6
 
7
- function installPoolErrorHandler(pool, label) {
7
+ const _poolClientErrorHandlers = new WeakMap()
8
+
9
+ export function isPgConnectionLossError(err) {
10
+ const seen = new Set()
11
+ let cur = err
12
+ for (let depth = 0; cur && depth < 5 && !seen.has(cur); depth++) {
13
+ seen.add(cur)
14
+ const code = String(cur?.code || cur?.errno || '').toUpperCase()
15
+ const msg = String(cur?.message || cur || '')
16
+ if (
17
+ code === 'ECONNREFUSED'
18
+ || code === 'ECONNRESET'
19
+ || code === 'EPIPE'
20
+ || code === 'ENETRESET'
21
+ || code === '57P01'
22
+ || code === '57P02'
23
+ || code === '57P03'
24
+ || /connection terminated(?: unexpectedly)?/i.test(msg)
25
+ || /server closed the connection unexpectedly/i.test(msg)
26
+ || /read ECONNRESET|socket hang up|connect ECONNREFUSED/i.test(msg)
27
+ || /cannot use a pool after calling end/i.test(msg)
28
+ ) return true
29
+ cur = cur?.cause
30
+ }
31
+ return false
32
+ }
33
+
34
+ function isSafePreDispatchRetryError(err) {
35
+ const code = String(err?.code || err?.errno || '').toUpperCase()
36
+ const msg = String(err?.message || err || '')
37
+ return code === 'ECONNREFUSED'
38
+ || /connect ECONNREFUSED/i.test(msg)
39
+ || /cannot use a pool after calling end/i.test(msg)
40
+ }
41
+
42
+ export function installPoolErrorHandler(pool, label, { onConnectionLoss } = {}) {
8
43
  if (!pool || typeof pool.on !== 'function') return pool
44
+ // pg-pool deliberately removes its idle error listener while a Client is
45
+ // checked out. Long-running advisory-lock owners (cycle1/2/3) can therefore
46
+ // receive a socket-end `error` between queries with no listener at all,
47
+ // which Node treats as an uncaught exception. Give every physical Client a
48
+ // permanent last-resort listener; pg-pool's own idle listener still owns
49
+ // eviction when the Client is back in the pool.
50
+ pool.on('connect', (client) => {
51
+ if (!client || typeof client.on !== 'function' || _poolClientErrorHandlers.has(client)) return
52
+ const handler = (err) => {
53
+ const code = err?.code || err?.errno || ''
54
+ const msg = err?.message || String(err || 'unknown error')
55
+ const pid = client?.processID ? ` pid=${client.processID}` : ''
56
+ const suffix = code ? ` ${code}` : ''
57
+ __mixdogMemoryLog(`[pg-adapter] ${label} client error${suffix}${pid}: ${msg}\n`)
58
+ if (isPgConnectionLossError(err) && typeof onConnectionLoss === 'function') {
59
+ try {
60
+ Promise.resolve(onConnectionLoss(err, client)).catch((recoverErr) => {
61
+ __mixdogMemoryLog(`[pg-adapter] ${label} client-error recovery failed: ${recoverErr?.message || recoverErr}\n`)
62
+ })
63
+ } catch (recoverErr) {
64
+ __mixdogMemoryLog(`[pg-adapter] ${label} client-error recovery failed: ${recoverErr?.message || recoverErr}\n`)
65
+ }
66
+ }
67
+ }
68
+ _poolClientErrorHandlers.set(client, handler)
69
+ client.on('error', handler)
70
+ })
9
71
  pool.on('error', (err, client) => {
10
72
  const code = err?.code || err?.errno || ''
11
73
  const msg = err?.message || String(err || 'unknown error')
@@ -139,7 +201,7 @@ function makeCompatDb(pgPool, schema, dataDir) {
139
201
  client.release()
140
202
  }
141
203
  } catch (err) {
142
- if (err?.code === 'ECONNREFUSED') _recoverPgConnection(dataDir, schema).catch(() => {})
204
+ if (isPgConnectionLossError(err)) _recoverPgConnection(dataDir, schema).catch(() => {})
143
205
  throw err
144
206
  }
145
207
  },
@@ -152,8 +214,10 @@ function makeCompatDb(pgPool, schema, dataDir) {
152
214
  // side effects. Recover in the background for the next caller and
153
215
  // propagate the original error immediately.
154
216
  const pool = instances.get(`${resolve(dataDir)}|${schema}`)?.pool ?? pgPool
155
- const client = await _checkedConnect(pool, schema)
217
+ let client = null
218
+ let releaseErr = null
156
219
  try {
220
+ client = await _checkedConnect(pool, schema)
157
221
  await client.query('BEGIN')
158
222
  const tx = {
159
223
  query: (sql, params) => client.query(sql, params),
@@ -163,11 +227,16 @@ function makeCompatDb(pgPool, schema, dataDir) {
163
227
  await client.query('COMMIT')
164
228
  return result
165
229
  } catch (err) {
166
- try { await client.query('ROLLBACK') } catch {}
167
- if (err?.code === 'ECONNREFUSED') _recoverPgConnection(dataDir, schema).catch(() => {})
230
+ if (client) {
231
+ try { await client.query('ROLLBACK') } catch {}
232
+ }
233
+ if (isPgConnectionLossError(err)) {
234
+ releaseErr = err instanceof Error ? err : new Error(String(err))
235
+ _recoverPgConnection(dataDir, schema).catch(() => {})
236
+ }
168
237
  throw err
169
238
  } finally {
170
- client.release()
239
+ if (client) client.release(releaseErr || undefined)
171
240
  }
172
241
  },
173
242
 
@@ -228,7 +297,7 @@ async function _recoverPgConnection(dataDir, schema) {
228
297
 
229
298
  const p = (async () => {
230
299
  _lastRecoverAt.set(dataDirKey, Date.now())
231
- __mixdogMemoryLog(`[pg-adapter] ECONNREFUSED on pool query — recovering PG for dataDir=${dataDirKey}\n`)
300
+ __mixdogMemoryLog(`[pg-adapter] connection loss — recovering PG for dataDir=${dataDirKey}\n`)
232
301
  // memory + trace schemas share one PG cluster/port; discard every cached
233
302
  // pool for this dataDir so a dead-port pool is never reused after restart.
234
303
  // Track every schema whose pool was actually closed here so ALL of them
@@ -275,9 +344,12 @@ async function withPgRetry(dataDir, schema, fn) {
275
344
  try {
276
345
  return await fn(pool0)
277
346
  } catch (err) {
278
- if (err?.code !== 'ECONNREFUSED') throw err
347
+ if (!isPgConnectionLossError(err)) throw err
279
348
  const recovered = await _recoverPgConnection(dataDir, schema)
280
- if (!recovered) throw err
349
+ // Only retry failures proven to happen before dispatch. ECONNRESET and
350
+ // server-termination errors can arrive after PostgreSQL applied a write;
351
+ // replaying a generic query would risk duplicate side effects.
352
+ if (!recovered || !isSafePreDispatchRetryError(err)) throw err
281
353
  const pool1 = instances.get(key)?.pool
282
354
  if (!pool1) throw err
283
355
  return await fn(pool1)
@@ -422,7 +494,9 @@ export async function ensurePgInstance(dataDir, opts = {}) {
422
494
  password: '', max: 5, idleTimeoutMillis: 30_000,
423
495
  connectionTimeoutMillis: 10_000,
424
496
  })
425
- installPoolErrorHandler(pgPool, `pool:${schema}`)
497
+ installPoolErrorHandler(pgPool, `pool:${schema}`, {
498
+ onConnectionLoss: () => _recoverPgConnection(dataDir, schema),
499
+ })
426
500
 
427
501
  // 4. Bootstrap extensions + schemas once (idempotent).
428
502
  await bootstrapInstance(pgPool, resolve(dataDir))
@@ -55,6 +55,51 @@ function isTcpPortFree(port) {
55
55
 
56
56
  const PG_PORT_MIN = 55432
57
57
  const PG_PORT_MAX = 55632
58
+ const delay = ms => new Promise(resolve => setTimeout(resolve, ms))
59
+
60
+ function isPidAlive(pid) {
61
+ if (!Number.isInteger(pid) || pid <= 0) return false
62
+ try {
63
+ process.kill(pid, 0)
64
+ return true
65
+ } catch (err) {
66
+ return err?.code === 'EPERM'
67
+ }
68
+ }
69
+
70
+ function readPostmasterInfo(pgdataDir) {
71
+ try {
72
+ const lines = readFileSync(join(pgdataDir, 'postmaster.pid'), 'utf8').split('\n')
73
+ const pid = parseInt(lines[0], 10)
74
+ const port = parseInt(lines[3], 10)
75
+ return {
76
+ pid: Number.isInteger(pid) && pid > 0 ? pid : null,
77
+ port: Number.isInteger(port) && port > 0 ? port : null,
78
+ }
79
+ } catch {
80
+ return { pid: null, port: null }
81
+ }
82
+ }
83
+
84
+ function pgIsReady(runtimeDir, env, port) {
85
+ const probe = spawnSync(pgBin(runtimeDir, 'pg_isready'), ['-h', '127.0.0.1', '-p', String(port)], {
86
+ env, stdio: 'pipe', timeout: 3_000, windowsHide: true,
87
+ })
88
+ return probe.status === 0
89
+ }
90
+
91
+ async function awaitExistingPostmaster({ runtimeDir, pgdataDir, env, waitMs }) {
92
+ const deadline = Date.now() + Math.max(0, Number(waitMs) || 0)
93
+ let info = readPostmasterInfo(pgdataDir)
94
+ while (info.pid && info.port) {
95
+ if (pgIsReady(runtimeDir, env, info.port)) return { state: 'ready', ...info }
96
+ if (!isPidAlive(info.pid)) return { state: 'dead', ...info }
97
+ if (Date.now() >= deadline) return { state: 'alive-not-ready', ...info }
98
+ await delay(250)
99
+ info = readPostmasterInfo(pgdataDir)
100
+ }
101
+ return { state: 'missing', ...info }
102
+ }
58
103
 
59
104
  async function findFreePort(preferred) {
60
105
  // I2: clamp out-of-range callers to the valid window.
@@ -135,7 +180,13 @@ export function reconcileConfV2(runtimeDir, pgdataDir) {
135
180
  // startPg
136
181
  // ---------------------------------------------------------------------------
137
182
 
138
- export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 55432, logPath }) {
183
+ export async function startPg({
184
+ runtimeDir,
185
+ pgdataDir,
186
+ port: preferredPort = 55432,
187
+ logPath,
188
+ existingWaitMs = 15_000,
189
+ }) {
139
190
  mkdirSync(pgdataDir, { recursive: true })
140
191
 
141
192
  if (process.platform === 'darwin') {
@@ -146,41 +197,35 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
146
197
  // the block was just appended (so the attach path can trigger pg_ctl reload).
147
198
  // Fresh-init path falls through to confAppend below which already includes v2.
148
199
  const v2Applied = ensureConfV2(pgdataDir)
200
+ const env = libEnv(runtimeDir)
149
201
 
150
202
  // Pre-check: if postmaster.pid exists and the instance is reachable, attach
151
203
  // rather than attempting a second pg_ctl start (which would crash the worker).
152
204
  const postmasterPidPath = join(pgdataDir, 'postmaster.pid')
153
205
  if (existsSync(postmasterPidPath)) {
154
- try {
155
- const lines = readFileSync(postmasterPidPath, 'utf8').split('\n')
156
- const existingPid = parseInt(lines[0], 10)
157
- // Line 4 (index 3) in postmaster.pid holds the port number.
158
- const existingPort = parseInt(lines[3], 10)
159
- if (existingPid > 0 && existingPort > 0) {
160
- // Liveness must be confirmed by pg_isready (status 0). A bare open TCP
161
- // port is insufficient: any unrelated listener that happened to grab
162
- // this port would be mistaken for our PG instance, causing a false
163
- // attach. pg_isready performs a real PG protocol handshake, so a
164
- // non-Postgres listener returns non-zero and we correctly fall through
165
- // to normal startup (which reclaims the stale postmaster.pid).
166
- const pgIsReady = pgBin(runtimeDir, 'pg_isready')
167
- const probe = spawnSync(pgIsReady, ['-h', '127.0.0.1', '-p', String(existingPort)], {
168
- env: libEnv(runtimeDir), stdio: 'pipe', timeout: 3_000, windowsHide: true,
169
- })
170
- const alive = probe.status === 0
171
- if (alive) {
172
- __mixdogMemoryLog(`[pg-process] attaching to existing PG pid=${existingPid} port=${existingPort}\n`)
173
- // Route through the single reconcile entry point so v1 → v2 conf
174
- // upgrades land on already-running instances without restart.
175
- if (v2Applied) reconcileConfV2(runtimeDir, pgdataDir)
176
- return { pid: existingPid, port: existingPort, attached: true }
177
- }
178
- // Dead instance — fall through to normal startup; pg_ctl will reclaim stale lock.
179
- }
180
- } catch {}
206
+ const existing = await awaitExistingPostmaster({
207
+ runtimeDir, pgdataDir, env, waitMs: existingWaitMs,
208
+ })
209
+ if (existing.state === 'ready') {
210
+ __mixdogMemoryLog(`[pg-process] attaching to existing PG pid=${existing.pid} port=${existing.port}\n`)
211
+ // Route through the single reconcile entry point so v1 → v2 conf
212
+ // upgrades land on already-running instances without restart.
213
+ if (v2Applied) reconcileConfV2(runtimeDir, pgdataDir)
214
+ return { pid: existing.pid, port: existing.port, attached: true }
215
+ }
216
+ if (existing.state === 'alive-not-ready') {
217
+ // A postmaster in startup/shutdown still owns this pgdata even when it no
218
+ // longer accepts connections. Starting the same directory on a "free"
219
+ // alternate port produces the observed endless "another server might be
220
+ // running" loop and can corrupt lifecycle state. The supervisor may
221
+ // finish a graceful stop, but this lower layer must never race it.
222
+ throw new Error(
223
+ `[pg-process] existing PG pid=${existing.pid} port=${existing.port} is alive but not ready; refusing concurrent start`,
224
+ )
225
+ }
226
+ // Dead instance fall through to normal startup; pg_ctl reclaims the stale lock.
181
227
  }
182
228
 
183
- const env = libEnv(runtimeDir)
184
229
  const initdb = pgBin(runtimeDir, 'initdb')
185
230
  const pgctl = pgBin(runtimeDir, 'pg_ctl')
186
231
 
@@ -248,7 +293,6 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
248
293
 
249
294
  __mixdogMemoryLog(`[pg-process] pg_ctl start -D ${pgdataDir} -p ${port}\n`)
250
295
 
251
- const pgIsReady = pgBin(runtimeDir, 'pg_isready')
252
296
  const startArgs = [
253
297
  'start',
254
298
  '-D', pgdataDir,
@@ -258,8 +302,6 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
258
302
 
259
303
  // Poll-sleep is intentionally NOT unref'd: while startPg is awaiting readiness
260
304
  // it must keep the process alive even if the event loop would otherwise drain.
261
- const sleep = ms => new Promise(res => setTimeout(res, ms))
262
-
263
305
  // Read pid + port from postmaster.pid (line 1 = pid, line 4 = port). Returns
264
306
  // null unless the file exists, pid > 0, and its port matches ours.
265
307
  function readPostmaster() {
@@ -280,7 +322,7 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
280
322
  for (let i = 0; i < 5; i++) {
281
323
  const pm = readPostmaster()
282
324
  if (pm) return pm.pid
283
- await sleep(100)
325
+ await delay(100)
284
326
  }
285
327
  return null
286
328
  }
@@ -312,10 +354,7 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
312
354
  const deadline = Date.now() + 30_000
313
355
  // eslint-disable-next-line no-constant-condition
314
356
  while (true) {
315
- const probe = spawnSync(pgIsReady, ['-h', '127.0.0.1', '-p', String(port)], {
316
- env, stdio: 'pipe', timeout: 3_000, windowsHide: true,
317
- })
318
- if (probe.status === 0) {
357
+ if (pgIsReady(runtimeDir, env, port)) {
319
358
  const pid = await confirmPid()
320
359
  // pid confirmed with matching port → ready. Otherwise keep polling until
321
360
  // the cap (postmaster.pid not yet written or port mismatch).
@@ -324,7 +363,7 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
324
363
  // pg_ctl exited nonzero before PG became reachable — real startup failure.
325
364
  if (closed && exitCode !== 0) return { ready: false, exited: true, stdout, stderr }
326
365
  if (Date.now() >= deadline) return { ready: false, timeout: true, stdout, stderr }
327
- await sleep(250)
366
+ await delay(250)
328
367
  }
329
368
  }
330
369
 
@@ -332,19 +371,19 @@ export async function startPg({ runtimeDir, pgdataDir, port: preferredPort = 554
332
371
  if (r.ready) return { pid: r.pid, port }
333
372
 
334
373
  const errText = r.stderr || r.stdout || ''
335
- // "another server might be running" try status + immediate stop before failing.
374
+ // A cross-process race can still land after the pre-check. Re-probe and
375
+ // attach if that winner becomes ready; never immediate-stop an unknown live
376
+ // postmaster, because synchronous_commit=off makes a crash-stop lossy.
336
377
  if (r.exited && errText.includes('another server might be running')) {
337
378
  __mixdogMemoryLog(`[pg-process] pg_ctl start: "another server might be running" — probing status\n`)
338
379
  const statusR = spawnSync(pgctl, ['status', '-D', pgdataDir], { env, stdio: 'pipe', timeout: 3_000, windowsHide: true })
339
380
  __mixdogMemoryLog(`[pg-process] pg_ctl status: ${statusR.stdout?.toString() || statusR.stderr?.toString() || 'no output'}\n`)
340
- const stopR = spawnSync(pgctl, ['stop', '-m', 'immediate', '-D', pgdataDir], { env, stdio: 'pipe', timeout: 3_000, windowsHide: true })
341
- if (stopR.status === 0) {
342
- // Retry start once after clearing the stale instance.
343
- const r2 = await startAndWaitReady()
344
- if (r2.ready) return { pid: r2.pid, port }
345
- __mixdogMemoryLog(`[pg-process] retry start after stop also failed: ${r2.stderr || ''}\n`)
346
- } else {
347
- __mixdogMemoryLog(`[pg-process] immediate stop failed: ${stopR.stderr?.toString() || ''} — treating as degraded\n`)
381
+ const existing = await awaitExistingPostmaster({
382
+ runtimeDir, pgdataDir, env, waitMs: existingWaitMs,
383
+ })
384
+ if (existing.state === 'ready') {
385
+ __mixdogMemoryLog(`[pg-process] attaching to race winner pid=${existing.pid} port=${existing.port}\n`)
386
+ return { pid: existing.pid, port: existing.port, attached: true }
348
387
  }
349
388
  }
350
389
 
@@ -419,6 +458,11 @@ export async function healthcheckPg({ port, host = '127.0.0.1' }) {
419
458
  host, port, user: 'postgres', database: 'postgres', password: '',
420
459
  connectionTimeoutMillis: 2000,
421
460
  })
461
+ // A socket can terminate after SELECT 1 resolves but before client.end()
462
+ // finishes. Raw pg.Client instances have no pool-level error listener, so
463
+ // keep this transient health probe from turning that narrow race into an
464
+ // uncaught EventEmitter error.
465
+ client.on('error', () => {})
422
466
  await client.connect()
423
467
  await client.query('SELECT 1')
424
468
  return true
@@ -341,10 +341,6 @@ function patchActiveInstance(fields, opts = {}) {
341
341
 
342
342
  // ── postmaster.pid helpers ───────────────────────────────────────────────────
343
343
 
344
- function readPostmasterPid(pgdata) {
345
- return readPostmasterInfo(pgdata).pid;
346
- }
347
-
348
344
  function readPostmasterInfo(pgdata) {
349
345
  try {
350
346
  const raw = readFileSync(join(pgdata, 'postmaster.pid'), 'utf8');
@@ -399,6 +395,18 @@ async function isPostmasterAlive(pid) {
399
395
  return isPostgresPid(pid);
400
396
  }
401
397
 
398
+ async function waitForPostmasterReady({ pid, port, healthcheckPg, timeoutMs = 30_000 }) {
399
+ const deadline = Date.now() + Math.max(0, Number(timeoutMs) || 0);
400
+ while (isPidAlive(pid)) {
401
+ try {
402
+ if (await healthcheckPg({ port })) return 'ready';
403
+ } catch {}
404
+ if (Date.now() >= deadline) return 'alive-not-ready';
405
+ await new Promise(resolve => setTimeout(resolve, 250));
406
+ }
407
+ return 'dead';
408
+ }
409
+
402
410
  // ── Internal: spawn a fresh PG instance ─────────────────────────────────────
403
411
 
404
412
  async function _startFresh(dataDir, pgdata, port, runtimeDir) {
@@ -473,7 +481,7 @@ async function tryReusePgInstance({ pgdata, runtimeDir, healthcheckPg, source =
473
481
  // ── Internal: full ensure logic (runs exclusively via _ensureInFlight) ────────
474
482
 
475
483
  async function _doEnsure(dataDir) {
476
- const { healthcheckPg, stopPg } = await _getPgProc();
484
+ const { healthcheckPg } = await _getPgProc();
477
485
  const pgdata = join(dataDir, 'pgdata');
478
486
 
479
487
  // Resolve runtimeDir via runtime-fetcher (cache-hits immediately when already downloaded).
@@ -563,22 +571,47 @@ async function _doEnsure(dataDir) {
563
571
  __mixdogMemoryLog(
564
572
  `[supervisor-pg] pg_port=${existingPort} recorded but healthcheck failed — recovering\n`,
565
573
  );
566
- const pmPid = readPostmasterPid(pgdata);
567
- if (pmPid && await isPostmasterAlive(pmPid)) {
568
- // postmaster alive but unhealthy: attempt graceful stop first
569
- __mixdogMemoryLog(`[supervisor-pg] postmaster PID ${pmPid} alive — attempting graceful stopPg\n`);
570
- try { await stopPg({ runtimeDir, pgdataDir: pgdata }); } catch (e) {
571
- __mixdogMemoryLog(`[supervisor-pg] graceful stopPg failed: ${e?.message} — continuing to fresh start\n`);
572
- }
573
- } else if (pmPid) {
574
- // postmaster dead: remove stale postmaster.pid so initdb/start is not blocked
575
- __mixdogMemoryLog(`[supervisor-pg] postmaster PID ${pmPid} dead — removing stale postmaster.pid\n`);
576
- try { unlinkSync(join(pgdata, 'postmaster.pid')); } catch {}
577
- }
578
574
  // Clear stale pg fields before restart
579
575
  patchActiveInstance({ pg_port: null, pg_started_at: null, pg_pgdata: null }, { timeoutMs: 1000, background: true });
580
576
  }
581
577
 
578
+ // postmaster.pid is the authoritative pgdata owner even when the discovery
579
+ // advert is absent/stale. The old path checked it only inside the
580
+ // `existingPort` branch, then allocated 55433 and tried to start the SAME
581
+ // pgdata while a recovering/rejecting server still owned 55432. Give crash
582
+ // recovery a bounded chance to finish and attach if it does. Never issue a
583
+ // stop from an ensure path: an alive postmaster owns the data directory,
584
+ // and interrupting recovery can turn a transient outage into a restart loop.
585
+ const pm = readPostmasterInfo(pgdata);
586
+ if (pm.pid && pm.port && await isPostmasterAlive(pm.pid)) {
587
+ __mixdogMemoryLog(`[supervisor-pg] postmaster PID ${pm.pid} alive but not ready — awaiting recovery\n`);
588
+ const state = await waitForPostmasterReady({
589
+ pid: pm.pid,
590
+ port: pm.port,
591
+ healthcheckPg,
592
+ });
593
+ if (state === 'ready') {
594
+ __mixdogMemoryLog(`[supervisor-pg] attaching to recovered PG pid=${pm.pid} port=${pm.port}\n`);
595
+ _live = { port: pm.port, pgdata, runtimeDir, proc: null };
596
+ patchActiveInstance({
597
+ pg_port: pm.port,
598
+ pg_started_at: ai?.pg_started_at ?? Date.now(),
599
+ pg_pgdata: pgdata,
600
+ pg_runtime_dir: runtimeDir,
601
+ }, { timeoutMs: 1000, background: true });
602
+ return { host: '127.0.0.1', port: pm.port, runtimeDir, pgdataDir: pgdata };
603
+ }
604
+ if (state === 'alive-not-ready') {
605
+ throw new Error(
606
+ `[supervisor-pg] existing postmaster PID ${pm.pid} remains alive but unhealthy; refusing concurrent start`,
607
+ );
608
+ }
609
+ }
610
+ if (pm.pid) {
611
+ __mixdogMemoryLog(`[supervisor-pg] postmaster PID ${pm.pid} dead — removing stale postmaster.pid\n`);
612
+ try { unlinkSync(join(pgdata, 'postmaster.pid')); } catch {}
613
+ }
614
+
582
615
  // ── Allocate a fresh port and spawn ───────────────────────────────────
583
616
  const port = await allocatePort();
584
617
  return await _startFresh(dataDir, pgdata, port, runtimeDir);
@@ -24,8 +24,8 @@ import { searchRelevantHybrid } from './memory-recall-store.mjs'
24
24
  import { fetchEntriesByIdsScoped } from './memory-recall-id-patch.mjs'
25
25
  import { retrieveEntries } from './memory-retrievers.mjs'
26
26
  import { buildPromotedExclusionClauses } from './memory-recall-scope-filter.mjs'
27
- import { cleanMemoryText } from './memory.mjs'
28
27
  import { insertTraceEvents } from './trace-store.mjs'
28
+ import { createQueryMaintenanceHandlers } from './query-maintenance-handlers.mjs'
29
29
  import {
30
30
  embedText,
31
31
  embedTexts,
@@ -45,6 +45,7 @@ export function createQueryHandlers({
45
45
  // instance (one memory runtime = one factory) so the 10s de-dup window
46
46
  // behaves exactly as it did when it was a top-level `let` in index.mjs.
47
47
  let _embeddingColdRecallLogAt = 0
48
+ const { dumpSessionRootChunks, entryStats } = createQueryMaintenanceHandlers({ getDb })
48
49
 
49
50
  // Raw-row priority lookup for narrow-window queries. Raw rows (is_root=0,
50
51
  // chunk_root IS NULL) are inserted immediately by ingestTranscriptFile before
@@ -866,127 +867,6 @@ export function createQueryHandlers({
866
867
  return { text: recallCapPrefix + renderSessionGroupedLines(sliced, { currentSessionId: _currentSessionHint, recencyOrder: sort === 'date' }) }
867
868
  }
868
869
 
869
- async function dumpSessionRootChunks(args = {}) {
870
- const db = getDb()
871
- const sessionId = String(args.sessionId || args.session_id || '').trim()
872
- if (!sessionId) return { text: '(no current session)', rows: [], chunks: [], isError: true }
873
- const includeRaw = args.includeRaw !== false
874
- const limit = Math.max(1, Math.min(1000, Number(args.limit) || 1000))
875
- const rootRows = (await db.query(`
876
- SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
877
- element, category, summary, status, score, last_seen_at, project_id
878
- FROM entries
879
- WHERE session_id = $1 AND is_root = 1
880
- ORDER BY COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
881
- LIMIT $2
882
- `, [sessionId, limit])).rows
883
- const roots = rootRows.map((r) => ({ ...r, members: [] }))
884
- const rootIds = roots.map((r) => Number(r.id)).filter((id) => Number.isFinite(id))
885
- const memberRows = rootIds.length > 0
886
- ? (await db.query(`
887
- SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root, project_id
888
- FROM entries
889
- WHERE chunk_root = ANY($1::bigint[]) AND is_root = 0
890
- ORDER BY chunk_root ASC, COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
891
- `, [rootIds])).rows
892
- : []
893
- const byRoot = new Map(roots.map((r) => [Number(r.id), r]))
894
- for (const m of memberRows) {
895
- const root = byRoot.get(Number(m.chunk_root))
896
- if (root) root.members.push(m)
897
- }
898
- let rawRows = []
899
- if (includeRaw) {
900
- rawRows = (await db.query(`
901
- SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root, project_id
902
- FROM entries
903
- WHERE session_id = $1
904
- AND is_root = 0
905
- AND (chunk_root IS NULL OR chunk_root = id)
906
- ORDER BY COALESCE(source_turn, 2147483647) ASC, ts ASC, id ASC
907
- LIMIT $2
908
- `, [sessionId, limit])).rows
909
- }
910
- const chunks = []
911
- for (const root of roots) {
912
- const memberText = root.members
913
- .map((m) => `${m.role === 'assistant' ? 'assistant' : m.role === 'user' ? 'user' : m.role}: ${cleanMemoryText(String(m.content ?? ''))}`)
914
- .filter(Boolean)
915
- .join('\n')
916
- const summary = [root.element, root.summary].map((v) => String(v || '').trim()).filter(Boolean).join(' — ')
917
- chunks.push({
918
- id: Number(root.id),
919
- kind: 'root',
920
- ts: Number(root.ts) || 0,
921
- sourceTurn: root.source_turn ?? null,
922
- category: root.category || null,
923
- summary,
924
- text: memberText || cleanMemoryText(String(root.content ?? '')),
925
- members: root.members,
926
- })
927
- }
928
- for (const raw of rawRows) {
929
- chunks.push({
930
- id: Number(raw.id),
931
- kind: 'raw',
932
- chunkRoot: raw.chunk_root ?? null,
933
- ts: Number(raw.ts) || 0,
934
- sourceTurn: raw.source_turn ?? null,
935
- category: null,
936
- summary: '',
937
- text: `${raw.role === 'assistant' ? 'assistant' : raw.role === 'user' ? 'user' : raw.role}: ${cleanMemoryText(String(raw.content ?? ''))}`,
938
- members: [],
939
- })
940
- }
941
- chunks.sort((a, b) => {
942
- const at = Number.isFinite(Number(a.sourceTurn)) ? Number(a.sourceTurn) : 2147483647
943
- const bt = Number.isFinite(Number(b.sourceTurn)) ? Number(b.sourceTurn) : 2147483647
944
- return (at - bt) || ((a.ts || 0) - (b.ts || 0)) || ((a.id || 0) - (b.id || 0))
945
- })
946
- const text = chunks.length
947
- ? chunks.map((chunk, idx) => {
948
- const label = chunk.kind === 'root'
949
- ? `# chunk ${idx + 1} root=${chunk.id}${chunk.category ? ` category=${chunk.category}` : ''}`
950
- : `${chunk.chunkRoot == null ? '# raw_pending' : '# raw_terminal'} ${idx + 1} id=${chunk.id}`
951
- const summary = chunk.summary ? `summary: ${chunk.summary}\n` : ''
952
- return `${label}\n${summary}${chunk.text}`.trim()
953
- }).join('\n\n')
954
- : '(no results)'
955
- return { text, rows: [...roots, ...rawRows], chunks }
956
- }
957
-
958
- async function entryStats() {
959
- const db = getDb()
960
- return await db.transaction(async (tx) => {
961
- const total = (await tx.query(`SELECT COUNT(*) c FROM entries`)).rows[0].c
962
- const roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1`)).rows[0].c
963
- const active_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active'`)).rows[0].c
964
- const archived_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'archived'`)).rows[0].c
965
- const unchunked_leaves = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE chunk_root IS NULL`)).rows[0].c
966
- const cycle2_pending_roots = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`)).rows[0].c
967
- const core_entries = (await tx.query(`SELECT COUNT(*) c FROM core_entries`)).rows[0].c
968
- const core_embed_null = (await tx.query(`SELECT COUNT(*) c FROM core_entries WHERE embedding IS NULL`)).rows[0].c
969
- const active_core_summaries = (await tx.query(`SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active' AND core_summary IS NOT NULL`)).rows[0].c
970
- const active_core_summary_missing = (await tx.query(`
971
- SELECT COUNT(*) c
972
- FROM entries
973
- WHERE is_root = 1
974
- AND status = 'active'
975
- AND (core_summary IS NULL OR btrim(core_summary) = '')
976
- `)).rows[0].c
977
- const byStatus = (await tx.query(`SELECT status, COUNT(*) c FROM entries WHERE is_root = 1 GROUP BY status`)).rows
978
- const byCategory = (await tx.query(`SELECT category, COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'active' GROUP BY category ORDER BY c DESC`)).rows
979
- const mvRows = (await tx.query(`SELECT relispopulated FROM pg_class WHERE relname = 'mv_hot_active' LIMIT 1`)).rows
980
- const mv_hot_active_populated = mvRows.length ? Boolean(mvRows[0].relispopulated) : null
981
- return {
982
- total, roots, active_roots, archived_roots, unchunked_leaves, cycle2_pending_roots,
983
- core_entries, core_embed_null, active_core_summaries, active_core_summary_missing,
984
- mv_hot_active_populated,
985
- byStatus, byCategory,
986
- }
987
- })
988
- }
989
-
990
870
  return {
991
871
  readRawRowsInWindow,
992
872
  recallSessionRows,