mixdog 0.7.8 → 0.7.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/.claude-plugin/marketplace.json +5 -2
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/CHANGELOG.md +40 -0
  4. package/README.md +198 -251
  5. package/bin/statusline-launcher.mjs +5 -1
  6. package/bin/statusline-lib.mjs +14 -6
  7. package/bin/statusline.mjs +14 -6
  8. package/hooks/lib/settings-loader.cjs +4 -3
  9. package/hooks/pre-tool-subagent.cjs +7 -2
  10. package/hooks/session-start.cjs +52 -24
  11. package/lib/mixdog-debug.cjs +163 -0
  12. package/native/prebuilt/linux-aarch64/mixdog-shim +0 -0
  13. package/native/prebuilt/linux-x86_64/mixdog-shim +0 -0
  14. package/native/prebuilt/macos-aarch64/mixdog-shim +0 -0
  15. package/native/prebuilt/macos-x86_64/mixdog-shim +0 -0
  16. package/native/prebuilt/windows-x86_64/mixdog-shim.exe +0 -0
  17. package/package.json +1 -1
  18. package/scripts/builtin-utils-smoke.mjs +14 -8
  19. package/scripts/bump.mjs +80 -0
  20. package/scripts/doctor.mjs +8 -3
  21. package/scripts/mutation-io-smoke.mjs +17 -1
  22. package/scripts/openai-oauth-catalog-smoke.mjs +53 -0
  23. package/scripts/permission-eval-smoke.mjs +18 -1
  24. package/scripts/statusline-launcher-smoke.mjs +2 -2
  25. package/scripts/webhook-selfheal-smoke.mjs +1 -3
  26. package/server-main.mjs +57 -3
  27. package/setup/config-merge.mjs +0 -1
  28. package/setup/install.mjs +241 -51
  29. package/setup/mixdog-cli.mjs +30 -3
  30. package/setup/setup-server.mjs +21 -33
  31. package/setup/setup.html +46 -11
  32. package/setup/tui.mjs +35 -316
  33. package/src/agent/orchestrator/config.mjs +0 -1
  34. package/src/agent/orchestrator/providers/anthropic-oauth.mjs +2 -5
  35. package/src/agent/orchestrator/providers/anthropic.mjs +243 -86
  36. package/src/agent/orchestrator/providers/gemini.mjs +386 -31
  37. package/src/agent/orchestrator/providers/grok-oauth.mjs +2 -5
  38. package/src/agent/orchestrator/providers/model-catalog.mjs +146 -13
  39. package/src/agent/orchestrator/providers/openai-compat-stream.mjs +366 -0
  40. package/src/agent/orchestrator/providers/openai-compat.mjs +74 -30
  41. package/src/agent/orchestrator/providers/openai-oauth-ws.mjs +2 -1
  42. package/src/agent/orchestrator/providers/openai-oauth.mjs +66 -13
  43. package/src/agent/orchestrator/providers/openai-ws.mjs +23 -0
  44. package/src/agent/orchestrator/session/manager.mjs +18 -4
  45. package/src/agent/orchestrator/stall-policy.mjs +6 -0
  46. package/src/agent/orchestrator/tools/builtin/native-edit-runner.mjs +29 -8
  47. package/src/agent/orchestrator/tools/graph-manifest.json +11 -11
  48. package/src/agent/orchestrator/tools/patch-manifest.json +11 -11
  49. package/src/channels/index.mjs +27 -8
  50. package/src/channels/lib/event-queue.mjs +24 -1
  51. package/src/channels/lib/hook-pipe-server.mjs +21 -8
  52. package/src/channels/lib/webhook.mjs +142 -20
  53. package/src/memory/lib/memory-cycle1.mjs +7 -3
  54. package/src/memory/lib/memory-recall-store.mjs +27 -10
  55. package/src/search/lib/backends/openai-oauth.mjs +6 -2
  56. package/src/search/lib/cache.mjs +55 -7
  57. package/src/shared/config.mjs +1 -1
  58. package/src/shared/llm/cost.mjs +2 -2
  59. package/src/shared/open-url.mjs +37 -0
  60. package/src/shared/seed.mjs +20 -3
  61. package/src/shared/user-data-guard.mjs +3 -1
  62. package/scripts/test-config-rmw-restore.mjs +0 -122
  63. package/setup/wizard.mjs +0 -696
@@ -1,26 +1,26 @@
1
1
  {
2
- "version": "0.6.5",
2
+ "version": "0.7.12",
3
3
  "_comment": "Rewritten by .github/workflows/patch-release.yml on each tagged release. assets maps platformKey (process.platform-process.arch, e.g. win32-x64, linux-x64, darwin-arm64) to { url, sha256 } of the mixdog-patch binary on the GitHub release. A local cargo build under native/mixdog-patch/target/release always takes precedence; otherwise the binary is fetched per this manifest into the data dir (apply is native-only — no JS apply engine).",
4
4
  "assets": {
5
5
  "darwin-arm64": {
6
- "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.6.5/mixdog-patch-darwin-arm64",
7
- "sha256": "d37afb583cd597a9599ea9feac76a853d215ffaadc2bb2b54cf71ef28848f7ae"
6
+ "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-darwin-arm64",
7
+ "sha256": "836a0b60a443b0a6a8c1bbe24d15a79ed70ee92c2f0fbc05374c4e9ed2536415"
8
8
  },
9
9
  "darwin-x64": {
10
- "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.6.5/mixdog-patch-darwin-x64",
11
- "sha256": "bc4dad6a7fdc2fcdfceb850bcaae43757ae304740ca503be955eb57cf8cd07e3"
10
+ "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-darwin-x64",
11
+ "sha256": "cab10c4e1e8b72d3958241dffdff764712ed74f4861d105bafa8258961215c98"
12
12
  },
13
13
  "linux-arm64": {
14
- "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.6.5/mixdog-patch-linux-arm64",
15
- "sha256": "ebe3fd45aaed0f383f7b7940733fd141dd0b93c4c44e08fbffab10a3e43b788c"
14
+ "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-linux-arm64",
15
+ "sha256": "a90c32ce3417a7d853f2723f82f3613cf2cd030fe885cf710cfc9f8e4b193264"
16
16
  },
17
17
  "linux-x64": {
18
- "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.6.5/mixdog-patch-linux-x64",
19
- "sha256": "627289a3b5c0156bc299d4ff7563e4e4536a5a5bf6329e7c9151e811c795928f"
18
+ "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-linux-x64",
19
+ "sha256": "0fea40ab98acd35bfb47515756024d1882a2abbaddce8a0b51642d20ac405577"
20
20
  },
21
21
  "win32-x64": {
22
- "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.6.5/mixdog-patch-win32-x64.exe",
23
- "sha256": "4282546719a8c149597d3afe21eaf31aab3079a371876bf7f91ff0bccbe316b2"
22
+ "url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-win32-x64.exe",
23
+ "sha256": "f8a74fb9bb7bf333fa441b800b76b28d83a2a7b4795e4a769122faf47a59d1f1"
24
24
  }
25
25
  }
26
26
  }
@@ -183,10 +183,18 @@ const _bootLogEarly = path.join(
183
183
  process.env.CLAUDE_PLUGIN_DATA || path.join(os.tmpdir(), "mixdog"),
184
184
  "boot.log"
185
185
  );
186
+ const {
187
+ isMixdogDebugEnabled: isMixdogDebug,
188
+ pruneStalePluginDataLogSiblings,
189
+ appendSessionStartCriticalLog,
190
+ DEFAULT_STALE_LOG_SIBLING_MAX,
191
+ } = _require("../../lib/mixdog-debug.cjs");
186
192
  // One-shot log rotation at worker boot (10 MB threshold, .1 suffix overwrite).
187
- try { if (fs.statSync(_bootLogEarly).size > 10 * 1024 * 1024) fs.renameSync(_bootLogEarly, _bootLogEarly + '.1') } catch {}
188
- fs.appendFileSync(_bootLogEarly, `[${localTimestamp()}] bootstrap start pid=${process.pid}
193
+ if (isMixdogDebug()) {
194
+ try { if (fs.statSync(_bootLogEarly).size > 10 * 1024 * 1024) fs.renameSync(_bootLogEarly, _bootLogEarly + '.1') } catch {}
195
+ fs.appendFileSync(_bootLogEarly, `[${localTimestamp()}] bootstrap start pid=${process.pid}
189
196
  `);
197
+ }
190
198
  const _bootLog = path.join(DATA_DIR, "boot.log");
191
199
  let config = loadConfig();
192
200
  let backend = createBackend(config);
@@ -238,6 +246,11 @@ try {
238
246
  try { if (_now - fs.statSync(_p).mtimeMs > _STALE_SESSION_TTL_MS) fs.unlinkSync(_p); } catch {}
239
247
  }
240
248
  } catch {}
249
+ // Count-based cap: drop oldest *.log siblings when plugin-data accumulates
250
+ // hundreds of per-process files (doctor warns above 300).
251
+ try {
252
+ pruneStalePluginDataLogSiblings(DATA_DIR, DEFAULT_STALE_LOG_SIBLING_MAX);
253
+ } catch {}
241
254
 
242
255
  // ── Buffered drop-trace writer (channels/index) ──────────────────────────────
243
256
  // Flushes every 1 s OR when buffer reaches 64 KB — whichever fires first.
@@ -2030,8 +2043,12 @@ backend.onInteraction = (interaction) => {
2030
2043
  const [, uuid, action] = match;
2031
2044
  const access = config.access;
2032
2045
  if (!access) {
2033
- fs.appendFileSync(_bootLog, `[${localTimestamp()}] perm interaction dropped: no access config
2034
- `);
2046
+ const _permDropLine = `[${localTimestamp()}] perm interaction dropped: no access config\n`;
2047
+ if (isMixdogDebug()) {
2048
+ fs.appendFileSync(_bootLog, _permDropLine);
2049
+ } else {
2050
+ appendSessionStartCriticalLog(DATA_DIR, `[channels] ${_permDropLine}`);
2051
+ }
2035
2052
  return;
2036
2053
  }
2037
2054
  if (access.allowFrom?.length > 0 && !access.allowFrom.includes(interaction.userId)) {
@@ -3016,12 +3033,14 @@ async function stop() {
3016
3033
  return false;
3017
3034
  };
3018
3035
  _channelFlagDetected = detectChannelFlag();
3019
- fs.appendFileSync(_bootLog, `[${localTimestamp()}] channelFlag: ${_channelFlagDetected}
3020
- `);
3036
+ if (isMixdogDebug()) {
3037
+ fs.appendFileSync(_bootLog, `[${localTimestamp()}] channelFlag: ${_channelFlagDetected}\n`);
3038
+ if (_channelFlagDetected) {
3039
+ fs.appendFileSync(_bootLog, `[${localTimestamp()}] channel mode detected — bridge auto-activated\n`);
3040
+ }
3041
+ }
3021
3042
  if (_channelFlagDetected) {
3022
3043
  channelBridgeActive = true;
3023
- fs.appendFileSync(_bootLog, `[${localTimestamp()}] channel mode detected \u2014 bridge auto-activated
3024
- `);
3025
3044
  }
3026
3045
  writeBridgeState(channelBridgeActive);
3027
3046
  const previousOwner = readActiveInstance();
@@ -1,4 +1,4 @@
1
- import { readdirSync, readFileSync, existsSync as fsExistsSync } from "fs";
1
+ import { readdirSync, readFileSync, existsSync as fsExistsSync, statSync, unlinkSync } from "fs";
2
2
  import { join } from "path";
3
3
  import { DATA_DIR } from "./config.mjs";
4
4
  import { ensureDir } from "./state-file.mjs";
@@ -7,6 +7,27 @@ import { renameWithRetrySync, writeJsonAtomicSync } from "../../shared/atomic-fi
7
7
  const QUEUE_DIR = join(DATA_DIR, "events", "queue");
8
8
  const IN_PROGRESS_DIR = join(DATA_DIR, "events", "in-progress");
9
9
  const PROCESSED_DIR = join(DATA_DIR, "events", "processed");
10
+ const PROCESSED_DIR_MAX_ENTRIES = 200;
11
+ function pruneProcessedDir() {
12
+ try {
13
+ if (!fsExistsSync(PROCESSED_DIR)) return;
14
+ const names = readdirSync(PROCESSED_DIR);
15
+ if (names.length <= PROCESSED_DIR_MAX_ENTRIES) return;
16
+ const ranked = [];
17
+ for (const name of names) {
18
+ try {
19
+ const st = statSync(join(PROCESSED_DIR, name));
20
+ ranked.push({ name, mtime: st.mtimeMs });
21
+ } catch {}
22
+ }
23
+ ranked.sort((a, b) => b.mtime - a.mtime);
24
+ for (let i = PROCESSED_DIR_MAX_ENTRIES; i < ranked.length; i++) {
25
+ try {
26
+ unlinkSync(join(PROCESSED_DIR, ranked[i].name));
27
+ } catch {}
28
+ }
29
+ } catch {}
30
+ }
10
31
  function finiteInt(value, { min, max, def }) {
11
32
  const n = Number(value);
12
33
  if (!Number.isFinite(n)) return def;
@@ -290,6 +311,7 @@ ${p.item.prompt}`).join("\n\n")}`;
290
311
  const fromQueue = join(QUEUE_DIR, file);
291
312
  const src = this.existsSync(fromInProgress) ? fromInProgress : fromQueue;
292
313
  renameWithRetrySync(src, join(PROCESSED_DIR, `${status}-${file}`));
314
+ pruneProcessedDir();
293
315
  } catch {
294
316
  }
295
317
  }
@@ -297,6 +319,7 @@ ${p.item.prompt}`).join("\n\n")}`;
297
319
  try {
298
320
  ensureDir(PROCESSED_DIR);
299
321
  renameWithRetrySync(join(IN_PROGRESS_DIR, file), join(PROCESSED_DIR, `${status}-${file}`));
322
+ pruneProcessedDir();
300
323
  } catch {
301
324
  }
302
325
  }
@@ -24,6 +24,11 @@ import { request as httpsRequest } from 'node:https'
24
24
  import { createRequire } from 'node:module'
25
25
 
26
26
  const moduleRequire = createRequire(import.meta.url)
27
+ const {
28
+ isMixdogDebugEnabled,
29
+ pruneStalePluginDataLogSiblings,
30
+ DEFAULT_STALE_LOG_SIBLING_MAX,
31
+ } = moduleRequire('../../../lib/mixdog-debug.cjs')
27
32
 
28
33
  // IPC transport path. Windows uses a named pipe (`\\.\pipe\…`); Unix uses a
29
34
  // Unix domain socket under XDG_RUNTIME_DIR (or /tmp as fallback). Node's
@@ -57,9 +62,13 @@ const POLL_INTERVAL_MS = 2000
57
62
  const SUBAGENT_TIMEOUT_MS = 120_000
58
63
  const DEFAULT_DISPATCH_TIMEOUT_MS = 15_000
59
64
  const SESSION_START_MEMORY_DISPATCH_TIMEOUT_MS = 125_000
60
- const SESSION_START_TRACE_ENABLED =
61
- process.env.MIXDOG_DEBUG_SESSION_START === '1' ||
62
- process.env.MIXDOG_DEBUG_SESSION_START === 'true'
65
+ const MIXDOG_DEBUG_ENABLED = isMixdogDebugEnabled()
66
+ let _hookPipeLogsPruned = false
67
+
68
+ function hookPipeDebugStderr(line) {
69
+ if (!MIXDOG_DEBUG_ENABLED) return
70
+ try { process.stderr.write(line) } catch {}
71
+ }
63
72
 
64
73
  let _started = false
65
74
  let _server = null
@@ -82,13 +91,17 @@ function formatError(err) {
82
91
  }
83
92
 
84
93
  function traceSessionStart(message) {
85
- if (!SESSION_START_TRACE_ENABLED) return
94
+ if (!MIXDOG_DEBUG_ENABLED) return
86
95
  const line = `[${new Date().toISOString()}] [hook-pipe][session-start] ${message}\n`
87
96
  try { process.stderr.write(line) } catch {}
88
97
  try {
89
98
  const dataDir = process.env.CLAUDE_PLUGIN_DATA ||
90
99
  join(homedir(), '.claude', 'plugins', 'data', 'mixdog-trib-plugin')
91
100
  mkdirSync(dataDir, { recursive: true })
101
+ if (!_hookPipeLogsPruned) {
102
+ _hookPipeLogsPruned = true
103
+ pruneStalePluginDataLogSiblings(dataDir, DEFAULT_STALE_LOG_SIBLING_MAX)
104
+ }
92
105
  appendFileSync(join(dataDir, 'session-start.log'), line)
93
106
  } catch {}
94
107
  }
@@ -320,10 +333,10 @@ async function handlePreToolSubagent(payload) {
320
333
  }
321
334
  const route = routeMod.shouldRoutePermissionToDiscord()
322
335
  if (route.route !== 'discord') {
323
- process.stderr.write(`[hook-pipe] pre-tool-subagent discord-route=off agent=${agentIdRaw || 'unknown'} tool=${toolName} reason=${route.reason || 'inactive'}\n`)
336
+ hookPipeDebugStderr(`[hook-pipe] pre-tool-subagent discord-route=off agent=${agentIdRaw || 'unknown'} tool=${toolName} reason=${route.reason || 'inactive'}\n`)
324
337
  return null
325
338
  }
326
- process.stderr.write(`[hook-pipe] pre-tool-subagent discord-route=on agent=${agentIdRaw || 'unknown'} tool=${toolName}\n`)
339
+ hookPipeDebugStderr(`[hook-pipe] pre-tool-subagent discord-route=on agent=${agentIdRaw || 'unknown'} tool=${toolName}\n`)
327
340
 
328
341
  let getDiscordToken
329
342
  try {
@@ -715,7 +728,7 @@ export function startHookPipeServer() {
715
728
  _server.on('error', (err) => {
716
729
  const msg = String(err?.message || err || '')
717
730
  if (err?.code === 'EADDRINUSE' || msg.includes('EADDRINUSE') || msg.includes('Failed to listen')) {
718
- process.stderr.write(`[hook-pipe] ${PIPE_PATH} already owned by a peer daemon; standby for hook IPC\n`)
731
+ hookPipeDebugStderr(`[hook-pipe] ${PIPE_PATH} already owned by a peer daemon; standby for hook IPC\n`)
719
732
  _server = null
720
733
  _started = false
721
734
  return
@@ -727,7 +740,7 @@ export function startHookPipeServer() {
727
740
  try {
728
741
  _server.listen(PIPE_PATH, () => {
729
742
  _started = true
730
- process.stderr.write(`[hook-pipe] listening on ${PIPE_PATH}\n`)
743
+ hookPipeDebugStderr(`[hook-pipe] listening on ${PIPE_PATH}\n`)
731
744
  })
732
745
  } catch (err) {
733
746
  process.stderr.write(`[hook-pipe] listen failed: ${err?.message || err}\n`)
@@ -4,7 +4,7 @@ import { join } from "path";
4
4
  import { spawn, spawnSync, execSync } from "child_process";
5
5
  import { DATA_DIR, isInQuietWindow } from "./config.mjs";
6
6
  import { getWebhookAuthtoken } from "../../shared/config.mjs";
7
- import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, watch as fsWatch } from "fs";
7
+ import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, renameSync, watch as fsWatch } from "fs";
8
8
  import { randomUUID } from "crypto";
9
9
  const WEBHOOKS_DIR = join(DATA_DIR, "webhooks");
10
10
  const WEBHOOK_LOG = join(DATA_DIR, "webhook.log");
@@ -144,39 +144,160 @@ function loadEndpointConfig(name) {
144
144
  // then {status:"done"|"failed"|"dedup"}. Earlier fields (payloadPreview,
145
145
  // headersSummary) are kept on the first line only; later status updates
146
146
  // reference the same `id` and are merged latest-wins at read time.
147
+ const DELIVERY_INDEX_MAX_IDS = 2000;
148
+ const DELIVERY_LOG_MAX_LINES = 10_000;
149
+ /** @type {Map<string, Map<string, object>>} */
150
+ const _deliveryIndexByEndpoint = new Map();
151
+ /** @type {Set<string>} */
152
+ const _deliveryIndexWarmed = new Set();
153
+ /** @type {Map<string, number>} */
154
+ const _deliveryLogLineCountByEndpoint = new Map();
155
+ /** @type {Map<string, number>} distinct-id count at last warm/compaction; drives the redundancy-based compaction trigger. */
156
+ const _deliveryKeptCountByEndpoint = new Map();
147
157
  function _deliveriesPath(name) {
148
158
  return join(WEBHOOKS_DIR, name, "deliveries.jsonl");
149
159
  }
150
- function appendDelivery(name, entry) {
151
- try {
152
- const dir = join(WEBHOOKS_DIR, name);
153
- if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
154
- const line = JSON.stringify({ ts: new Date().toISOString(), ...entry }) + "\n";
155
- appendFileSync(_deliveriesPath(name), line);
156
- return true;
157
- } catch (err) {
158
- logWebhook(`${name}: deliveries append failed: ${err?.message ?? err}`);
159
- return false;
160
+ function _mergeDeliveryRows(prior, entry) {
161
+ return prior ? { ...prior, ...entry } : entry;
162
+ }
163
+ function _isBlockingDeliveryStatus(status) {
164
+ return status === "received" || status === "processing" || status === "done";
165
+ }
166
+ function _deliveryIndexFor(name) {
167
+ let map = _deliveryIndexByEndpoint.get(name);
168
+ if (!map) {
169
+ map = new Map();
170
+ _deliveryIndexByEndpoint.set(name, map);
160
171
  }
172
+ return map;
161
173
  }
162
- function readDeliveries(name) {
174
+ function _pruneDeliveryIndexMap(byId) {
175
+ if (byId.size <= DELIVERY_INDEX_MAX_IDS) return;
176
+ const rows = [...byId.entries()];
177
+ const blocking = rows.filter(([, e]) => _isBlockingDeliveryStatus(e.status));
178
+ const nonBlocking = rows.filter(([, e]) => !_isBlockingDeliveryStatus(e.status));
179
+ nonBlocking.sort((a, b) => String(b[1].ts || "").localeCompare(String(a[1].ts || "")));
180
+ const keepIds = new Set(blocking.map(([id]) => id));
181
+ for (const [id] of nonBlocking) {
182
+ if (keepIds.size >= DELIVERY_INDEX_MAX_IDS) break;
183
+ keepIds.add(id);
184
+ }
185
+ for (const id of byId.keys()) {
186
+ if (!keepIds.has(id)) byId.delete(id);
187
+ }
188
+ }
189
+ function _deliveryLogLineCount(name) {
190
+ return _deliveryLogLineCountByEndpoint.get(name) ?? 0;
191
+ }
192
+ function _setDeliveryLogLineCount(name, n) {
193
+ _deliveryLogLineCountByEndpoint.set(name, Math.max(0, n));
194
+ }
195
+ function _bumpDeliveryLogLineCount(name, delta = 1) {
196
+ _setDeliveryLogLineCount(name, _deliveryLogLineCount(name) + delta);
197
+ }
198
+ function _readDeliveriesFileMerged(name) {
163
199
  const p = _deliveriesPath(name);
164
- if (!existsSync(p)) return [];
165
200
  const byId = new Map();
201
+ let lineCount = 0;
202
+ if (!existsSync(p)) return { byId, lineCount };
166
203
  try {
167
204
  const raw = readFileSync(p, "utf8");
168
205
  for (const line of raw.split("\n")) {
169
206
  if (!line) continue;
207
+ lineCount++;
170
208
  try {
171
209
  const entry = JSON.parse(line);
172
- if (!entry || !entry.id) continue;
173
- const prior = byId.get(entry.id);
174
- const merged = prior ? { ...prior, ...entry } : entry;
175
- byId.set(entry.id, merged);
210
+ if (!entry?.id) continue;
211
+ byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
176
212
  } catch {}
177
213
  }
178
214
  } catch {}
179
- return [...byId.values()];
215
+ return { byId, lineCount };
216
+ }
217
+ function _ingestDeliveriesFileIntoIndex(name) {
218
+ const { byId: merged, lineCount } = _readDeliveriesFileMerged(name);
219
+ const byId = _deliveryIndexFor(name);
220
+ byId.clear();
221
+ for (const [id, row] of merged) byId.set(id, row);
222
+ _pruneDeliveryIndexMap(byId);
223
+ _setDeliveryLogLineCount(name, lineCount);
224
+ _deliveryKeptCountByEndpoint.set(name, merged.size);
225
+ }
226
+ function _ensureDeliveryIndex(name) {
227
+ if (_deliveryIndexWarmed.has(name)) return;
228
+ _deliveryIndexWarmed.add(name);
229
+ _ingestDeliveriesFileIntoIndex(name);
230
+ }
231
+ function _applyDeliveryEntryToIndex(name, entry) {
232
+ if (!entry?.id) return;
233
+ const byId = _deliveryIndexFor(name);
234
+ byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
235
+ _pruneDeliveryIndexMap(byId);
236
+ }
237
+ function _compactDeliveriesLogIfNeeded(name) {
238
+ // Redundancy-based trigger: compact only when the log holds meaningfully more
239
+ // lines than distinct ids (i.e. there are status-update rows to collapse).
240
+ // The threshold scales with the distinct-id count so an endpoint with many
241
+ // legitimate blocking ids does NOT re-compact on every append (which would
242
+ // re-read the whole log permanently once distinct > DELIVERY_LOG_MAX_LINES).
243
+ const kept = _deliveryKeptCountByEndpoint.get(name) ?? _deliveryIndexFor(name).size;
244
+ const threshold = Math.max(DELIVERY_LOG_MAX_LINES, kept * 2);
245
+ if (_deliveryLogLineCount(name) <= threshold) return;
246
+ const { byId: merged } = _readDeliveriesFileMerged(name);
247
+ const rows = [...merged.values()];
248
+ const blocking = rows.filter((e) => _isBlockingDeliveryStatus(e.status));
249
+ const nonBlocking = rows.filter((e) => !_isBlockingDeliveryStatus(e.status));
250
+ nonBlocking.sort((a, b) => String(b.ts || "").localeCompare(String(a.ts || "")));
251
+ const keep = new Map();
252
+ for (const e of blocking) keep.set(e.id, e);
253
+ for (const e of nonBlocking) {
254
+ if (keep.size >= DELIVERY_INDEX_MAX_IDS) break;
255
+ if (!keep.has(e.id)) keep.set(e.id, e);
256
+ }
257
+ const lines = [...keep.values()]
258
+ .sort((a, b) => String(a.ts || "").localeCompare(String(b.ts || "")))
259
+ .map((e) => JSON.stringify(e) + "\n")
260
+ .join("");
261
+ const p = _deliveriesPath(name);
262
+ const tmp = `${p}.compact-${process.pid}-${Date.now()}.tmp`;
263
+ try {
264
+ writeFileSync(tmp, lines);
265
+ renameSync(tmp, p);
266
+ // Refresh index + counters by RE-READING the post-rename file (not the
267
+ // pre-rename `keep` snapshot). The webhook daemon is the single writer and
268
+ // append+compact run synchronously in one process, so no append can
269
+ // interleave between the fresh read and the rename; re-reading keeps the
270
+ // warmed state exactly matching on-disk content.
271
+ _ingestDeliveriesFileIntoIndex(name);
272
+ } catch (err) {
273
+ logWebhook(`${name}: deliveries compact failed: ${err?.message ?? err}`);
274
+ try {
275
+ if (existsSync(tmp)) unlinkSync(tmp);
276
+ } catch {}
277
+ }
278
+ }
279
+ function appendDelivery(name, entry) {
280
+ try {
281
+ const dir = join(WEBHOOKS_DIR, name);
282
+ if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
283
+ const full = { ts: new Date().toISOString(), ...entry };
284
+ const line = JSON.stringify(full) + "\n";
285
+ appendFileSync(_deliveriesPath(name), line);
286
+ const wasWarmed = _deliveryIndexWarmed.has(name);
287
+ _ensureDeliveryIndex(name);
288
+ if (wasWarmed) _bumpDeliveryLogLineCount(name, 1);
289
+ _applyDeliveryEntryToIndex(name, full);
290
+ _compactDeliveriesLogIfNeeded(name);
291
+ return true;
292
+ } catch (err) {
293
+ logWebhook(`${name}: deliveries append failed: ${err?.message ?? err}`);
294
+ return false;
295
+ }
296
+ }
297
+ function readDeliveries(name) {
298
+ _ensureDeliveryIndex(name);
299
+ const byId = _deliveryIndexByEndpoint.get(name);
300
+ return byId ? [...byId.values()] : [];
180
301
  }
181
302
  // Dedup gate against a still-active claim or a successful prior delivery.
182
303
  // Only rows with status "received" (non-terminal claim) or "done"
@@ -186,12 +307,13 @@ function readDeliveries(name) {
186
307
  // scoping, every prior row would permanently dedup the id and stop
187
308
  // legit redelivery.
188
309
  function deliveryExists(name, id) {
189
- const list = readDeliveries(name);
190
310
  // "processing" must also dedup: a delegate dispatch in flight (up to
191
311
  // DISPATCH_TIMEOUT_MS = 10 min) would otherwise be duplicated by a
192
312
  // retried delivery of the same id while the first handler is still
193
313
  // running. Block on any non-terminal status.
194
- return list.some((e) => e.id === id && (e.status === "received" || e.status === "processing" || e.status === "done"));
314
+ _ensureDeliveryIndex(name);
315
+ const entry = _deliveryIndexFor(name).get(id);
316
+ return Boolean(entry && _isBlockingDeliveryStatus(entry.status));
195
317
  }
196
318
  function extractDeliveryId(headers) {
197
319
  return headers["x-github-delivery"]
@@ -545,9 +545,13 @@ async function _runCycle1Impl(db, config = {}, options = {}, dataDir = null) {
545
545
  }
546
546
  }
547
547
 
548
- // Fire every window at once: windowCount is bounded by fetchLimit / batchSize
549
- // (<= session_cap), so there is no need to throttle the fan-out.
550
- const sem = createSemaphore(Math.max(1, windows.length))
548
+ // Cap fan-out concurrency so a large batch (or a manual run) doesn't fire all
549
+ // window LLM calls at once and spike the provider / collide with the global
550
+ // agent-IPC limit. Small batches (<= cap) still run fully parallel.
551
+ const cycle1Concurrency = Math.max(1, Number(
552
+ config.cycle1_concurrency ?? config.concurrency ?? options.concurrency ?? options.maxConcurrent ?? 4,
553
+ ))
554
+ const sem = createSemaphore(Math.min(Math.max(1, windows.length), cycle1Concurrency))
551
555
  const settled = await Promise.allSettled(
552
556
  windows.map((rows, idx) => sem(() => {
553
557
  throwIfAborted(signal)
@@ -504,6 +504,32 @@ LEFT JOIN exact x ON x.id = c.id`
504
504
  const rootIdsForReturn = []
505
505
  const seen = new Set()
506
506
 
507
+ // Batch-resolve member-chunk roots in ONE query (was an N+1: a per-row SELECT
508
+ // inside the loop below). Collect the distinct in-scope chunk_root ids, fetch
509
+ // all matching roots at once, then resolve each member from rootById.
510
+ const memberRootIds = []
511
+ const memberRootSeen = new Set()
512
+ for (const { id } of filtered) {
513
+ const r0 = byId.get(id)
514
+ if (!r0 || r0.is_root === 1) continue
515
+ if (r0.chunk_root != null && r0.chunk_root !== r0.id) {
516
+ const rid = Number(r0.chunk_root)
517
+ if (!memberRootSeen.has(rid)) { memberRootSeen.add(rid); memberRootIds.push(rid) }
518
+ }
519
+ }
520
+ const rootById = new Map()
521
+ if (memberRootIds.length > 0) {
522
+ const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
523
+ const { rows: rootRows } = await recallReadQuery(
524
+ db,
525
+ `SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
526
+ element, category, summary, project_id, status, score, last_seen_at
527
+ FROM entries WHERE id = ANY($1::bigint[]) AND is_root = 1 ${rootScopeClause}`,
528
+ [memberRootIds, ...rootScopeParams],
529
+ )
530
+ for (const rr of rootRows) rootById.set(Number(rr.id), rr)
531
+ }
532
+
507
533
  for (const { id, rrf, retrievalScore } of filtered) {
508
534
  const row = byId.get(id)
509
535
  if (!row) continue
@@ -511,16 +537,7 @@ LEFT JOIN exact x ON x.id = c.id`
511
537
  if (row.is_root === 1) {
512
538
  targetRow = row
513
539
  } else if (row.chunk_root != null && row.chunk_root !== row.id) {
514
- // $1 = chunk_root id, scope param (if any) = $2
515
- const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
516
- const { rows: rootRows } = await recallReadQuery(
517
- db,
518
- `SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
519
- element, category, summary, project_id, status, score, last_seen_at
520
- FROM entries WHERE id = $1 AND is_root = 1 ${rootScopeClause}`,
521
- [row.chunk_root, ...rootScopeParams],
522
- )
523
- const r = rootRows[0]
540
+ const r = rootById.get(Number(row.chunk_root))
524
541
  if (!r) continue
525
542
  memberHitRootIds.add(r.id)
526
543
  targetRow = r
@@ -5,7 +5,11 @@
5
5
  * Calls Codex WebSocket endpoint via sendViaWebSocket with web_search server
6
6
  * tool. Model is config-driven (search.models.openai default 'gpt-5.4-mini').
7
7
  */
8
- import { OpenAIOAuthProvider, ensureLatestCodexModel } from '../../../agent/orchestrator/providers/openai-oauth.mjs'
8
+ import {
9
+ OpenAIOAuthProvider,
10
+ ensureLatestCodexModel,
11
+ codexModelSupportsServiceTier,
12
+ } from '../../../agent/orchestrator/providers/openai-oauth.mjs'
9
13
  import {
10
14
  OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
11
15
  buildOpenAISearchPrompt,
@@ -66,7 +70,7 @@ export async function searchViaOpenAIOAuth({
66
70
  parallel_tool_calls: false,
67
71
  tools: [buildOpenAIWebSearchTool({ site, type, locale, contextSize })],
68
72
  }
69
- if (fast === true) body.service_tier = 'priority'
73
+ if (fast === true && codexModelSupportsServiceTier(useModel, 'priority')) body.service_tier = 'priority'
70
74
  // Route through provider.send() (not sendViaWebSocket directly) so the search
71
75
  // request inherits the 401/403 force-refresh retry + HTTP/SSE fallback. A
72
76
  // stale token or unhealthy WebSocket then recovers instead of hard-failing.
@@ -5,6 +5,52 @@ const DEFAULT_CACHE_STATE = {
5
5
  entries: {},
6
6
  }
7
7
 
8
+ // Size bounds on top of TTL expiry so cache.local.json can't grow unbounded.
9
+ const MAX_CACHE_ENTRIES = 500
10
+ const MAX_CACHE_BYTES = 8 * 1024 * 1024
11
+
12
+ // Approximate serialized size of ONE entry (never the whole map), so insert and
13
+ // evict stay cheap. A running total on the state (`__approxBytes`) lets the byte
14
+ // cap be checked without re-serializing every entry on each set.
15
+ function approxEntryBytes(key, entry) {
16
+ try { return String(key).length + JSON.stringify(entry).length } catch { return String(key).length }
17
+ }
18
+ function cacheApproxBytes(state) {
19
+ if (typeof state.__approxBytes === 'number') return state.__approxBytes
20
+ let total = 0
21
+ for (const [k, e] of Object.entries(state.entries)) total += approxEntryBytes(k, e)
22
+ state.__approxBytes = total
23
+ return total
24
+ }
25
+ function removeCacheEntry(state, key) {
26
+ const e = state.entries[key]
27
+ if (e === undefined) return
28
+ if (typeof state.__approxBytes === 'number') state.__approxBytes -= approxEntryBytes(key, e)
29
+ delete state.entries[key]
30
+ }
31
+ function evictOldestCacheEntry(state) {
32
+ let oldestKey = null
33
+ let oldestAt = Infinity
34
+ for (const [k, e] of Object.entries(state.entries)) {
35
+ const at = e?.cachedAt ?? 0
36
+ if (at < oldestAt) { oldestAt = at; oldestKey = k }
37
+ }
38
+ if (oldestKey == null) return false
39
+ removeCacheEntry(state, oldestKey)
40
+ return true
41
+ }
42
+ // Best-effort: evict oldest entries until under both the count and byte caps.
43
+ function enforceCacheSizeBounds(state) {
44
+ try {
45
+ while (Object.keys(state.entries).length > MAX_CACHE_ENTRIES) {
46
+ if (!evictOldestCacheEntry(state)) break
47
+ }
48
+ while (cacheApproxBytes(state) > MAX_CACHE_BYTES && Object.keys(state.entries).length > 0) {
49
+ if (!evictOldestCacheEntry(state)) break
50
+ }
51
+ } catch { /* size bounding is best-effort */ }
52
+ }
53
+
8
54
  const FLUSH_DELAY_MS = 5000
9
55
 
10
56
  let cacheDirty = false
@@ -75,6 +121,8 @@ export function loadCacheState() {
75
121
  _instance = state
76
122
  activeCacheState = state
77
123
  pruneExpiredEntries(state)
124
+ cacheApproxBytes(state)
125
+ enforceCacheSizeBounds(state)
78
126
  return state
79
127
  }
80
128
 
@@ -90,7 +138,7 @@ export function getCachedEntry(state, key) {
90
138
  const entry = state.entries[key]
91
139
  if (!entry) return null
92
140
  if (entry.expiresAt && entry.expiresAt <= nowMs()) {
93
- delete state.entries[key]
141
+ removeCacheEntry(state, key)
94
142
  scheduleCacheFlush(state)
95
143
  return null
96
144
  }
@@ -99,11 +147,11 @@ export function getCachedEntry(state, key) {
99
147
 
100
148
  export function setCachedEntry(state, key, payload, ttlMs) {
101
149
  const cachedAt = nowMs()
102
- state.entries[key] = {
103
- cachedAt,
104
- expiresAt: cachedAt + ttlMs,
105
- payload,
106
- }
150
+ if (state.entries[key] !== undefined) removeCacheEntry(state, key)
151
+ const entry = { cachedAt, expiresAt: cachedAt + ttlMs, payload }
152
+ state.entries[key] = entry
153
+ if (typeof state.__approxBytes === 'number') state.__approxBytes += approxEntryBytes(key, entry)
154
+ enforceCacheSizeBounds(state)
107
155
  scheduleCacheFlush(state)
108
156
  return state.entries[key]
109
157
  }
@@ -121,7 +169,7 @@ function pruneExpiredEntries(state) {
121
169
  let dirty = false
122
170
  for (const [key, entry] of Object.entries(state.entries)) {
123
171
  if (entry?.expiresAt && entry.expiresAt <= current) {
124
- delete state.entries[key]
172
+ removeCacheEntry(state, key)
125
173
  dirty = true
126
174
  }
127
175
  }
@@ -299,7 +299,7 @@ export function getSearchApiKey(provider) {
299
299
  // exports keep working, then MIXDOG_AGENT_<P>_APIKEY, then the OS keychain.
300
300
  const AGENT_PROVIDER_ENV = Object.freeze({
301
301
  openai: 'OPENAI_API_KEY', anthropic: 'ANTHROPIC_API_KEY', gemini: 'GEMINI_API_KEY',
302
- deepseek: 'DEEPSEEK_API_KEY', xai: 'XAI_API_KEY', nvidia: 'NVIDIA_API_KEY',
302
+ deepseek: 'DEEPSEEK_API_KEY', xai: 'XAI_API_KEY',
303
303
  })
304
304
 
305
305
  // Last-resort env aliases honored AFTER the standard env / MIXDOG_AGENT_* /
@@ -17,7 +17,7 @@ import { getModelMetadataSync } from '../../agent/orchestrator/providers/model-c
17
17
  // count *including* the cached portion (inclusive). Anthropic reports the
18
18
  // uncached remainder only and bills cached_read / cached_write as separate
19
19
  // additive slots (additive). Cost and prompt-total math has to branch on this.
20
- // OpenAI-compatible direct providers (deepseek / nvidia / ollama / lmstudio)
20
+ // OpenAI-compatible direct providers (deepseek / ollama / lmstudio)
21
21
  // go through the OpenAI SDK and likewise report an inclusive prompt_tokens
22
22
  // with a separate cached-tokens detail — so they are inclusive too. Omitting
23
23
  // them bills the cached portion at the full input rate AND re-adds it as a
@@ -31,7 +31,7 @@ export function isInclusiveProvider(provider) {
31
31
  // usage rows — without it, cached tokens would be double-billed in the
32
32
  // cost fallback and prompt totals.
33
33
  return p.includes('openai') || p.includes('codex') || p.includes('gemini') || p.includes('google') || p.includes('xai') || p.includes('grok')
34
- || p.includes('deepseek') || p.includes('nvidia') || p.includes('ollama') || p.includes('lmstudio') || p.includes('groq') || p.includes('openrouter');
34
+ || p.includes('deepseek') || p.includes('ollama') || p.includes('lmstudio') || p.includes('groq') || p.includes('openrouter');
35
35
  }
36
36
 
37
37
  /**