mixdog 0.7.8 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +5 -2
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +40 -0
- package/README.md +198 -251
- package/bin/statusline-launcher.mjs +5 -1
- package/bin/statusline-lib.mjs +14 -6
- package/bin/statusline.mjs +14 -6
- package/hooks/lib/settings-loader.cjs +4 -3
- package/hooks/pre-tool-subagent.cjs +7 -2
- package/hooks/session-start.cjs +52 -24
- package/lib/mixdog-debug.cjs +163 -0
- package/native/prebuilt/linux-aarch64/mixdog-shim +0 -0
- package/native/prebuilt/linux-x86_64/mixdog-shim +0 -0
- package/native/prebuilt/macos-aarch64/mixdog-shim +0 -0
- package/native/prebuilt/macos-x86_64/mixdog-shim +0 -0
- package/native/prebuilt/windows-x86_64/mixdog-shim.exe +0 -0
- package/package.json +1 -1
- package/scripts/builtin-utils-smoke.mjs +14 -8
- package/scripts/bump.mjs +80 -0
- package/scripts/doctor.mjs +8 -3
- package/scripts/mutation-io-smoke.mjs +17 -1
- package/scripts/openai-oauth-catalog-smoke.mjs +53 -0
- package/scripts/permission-eval-smoke.mjs +18 -1
- package/scripts/statusline-launcher-smoke.mjs +2 -2
- package/scripts/webhook-selfheal-smoke.mjs +1 -3
- package/server-main.mjs +57 -3
- package/setup/config-merge.mjs +0 -1
- package/setup/install.mjs +241 -51
- package/setup/mixdog-cli.mjs +30 -3
- package/setup/setup-server.mjs +21 -33
- package/setup/setup.html +46 -11
- package/setup/tui.mjs +35 -316
- package/src/agent/orchestrator/config.mjs +0 -1
- package/src/agent/orchestrator/providers/anthropic-oauth.mjs +2 -5
- package/src/agent/orchestrator/providers/anthropic.mjs +243 -86
- package/src/agent/orchestrator/providers/gemini.mjs +386 -31
- package/src/agent/orchestrator/providers/grok-oauth.mjs +2 -5
- package/src/agent/orchestrator/providers/model-catalog.mjs +146 -13
- package/src/agent/orchestrator/providers/openai-compat-stream.mjs +366 -0
- package/src/agent/orchestrator/providers/openai-compat.mjs +74 -30
- package/src/agent/orchestrator/providers/openai-oauth-ws.mjs +2 -1
- package/src/agent/orchestrator/providers/openai-oauth.mjs +66 -13
- package/src/agent/orchestrator/providers/openai-ws.mjs +23 -0
- package/src/agent/orchestrator/session/manager.mjs +18 -4
- package/src/agent/orchestrator/stall-policy.mjs +6 -0
- package/src/agent/orchestrator/tools/builtin/native-edit-runner.mjs +29 -8
- package/src/agent/orchestrator/tools/graph-manifest.json +11 -11
- package/src/agent/orchestrator/tools/patch-manifest.json +11 -11
- package/src/channels/index.mjs +27 -8
- package/src/channels/lib/event-queue.mjs +24 -1
- package/src/channels/lib/hook-pipe-server.mjs +21 -8
- package/src/channels/lib/webhook.mjs +142 -20
- package/src/memory/lib/memory-cycle1.mjs +7 -3
- package/src/memory/lib/memory-recall-store.mjs +27 -10
- package/src/search/lib/backends/openai-oauth.mjs +6 -2
- package/src/search/lib/cache.mjs +55 -7
- package/src/shared/config.mjs +1 -1
- package/src/shared/llm/cost.mjs +2 -2
- package/src/shared/open-url.mjs +37 -0
- package/src/shared/seed.mjs +20 -3
- package/src/shared/user-data-guard.mjs +3 -1
- package/scripts/test-config-rmw-restore.mjs +0 -122
- package/setup/wizard.mjs +0 -696
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "0.
|
|
2
|
+
"version": "0.7.12",
|
|
3
3
|
"_comment": "Rewritten by .github/workflows/patch-release.yml on each tagged release. assets maps platformKey (process.platform-process.arch, e.g. win32-x64, linux-x64, darwin-arm64) to { url, sha256 } of the mixdog-patch binary on the GitHub release. A local cargo build under native/mixdog-patch/target/release always takes precedence; otherwise the binary is fetched per this manifest into the data dir (apply is native-only — no JS apply engine).",
|
|
4
4
|
"assets": {
|
|
5
5
|
"darwin-arm64": {
|
|
6
|
-
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.
|
|
7
|
-
"sha256": "
|
|
6
|
+
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-darwin-arm64",
|
|
7
|
+
"sha256": "836a0b60a443b0a6a8c1bbe24d15a79ed70ee92c2f0fbc05374c4e9ed2536415"
|
|
8
8
|
},
|
|
9
9
|
"darwin-x64": {
|
|
10
|
-
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.
|
|
11
|
-
"sha256": "
|
|
10
|
+
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-darwin-x64",
|
|
11
|
+
"sha256": "cab10c4e1e8b72d3958241dffdff764712ed74f4861d105bafa8258961215c98"
|
|
12
12
|
},
|
|
13
13
|
"linux-arm64": {
|
|
14
|
-
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.
|
|
15
|
-
"sha256": "
|
|
14
|
+
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-linux-arm64",
|
|
15
|
+
"sha256": "a90c32ce3417a7d853f2723f82f3613cf2cd030fe885cf710cfc9f8e4b193264"
|
|
16
16
|
},
|
|
17
17
|
"linux-x64": {
|
|
18
|
-
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.
|
|
19
|
-
"sha256": "
|
|
18
|
+
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-linux-x64",
|
|
19
|
+
"sha256": "0fea40ab98acd35bfb47515756024d1882a2abbaddce8a0b51642d20ac405577"
|
|
20
20
|
},
|
|
21
21
|
"win32-x64": {
|
|
22
|
-
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.
|
|
23
|
-
"sha256": "
|
|
22
|
+
"url": "https://github.com/trib-plugin/mixdog/releases/download/v0.7.12/mixdog-patch-win32-x64.exe",
|
|
23
|
+
"sha256": "f8a74fb9bb7bf333fa441b800b76b28d83a2a7b4795e4a769122faf47a59d1f1"
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
}
|
package/src/channels/index.mjs
CHANGED
|
@@ -183,10 +183,18 @@ const _bootLogEarly = path.join(
|
|
|
183
183
|
process.env.CLAUDE_PLUGIN_DATA || path.join(os.tmpdir(), "mixdog"),
|
|
184
184
|
"boot.log"
|
|
185
185
|
);
|
|
186
|
+
const {
|
|
187
|
+
isMixdogDebugEnabled: isMixdogDebug,
|
|
188
|
+
pruneStalePluginDataLogSiblings,
|
|
189
|
+
appendSessionStartCriticalLog,
|
|
190
|
+
DEFAULT_STALE_LOG_SIBLING_MAX,
|
|
191
|
+
} = _require("../../lib/mixdog-debug.cjs");
|
|
186
192
|
// One-shot log rotation at worker boot (10 MB threshold, .1 suffix overwrite).
|
|
187
|
-
|
|
188
|
-
fs.
|
|
193
|
+
if (isMixdogDebug()) {
|
|
194
|
+
try { if (fs.statSync(_bootLogEarly).size > 10 * 1024 * 1024) fs.renameSync(_bootLogEarly, _bootLogEarly + '.1') } catch {}
|
|
195
|
+
fs.appendFileSync(_bootLogEarly, `[${localTimestamp()}] bootstrap start pid=${process.pid}
|
|
189
196
|
`);
|
|
197
|
+
}
|
|
190
198
|
const _bootLog = path.join(DATA_DIR, "boot.log");
|
|
191
199
|
let config = loadConfig();
|
|
192
200
|
let backend = createBackend(config);
|
|
@@ -238,6 +246,11 @@ try {
|
|
|
238
246
|
try { if (_now - fs.statSync(_p).mtimeMs > _STALE_SESSION_TTL_MS) fs.unlinkSync(_p); } catch {}
|
|
239
247
|
}
|
|
240
248
|
} catch {}
|
|
249
|
+
// Count-based cap: drop oldest *.log siblings when plugin-data accumulates
|
|
250
|
+
// hundreds of per-process files (doctor warns above 300).
|
|
251
|
+
try {
|
|
252
|
+
pruneStalePluginDataLogSiblings(DATA_DIR, DEFAULT_STALE_LOG_SIBLING_MAX);
|
|
253
|
+
} catch {}
|
|
241
254
|
|
|
242
255
|
// ── Buffered drop-trace writer (channels/index) ──────────────────────────────
|
|
243
256
|
// Flushes every 1 s OR when buffer reaches 64 KB — whichever fires first.
|
|
@@ -2030,8 +2043,12 @@ backend.onInteraction = (interaction) => {
|
|
|
2030
2043
|
const [, uuid, action] = match;
|
|
2031
2044
|
const access = config.access;
|
|
2032
2045
|
if (!access) {
|
|
2033
|
-
|
|
2034
|
-
|
|
2046
|
+
const _permDropLine = `[${localTimestamp()}] perm interaction dropped: no access config\n`;
|
|
2047
|
+
if (isMixdogDebug()) {
|
|
2048
|
+
fs.appendFileSync(_bootLog, _permDropLine);
|
|
2049
|
+
} else {
|
|
2050
|
+
appendSessionStartCriticalLog(DATA_DIR, `[channels] ${_permDropLine}`);
|
|
2051
|
+
}
|
|
2035
2052
|
return;
|
|
2036
2053
|
}
|
|
2037
2054
|
if (access.allowFrom?.length > 0 && !access.allowFrom.includes(interaction.userId)) {
|
|
@@ -3016,12 +3033,14 @@ async function stop() {
|
|
|
3016
3033
|
return false;
|
|
3017
3034
|
};
|
|
3018
3035
|
_channelFlagDetected = detectChannelFlag();
|
|
3019
|
-
|
|
3020
|
-
`);
|
|
3036
|
+
if (isMixdogDebug()) {
|
|
3037
|
+
fs.appendFileSync(_bootLog, `[${localTimestamp()}] channelFlag: ${_channelFlagDetected}\n`);
|
|
3038
|
+
if (_channelFlagDetected) {
|
|
3039
|
+
fs.appendFileSync(_bootLog, `[${localTimestamp()}] channel mode detected — bridge auto-activated\n`);
|
|
3040
|
+
}
|
|
3041
|
+
}
|
|
3021
3042
|
if (_channelFlagDetected) {
|
|
3022
3043
|
channelBridgeActive = true;
|
|
3023
|
-
fs.appendFileSync(_bootLog, `[${localTimestamp()}] channel mode detected \u2014 bridge auto-activated
|
|
3024
|
-
`);
|
|
3025
3044
|
}
|
|
3026
3045
|
writeBridgeState(channelBridgeActive);
|
|
3027
3046
|
const previousOwner = readActiveInstance();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readdirSync, readFileSync, existsSync as fsExistsSync } from "fs";
|
|
1
|
+
import { readdirSync, readFileSync, existsSync as fsExistsSync, statSync, unlinkSync } from "fs";
|
|
2
2
|
import { join } from "path";
|
|
3
3
|
import { DATA_DIR } from "./config.mjs";
|
|
4
4
|
import { ensureDir } from "./state-file.mjs";
|
|
@@ -7,6 +7,27 @@ import { renameWithRetrySync, writeJsonAtomicSync } from "../../shared/atomic-fi
|
|
|
7
7
|
const QUEUE_DIR = join(DATA_DIR, "events", "queue");
|
|
8
8
|
const IN_PROGRESS_DIR = join(DATA_DIR, "events", "in-progress");
|
|
9
9
|
const PROCESSED_DIR = join(DATA_DIR, "events", "processed");
|
|
10
|
+
const PROCESSED_DIR_MAX_ENTRIES = 200;
|
|
11
|
+
function pruneProcessedDir() {
|
|
12
|
+
try {
|
|
13
|
+
if (!fsExistsSync(PROCESSED_DIR)) return;
|
|
14
|
+
const names = readdirSync(PROCESSED_DIR);
|
|
15
|
+
if (names.length <= PROCESSED_DIR_MAX_ENTRIES) return;
|
|
16
|
+
const ranked = [];
|
|
17
|
+
for (const name of names) {
|
|
18
|
+
try {
|
|
19
|
+
const st = statSync(join(PROCESSED_DIR, name));
|
|
20
|
+
ranked.push({ name, mtime: st.mtimeMs });
|
|
21
|
+
} catch {}
|
|
22
|
+
}
|
|
23
|
+
ranked.sort((a, b) => b.mtime - a.mtime);
|
|
24
|
+
for (let i = PROCESSED_DIR_MAX_ENTRIES; i < ranked.length; i++) {
|
|
25
|
+
try {
|
|
26
|
+
unlinkSync(join(PROCESSED_DIR, ranked[i].name));
|
|
27
|
+
} catch {}
|
|
28
|
+
}
|
|
29
|
+
} catch {}
|
|
30
|
+
}
|
|
10
31
|
function finiteInt(value, { min, max, def }) {
|
|
11
32
|
const n = Number(value);
|
|
12
33
|
if (!Number.isFinite(n)) return def;
|
|
@@ -290,6 +311,7 @@ ${p.item.prompt}`).join("\n\n")}`;
|
|
|
290
311
|
const fromQueue = join(QUEUE_DIR, file);
|
|
291
312
|
const src = this.existsSync(fromInProgress) ? fromInProgress : fromQueue;
|
|
292
313
|
renameWithRetrySync(src, join(PROCESSED_DIR, `${status}-${file}`));
|
|
314
|
+
pruneProcessedDir();
|
|
293
315
|
} catch {
|
|
294
316
|
}
|
|
295
317
|
}
|
|
@@ -297,6 +319,7 @@ ${p.item.prompt}`).join("\n\n")}`;
|
|
|
297
319
|
try {
|
|
298
320
|
ensureDir(PROCESSED_DIR);
|
|
299
321
|
renameWithRetrySync(join(IN_PROGRESS_DIR, file), join(PROCESSED_DIR, `${status}-${file}`));
|
|
322
|
+
pruneProcessedDir();
|
|
300
323
|
} catch {
|
|
301
324
|
}
|
|
302
325
|
}
|
|
@@ -24,6 +24,11 @@ import { request as httpsRequest } from 'node:https'
|
|
|
24
24
|
import { createRequire } from 'node:module'
|
|
25
25
|
|
|
26
26
|
const moduleRequire = createRequire(import.meta.url)
|
|
27
|
+
const {
|
|
28
|
+
isMixdogDebugEnabled,
|
|
29
|
+
pruneStalePluginDataLogSiblings,
|
|
30
|
+
DEFAULT_STALE_LOG_SIBLING_MAX,
|
|
31
|
+
} = moduleRequire('../../../lib/mixdog-debug.cjs')
|
|
27
32
|
|
|
28
33
|
// IPC transport path. Windows uses a named pipe (`\\.\pipe\…`); Unix uses a
|
|
29
34
|
// Unix domain socket under XDG_RUNTIME_DIR (or /tmp as fallback). Node's
|
|
@@ -57,9 +62,13 @@ const POLL_INTERVAL_MS = 2000
|
|
|
57
62
|
const SUBAGENT_TIMEOUT_MS = 120_000
|
|
58
63
|
const DEFAULT_DISPATCH_TIMEOUT_MS = 15_000
|
|
59
64
|
const SESSION_START_MEMORY_DISPATCH_TIMEOUT_MS = 125_000
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
const MIXDOG_DEBUG_ENABLED = isMixdogDebugEnabled()
|
|
66
|
+
let _hookPipeLogsPruned = false
|
|
67
|
+
|
|
68
|
+
function hookPipeDebugStderr(line) {
|
|
69
|
+
if (!MIXDOG_DEBUG_ENABLED) return
|
|
70
|
+
try { process.stderr.write(line) } catch {}
|
|
71
|
+
}
|
|
63
72
|
|
|
64
73
|
let _started = false
|
|
65
74
|
let _server = null
|
|
@@ -82,13 +91,17 @@ function formatError(err) {
|
|
|
82
91
|
}
|
|
83
92
|
|
|
84
93
|
function traceSessionStart(message) {
|
|
85
|
-
if (!
|
|
94
|
+
if (!MIXDOG_DEBUG_ENABLED) return
|
|
86
95
|
const line = `[${new Date().toISOString()}] [hook-pipe][session-start] ${message}\n`
|
|
87
96
|
try { process.stderr.write(line) } catch {}
|
|
88
97
|
try {
|
|
89
98
|
const dataDir = process.env.CLAUDE_PLUGIN_DATA ||
|
|
90
99
|
join(homedir(), '.claude', 'plugins', 'data', 'mixdog-trib-plugin')
|
|
91
100
|
mkdirSync(dataDir, { recursive: true })
|
|
101
|
+
if (!_hookPipeLogsPruned) {
|
|
102
|
+
_hookPipeLogsPruned = true
|
|
103
|
+
pruneStalePluginDataLogSiblings(dataDir, DEFAULT_STALE_LOG_SIBLING_MAX)
|
|
104
|
+
}
|
|
92
105
|
appendFileSync(join(dataDir, 'session-start.log'), line)
|
|
93
106
|
} catch {}
|
|
94
107
|
}
|
|
@@ -320,10 +333,10 @@ async function handlePreToolSubagent(payload) {
|
|
|
320
333
|
}
|
|
321
334
|
const route = routeMod.shouldRoutePermissionToDiscord()
|
|
322
335
|
if (route.route !== 'discord') {
|
|
323
|
-
|
|
336
|
+
hookPipeDebugStderr(`[hook-pipe] pre-tool-subagent discord-route=off agent=${agentIdRaw || 'unknown'} tool=${toolName} reason=${route.reason || 'inactive'}\n`)
|
|
324
337
|
return null
|
|
325
338
|
}
|
|
326
|
-
|
|
339
|
+
hookPipeDebugStderr(`[hook-pipe] pre-tool-subagent discord-route=on agent=${agentIdRaw || 'unknown'} tool=${toolName}\n`)
|
|
327
340
|
|
|
328
341
|
let getDiscordToken
|
|
329
342
|
try {
|
|
@@ -715,7 +728,7 @@ export function startHookPipeServer() {
|
|
|
715
728
|
_server.on('error', (err) => {
|
|
716
729
|
const msg = String(err?.message || err || '')
|
|
717
730
|
if (err?.code === 'EADDRINUSE' || msg.includes('EADDRINUSE') || msg.includes('Failed to listen')) {
|
|
718
|
-
|
|
731
|
+
hookPipeDebugStderr(`[hook-pipe] ${PIPE_PATH} already owned by a peer daemon; standby for hook IPC\n`)
|
|
719
732
|
_server = null
|
|
720
733
|
_started = false
|
|
721
734
|
return
|
|
@@ -727,7 +740,7 @@ export function startHookPipeServer() {
|
|
|
727
740
|
try {
|
|
728
741
|
_server.listen(PIPE_PATH, () => {
|
|
729
742
|
_started = true
|
|
730
|
-
|
|
743
|
+
hookPipeDebugStderr(`[hook-pipe] listening on ${PIPE_PATH}\n`)
|
|
731
744
|
})
|
|
732
745
|
} catch (err) {
|
|
733
746
|
process.stderr.write(`[hook-pipe] listen failed: ${err?.message || err}\n`)
|
|
@@ -4,7 +4,7 @@ import { join } from "path";
|
|
|
4
4
|
import { spawn, spawnSync, execSync } from "child_process";
|
|
5
5
|
import { DATA_DIR, isInQuietWindow } from "./config.mjs";
|
|
6
6
|
import { getWebhookAuthtoken } from "../../shared/config.mjs";
|
|
7
|
-
import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, watch as fsWatch } from "fs";
|
|
7
|
+
import { appendFileSync, readFileSync, readdirSync, mkdirSync, writeFileSync, unlinkSync, statSync, existsSync, renameSync, watch as fsWatch } from "fs";
|
|
8
8
|
import { randomUUID } from "crypto";
|
|
9
9
|
const WEBHOOKS_DIR = join(DATA_DIR, "webhooks");
|
|
10
10
|
const WEBHOOK_LOG = join(DATA_DIR, "webhook.log");
|
|
@@ -144,39 +144,160 @@ function loadEndpointConfig(name) {
|
|
|
144
144
|
// then {status:"done"|"failed"|"dedup"}. Earlier fields (payloadPreview,
|
|
145
145
|
// headersSummary) are kept on the first line only; later status updates
|
|
146
146
|
// reference the same `id` and are merged latest-wins at read time.
|
|
147
|
+
const DELIVERY_INDEX_MAX_IDS = 2000;
|
|
148
|
+
const DELIVERY_LOG_MAX_LINES = 10_000;
|
|
149
|
+
/** @type {Map<string, Map<string, object>>} */
|
|
150
|
+
const _deliveryIndexByEndpoint = new Map();
|
|
151
|
+
/** @type {Set<string>} */
|
|
152
|
+
const _deliveryIndexWarmed = new Set();
|
|
153
|
+
/** @type {Map<string, number>} */
|
|
154
|
+
const _deliveryLogLineCountByEndpoint = new Map();
|
|
155
|
+
/** @type {Map<string, number>} distinct-id count at last warm/compaction; drives the redundancy-based compaction trigger. */
|
|
156
|
+
const _deliveryKeptCountByEndpoint = new Map();
|
|
147
157
|
function _deliveriesPath(name) {
|
|
148
158
|
return join(WEBHOOKS_DIR, name, "deliveries.jsonl");
|
|
149
159
|
}
|
|
150
|
-
function
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
+
function _mergeDeliveryRows(prior, entry) {
|
|
161
|
+
return prior ? { ...prior, ...entry } : entry;
|
|
162
|
+
}
|
|
163
|
+
function _isBlockingDeliveryStatus(status) {
|
|
164
|
+
return status === "received" || status === "processing" || status === "done";
|
|
165
|
+
}
|
|
166
|
+
function _deliveryIndexFor(name) {
|
|
167
|
+
let map = _deliveryIndexByEndpoint.get(name);
|
|
168
|
+
if (!map) {
|
|
169
|
+
map = new Map();
|
|
170
|
+
_deliveryIndexByEndpoint.set(name, map);
|
|
160
171
|
}
|
|
172
|
+
return map;
|
|
161
173
|
}
|
|
162
|
-
function
|
|
174
|
+
function _pruneDeliveryIndexMap(byId) {
|
|
175
|
+
if (byId.size <= DELIVERY_INDEX_MAX_IDS) return;
|
|
176
|
+
const rows = [...byId.entries()];
|
|
177
|
+
const blocking = rows.filter(([, e]) => _isBlockingDeliveryStatus(e.status));
|
|
178
|
+
const nonBlocking = rows.filter(([, e]) => !_isBlockingDeliveryStatus(e.status));
|
|
179
|
+
nonBlocking.sort((a, b) => String(b[1].ts || "").localeCompare(String(a[1].ts || "")));
|
|
180
|
+
const keepIds = new Set(blocking.map(([id]) => id));
|
|
181
|
+
for (const [id] of nonBlocking) {
|
|
182
|
+
if (keepIds.size >= DELIVERY_INDEX_MAX_IDS) break;
|
|
183
|
+
keepIds.add(id);
|
|
184
|
+
}
|
|
185
|
+
for (const id of byId.keys()) {
|
|
186
|
+
if (!keepIds.has(id)) byId.delete(id);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
function _deliveryLogLineCount(name) {
|
|
190
|
+
return _deliveryLogLineCountByEndpoint.get(name) ?? 0;
|
|
191
|
+
}
|
|
192
|
+
function _setDeliveryLogLineCount(name, n) {
|
|
193
|
+
_deliveryLogLineCountByEndpoint.set(name, Math.max(0, n));
|
|
194
|
+
}
|
|
195
|
+
function _bumpDeliveryLogLineCount(name, delta = 1) {
|
|
196
|
+
_setDeliveryLogLineCount(name, _deliveryLogLineCount(name) + delta);
|
|
197
|
+
}
|
|
198
|
+
function _readDeliveriesFileMerged(name) {
|
|
163
199
|
const p = _deliveriesPath(name);
|
|
164
|
-
if (!existsSync(p)) return [];
|
|
165
200
|
const byId = new Map();
|
|
201
|
+
let lineCount = 0;
|
|
202
|
+
if (!existsSync(p)) return { byId, lineCount };
|
|
166
203
|
try {
|
|
167
204
|
const raw = readFileSync(p, "utf8");
|
|
168
205
|
for (const line of raw.split("\n")) {
|
|
169
206
|
if (!line) continue;
|
|
207
|
+
lineCount++;
|
|
170
208
|
try {
|
|
171
209
|
const entry = JSON.parse(line);
|
|
172
|
-
if (!entry
|
|
173
|
-
|
|
174
|
-
const merged = prior ? { ...prior, ...entry } : entry;
|
|
175
|
-
byId.set(entry.id, merged);
|
|
210
|
+
if (!entry?.id) continue;
|
|
211
|
+
byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
|
|
176
212
|
} catch {}
|
|
177
213
|
}
|
|
178
214
|
} catch {}
|
|
179
|
-
return
|
|
215
|
+
return { byId, lineCount };
|
|
216
|
+
}
|
|
217
|
+
function _ingestDeliveriesFileIntoIndex(name) {
|
|
218
|
+
const { byId: merged, lineCount } = _readDeliveriesFileMerged(name);
|
|
219
|
+
const byId = _deliveryIndexFor(name);
|
|
220
|
+
byId.clear();
|
|
221
|
+
for (const [id, row] of merged) byId.set(id, row);
|
|
222
|
+
_pruneDeliveryIndexMap(byId);
|
|
223
|
+
_setDeliveryLogLineCount(name, lineCount);
|
|
224
|
+
_deliveryKeptCountByEndpoint.set(name, merged.size);
|
|
225
|
+
}
|
|
226
|
+
function _ensureDeliveryIndex(name) {
|
|
227
|
+
if (_deliveryIndexWarmed.has(name)) return;
|
|
228
|
+
_deliveryIndexWarmed.add(name);
|
|
229
|
+
_ingestDeliveriesFileIntoIndex(name);
|
|
230
|
+
}
|
|
231
|
+
function _applyDeliveryEntryToIndex(name, entry) {
|
|
232
|
+
if (!entry?.id) return;
|
|
233
|
+
const byId = _deliveryIndexFor(name);
|
|
234
|
+
byId.set(entry.id, _mergeDeliveryRows(byId.get(entry.id), entry));
|
|
235
|
+
_pruneDeliveryIndexMap(byId);
|
|
236
|
+
}
|
|
237
|
+
function _compactDeliveriesLogIfNeeded(name) {
|
|
238
|
+
// Redundancy-based trigger: compact only when the log holds meaningfully more
|
|
239
|
+
// lines than distinct ids (i.e. there are status-update rows to collapse).
|
|
240
|
+
// The threshold scales with the distinct-id count so an endpoint with many
|
|
241
|
+
// legitimate blocking ids does NOT re-compact on every append (which would
|
|
242
|
+
// re-read the whole log permanently once distinct > DELIVERY_LOG_MAX_LINES).
|
|
243
|
+
const kept = _deliveryKeptCountByEndpoint.get(name) ?? _deliveryIndexFor(name).size;
|
|
244
|
+
const threshold = Math.max(DELIVERY_LOG_MAX_LINES, kept * 2);
|
|
245
|
+
if (_deliveryLogLineCount(name) <= threshold) return;
|
|
246
|
+
const { byId: merged } = _readDeliveriesFileMerged(name);
|
|
247
|
+
const rows = [...merged.values()];
|
|
248
|
+
const blocking = rows.filter((e) => _isBlockingDeliveryStatus(e.status));
|
|
249
|
+
const nonBlocking = rows.filter((e) => !_isBlockingDeliveryStatus(e.status));
|
|
250
|
+
nonBlocking.sort((a, b) => String(b.ts || "").localeCompare(String(a.ts || "")));
|
|
251
|
+
const keep = new Map();
|
|
252
|
+
for (const e of blocking) keep.set(e.id, e);
|
|
253
|
+
for (const e of nonBlocking) {
|
|
254
|
+
if (keep.size >= DELIVERY_INDEX_MAX_IDS) break;
|
|
255
|
+
if (!keep.has(e.id)) keep.set(e.id, e);
|
|
256
|
+
}
|
|
257
|
+
const lines = [...keep.values()]
|
|
258
|
+
.sort((a, b) => String(a.ts || "").localeCompare(String(b.ts || "")))
|
|
259
|
+
.map((e) => JSON.stringify(e) + "\n")
|
|
260
|
+
.join("");
|
|
261
|
+
const p = _deliveriesPath(name);
|
|
262
|
+
const tmp = `${p}.compact-${process.pid}-${Date.now()}.tmp`;
|
|
263
|
+
try {
|
|
264
|
+
writeFileSync(tmp, lines);
|
|
265
|
+
renameSync(tmp, p);
|
|
266
|
+
// Refresh index + counters by RE-READING the post-rename file (not the
|
|
267
|
+
// pre-rename `keep` snapshot). The webhook daemon is the single writer and
|
|
268
|
+
// append+compact run synchronously in one process, so no append can
|
|
269
|
+
// interleave between the fresh read and the rename; re-reading keeps the
|
|
270
|
+
// warmed state exactly matching on-disk content.
|
|
271
|
+
_ingestDeliveriesFileIntoIndex(name);
|
|
272
|
+
} catch (err) {
|
|
273
|
+
logWebhook(`${name}: deliveries compact failed: ${err?.message ?? err}`);
|
|
274
|
+
try {
|
|
275
|
+
if (existsSync(tmp)) unlinkSync(tmp);
|
|
276
|
+
} catch {}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
function appendDelivery(name, entry) {
|
|
280
|
+
try {
|
|
281
|
+
const dir = join(WEBHOOKS_DIR, name);
|
|
282
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
283
|
+
const full = { ts: new Date().toISOString(), ...entry };
|
|
284
|
+
const line = JSON.stringify(full) + "\n";
|
|
285
|
+
appendFileSync(_deliveriesPath(name), line);
|
|
286
|
+
const wasWarmed = _deliveryIndexWarmed.has(name);
|
|
287
|
+
_ensureDeliveryIndex(name);
|
|
288
|
+
if (wasWarmed) _bumpDeliveryLogLineCount(name, 1);
|
|
289
|
+
_applyDeliveryEntryToIndex(name, full);
|
|
290
|
+
_compactDeliveriesLogIfNeeded(name);
|
|
291
|
+
return true;
|
|
292
|
+
} catch (err) {
|
|
293
|
+
logWebhook(`${name}: deliveries append failed: ${err?.message ?? err}`);
|
|
294
|
+
return false;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
function readDeliveries(name) {
|
|
298
|
+
_ensureDeliveryIndex(name);
|
|
299
|
+
const byId = _deliveryIndexByEndpoint.get(name);
|
|
300
|
+
return byId ? [...byId.values()] : [];
|
|
180
301
|
}
|
|
181
302
|
// Dedup gate against a still-active claim or a successful prior delivery.
|
|
182
303
|
// Only rows with status "received" (non-terminal claim) or "done"
|
|
@@ -186,12 +307,13 @@ function readDeliveries(name) {
|
|
|
186
307
|
// scoping, every prior row would permanently dedup the id and stop
|
|
187
308
|
// legit redelivery.
|
|
188
309
|
function deliveryExists(name, id) {
|
|
189
|
-
const list = readDeliveries(name);
|
|
190
310
|
// "processing" must also dedup: a delegate dispatch in flight (up to
|
|
191
311
|
// DISPATCH_TIMEOUT_MS = 10 min) would otherwise be duplicated by a
|
|
192
312
|
// retried delivery of the same id while the first handler is still
|
|
193
313
|
// running. Block on any non-terminal status.
|
|
194
|
-
|
|
314
|
+
_ensureDeliveryIndex(name);
|
|
315
|
+
const entry = _deliveryIndexFor(name).get(id);
|
|
316
|
+
return Boolean(entry && _isBlockingDeliveryStatus(entry.status));
|
|
195
317
|
}
|
|
196
318
|
function extractDeliveryId(headers) {
|
|
197
319
|
return headers["x-github-delivery"]
|
|
@@ -545,9 +545,13 @@ async function _runCycle1Impl(db, config = {}, options = {}, dataDir = null) {
|
|
|
545
545
|
}
|
|
546
546
|
}
|
|
547
547
|
|
|
548
|
-
//
|
|
549
|
-
//
|
|
550
|
-
|
|
548
|
+
// Cap fan-out concurrency so a large batch (or a manual run) doesn't fire all
|
|
549
|
+
// window LLM calls at once and spike the provider / collide with the global
|
|
550
|
+
// agent-IPC limit. Small batches (<= cap) still run fully parallel.
|
|
551
|
+
const cycle1Concurrency = Math.max(1, Number(
|
|
552
|
+
config.cycle1_concurrency ?? config.concurrency ?? options.concurrency ?? options.maxConcurrent ?? 4,
|
|
553
|
+
))
|
|
554
|
+
const sem = createSemaphore(Math.min(Math.max(1, windows.length), cycle1Concurrency))
|
|
551
555
|
const settled = await Promise.allSettled(
|
|
552
556
|
windows.map((rows, idx) => sem(() => {
|
|
553
557
|
throwIfAborted(signal)
|
|
@@ -504,6 +504,32 @@ LEFT JOIN exact x ON x.id = c.id`
|
|
|
504
504
|
const rootIdsForReturn = []
|
|
505
505
|
const seen = new Set()
|
|
506
506
|
|
|
507
|
+
// Batch-resolve member-chunk roots in ONE query (was an N+1: a per-row SELECT
|
|
508
|
+
// inside the loop below). Collect the distinct in-scope chunk_root ids, fetch
|
|
509
|
+
// all matching roots at once, then resolve each member from rootById.
|
|
510
|
+
const memberRootIds = []
|
|
511
|
+
const memberRootSeen = new Set()
|
|
512
|
+
for (const { id } of filtered) {
|
|
513
|
+
const r0 = byId.get(id)
|
|
514
|
+
if (!r0 || r0.is_root === 1) continue
|
|
515
|
+
if (r0.chunk_root != null && r0.chunk_root !== r0.id) {
|
|
516
|
+
const rid = Number(r0.chunk_root)
|
|
517
|
+
if (!memberRootSeen.has(rid)) { memberRootSeen.add(rid); memberRootIds.push(rid) }
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
const rootById = new Map()
|
|
521
|
+
if (memberRootIds.length > 0) {
|
|
522
|
+
const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
|
|
523
|
+
const { rows: rootRows } = await recallReadQuery(
|
|
524
|
+
db,
|
|
525
|
+
`SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
526
|
+
element, category, summary, project_id, status, score, last_seen_at
|
|
527
|
+
FROM entries WHERE id = ANY($1::bigint[]) AND is_root = 1 ${rootScopeClause}`,
|
|
528
|
+
[memberRootIds, ...rootScopeParams],
|
|
529
|
+
)
|
|
530
|
+
for (const rr of rootRows) rootById.set(Number(rr.id), rr)
|
|
531
|
+
}
|
|
532
|
+
|
|
507
533
|
for (const { id, rrf, retrievalScore } of filtered) {
|
|
508
534
|
const row = byId.get(id)
|
|
509
535
|
if (!row) continue
|
|
@@ -511,16 +537,7 @@ LEFT JOIN exact x ON x.id = c.id`
|
|
|
511
537
|
if (row.is_root === 1) {
|
|
512
538
|
targetRow = row
|
|
513
539
|
} else if (row.chunk_root != null && row.chunk_root !== row.id) {
|
|
514
|
-
|
|
515
|
-
const { clause: rootScopeClause, params: rootScopeParams } = buildScopeClause(2)
|
|
516
|
-
const { rows: rootRows } = await recallReadQuery(
|
|
517
|
-
db,
|
|
518
|
-
`SELECT id, ts, role, content, session_id, source_turn, chunk_root, is_root,
|
|
519
|
-
element, category, summary, project_id, status, score, last_seen_at
|
|
520
|
-
FROM entries WHERE id = $1 AND is_root = 1 ${rootScopeClause}`,
|
|
521
|
-
[row.chunk_root, ...rootScopeParams],
|
|
522
|
-
)
|
|
523
|
-
const r = rootRows[0]
|
|
540
|
+
const r = rootById.get(Number(row.chunk_root))
|
|
524
541
|
if (!r) continue
|
|
525
542
|
memberHitRootIds.add(r.id)
|
|
526
543
|
targetRow = r
|
|
@@ -5,7 +5,11 @@
|
|
|
5
5
|
* Calls Codex WebSocket endpoint via sendViaWebSocket with web_search server
|
|
6
6
|
* tool. Model is config-driven (search.models.openai default 'gpt-5.4-mini').
|
|
7
7
|
*/
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
OpenAIOAuthProvider,
|
|
10
|
+
ensureLatestCodexModel,
|
|
11
|
+
codexModelSupportsServiceTier,
|
|
12
|
+
} from '../../../agent/orchestrator/providers/openai-oauth.mjs'
|
|
9
13
|
import {
|
|
10
14
|
OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
|
|
11
15
|
buildOpenAISearchPrompt,
|
|
@@ -66,7 +70,7 @@ export async function searchViaOpenAIOAuth({
|
|
|
66
70
|
parallel_tool_calls: false,
|
|
67
71
|
tools: [buildOpenAIWebSearchTool({ site, type, locale, contextSize })],
|
|
68
72
|
}
|
|
69
|
-
if (fast === true) body.service_tier = 'priority'
|
|
73
|
+
if (fast === true && codexModelSupportsServiceTier(useModel, 'priority')) body.service_tier = 'priority'
|
|
70
74
|
// Route through provider.send() (not sendViaWebSocket directly) so the search
|
|
71
75
|
// request inherits the 401/403 force-refresh retry + HTTP/SSE fallback. A
|
|
72
76
|
// stale token or unhealthy WebSocket then recovers instead of hard-failing.
|
package/src/search/lib/cache.mjs
CHANGED
|
@@ -5,6 +5,52 @@ const DEFAULT_CACHE_STATE = {
|
|
|
5
5
|
entries: {},
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
+
// Size bounds on top of TTL expiry so cache.local.json can't grow unbounded.
|
|
9
|
+
const MAX_CACHE_ENTRIES = 500
|
|
10
|
+
const MAX_CACHE_BYTES = 8 * 1024 * 1024
|
|
11
|
+
|
|
12
|
+
// Approximate serialized size of ONE entry (never the whole map), so insert and
|
|
13
|
+
// evict stay cheap. A running total on the state (`__approxBytes`) lets the byte
|
|
14
|
+
// cap be checked without re-serializing every entry on each set.
|
|
15
|
+
function approxEntryBytes(key, entry) {
|
|
16
|
+
try { return String(key).length + JSON.stringify(entry).length } catch { return String(key).length }
|
|
17
|
+
}
|
|
18
|
+
function cacheApproxBytes(state) {
|
|
19
|
+
if (typeof state.__approxBytes === 'number') return state.__approxBytes
|
|
20
|
+
let total = 0
|
|
21
|
+
for (const [k, e] of Object.entries(state.entries)) total += approxEntryBytes(k, e)
|
|
22
|
+
state.__approxBytes = total
|
|
23
|
+
return total
|
|
24
|
+
}
|
|
25
|
+
function removeCacheEntry(state, key) {
|
|
26
|
+
const e = state.entries[key]
|
|
27
|
+
if (e === undefined) return
|
|
28
|
+
if (typeof state.__approxBytes === 'number') state.__approxBytes -= approxEntryBytes(key, e)
|
|
29
|
+
delete state.entries[key]
|
|
30
|
+
}
|
|
31
|
+
function evictOldestCacheEntry(state) {
|
|
32
|
+
let oldestKey = null
|
|
33
|
+
let oldestAt = Infinity
|
|
34
|
+
for (const [k, e] of Object.entries(state.entries)) {
|
|
35
|
+
const at = e?.cachedAt ?? 0
|
|
36
|
+
if (at < oldestAt) { oldestAt = at; oldestKey = k }
|
|
37
|
+
}
|
|
38
|
+
if (oldestKey == null) return false
|
|
39
|
+
removeCacheEntry(state, oldestKey)
|
|
40
|
+
return true
|
|
41
|
+
}
|
|
42
|
+
// Best-effort: evict oldest entries until under both the count and byte caps.
|
|
43
|
+
function enforceCacheSizeBounds(state) {
|
|
44
|
+
try {
|
|
45
|
+
while (Object.keys(state.entries).length > MAX_CACHE_ENTRIES) {
|
|
46
|
+
if (!evictOldestCacheEntry(state)) break
|
|
47
|
+
}
|
|
48
|
+
while (cacheApproxBytes(state) > MAX_CACHE_BYTES && Object.keys(state.entries).length > 0) {
|
|
49
|
+
if (!evictOldestCacheEntry(state)) break
|
|
50
|
+
}
|
|
51
|
+
} catch { /* size bounding is best-effort */ }
|
|
52
|
+
}
|
|
53
|
+
|
|
8
54
|
const FLUSH_DELAY_MS = 5000
|
|
9
55
|
|
|
10
56
|
let cacheDirty = false
|
|
@@ -75,6 +121,8 @@ export function loadCacheState() {
|
|
|
75
121
|
_instance = state
|
|
76
122
|
activeCacheState = state
|
|
77
123
|
pruneExpiredEntries(state)
|
|
124
|
+
cacheApproxBytes(state)
|
|
125
|
+
enforceCacheSizeBounds(state)
|
|
78
126
|
return state
|
|
79
127
|
}
|
|
80
128
|
|
|
@@ -90,7 +138,7 @@ export function getCachedEntry(state, key) {
|
|
|
90
138
|
const entry = state.entries[key]
|
|
91
139
|
if (!entry) return null
|
|
92
140
|
if (entry.expiresAt && entry.expiresAt <= nowMs()) {
|
|
93
|
-
|
|
141
|
+
removeCacheEntry(state, key)
|
|
94
142
|
scheduleCacheFlush(state)
|
|
95
143
|
return null
|
|
96
144
|
}
|
|
@@ -99,11 +147,11 @@ export function getCachedEntry(state, key) {
|
|
|
99
147
|
|
|
100
148
|
export function setCachedEntry(state, key, payload, ttlMs) {
|
|
101
149
|
const cachedAt = nowMs()
|
|
102
|
-
state.entries[key]
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
150
|
+
if (state.entries[key] !== undefined) removeCacheEntry(state, key)
|
|
151
|
+
const entry = { cachedAt, expiresAt: cachedAt + ttlMs, payload }
|
|
152
|
+
state.entries[key] = entry
|
|
153
|
+
if (typeof state.__approxBytes === 'number') state.__approxBytes += approxEntryBytes(key, entry)
|
|
154
|
+
enforceCacheSizeBounds(state)
|
|
107
155
|
scheduleCacheFlush(state)
|
|
108
156
|
return state.entries[key]
|
|
109
157
|
}
|
|
@@ -121,7 +169,7 @@ function pruneExpiredEntries(state) {
|
|
|
121
169
|
let dirty = false
|
|
122
170
|
for (const [key, entry] of Object.entries(state.entries)) {
|
|
123
171
|
if (entry?.expiresAt && entry.expiresAt <= current) {
|
|
124
|
-
|
|
172
|
+
removeCacheEntry(state, key)
|
|
125
173
|
dirty = true
|
|
126
174
|
}
|
|
127
175
|
}
|
package/src/shared/config.mjs
CHANGED
|
@@ -299,7 +299,7 @@ export function getSearchApiKey(provider) {
|
|
|
299
299
|
// exports keep working, then MIXDOG_AGENT_<P>_APIKEY, then the OS keychain.
|
|
300
300
|
const AGENT_PROVIDER_ENV = Object.freeze({
|
|
301
301
|
openai: 'OPENAI_API_KEY', anthropic: 'ANTHROPIC_API_KEY', gemini: 'GEMINI_API_KEY',
|
|
302
|
-
deepseek: 'DEEPSEEK_API_KEY', xai: 'XAI_API_KEY',
|
|
302
|
+
deepseek: 'DEEPSEEK_API_KEY', xai: 'XAI_API_KEY',
|
|
303
303
|
})
|
|
304
304
|
|
|
305
305
|
// Last-resort env aliases honored AFTER the standard env / MIXDOG_AGENT_* /
|
package/src/shared/llm/cost.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import { getModelMetadataSync } from '../../agent/orchestrator/providers/model-c
|
|
|
17
17
|
// count *including* the cached portion (inclusive). Anthropic reports the
|
|
18
18
|
// uncached remainder only and bills cached_read / cached_write as separate
|
|
19
19
|
// additive slots (additive). Cost and prompt-total math has to branch on this.
|
|
20
|
-
// OpenAI-compatible direct providers (deepseek /
|
|
20
|
+
// OpenAI-compatible direct providers (deepseek / ollama / lmstudio)
|
|
21
21
|
// go through the OpenAI SDK and likewise report an inclusive prompt_tokens
|
|
22
22
|
// with a separate cached-tokens detail — so they are inclusive too. Omitting
|
|
23
23
|
// them bills the cached portion at the full input rate AND re-adds it as a
|
|
@@ -31,7 +31,7 @@ export function isInclusiveProvider(provider) {
|
|
|
31
31
|
// usage rows — without it, cached tokens would be double-billed in the
|
|
32
32
|
// cost fallback and prompt totals.
|
|
33
33
|
return p.includes('openai') || p.includes('codex') || p.includes('gemini') || p.includes('google') || p.includes('xai') || p.includes('grok')
|
|
34
|
-
|| p.includes('deepseek') || p.includes('
|
|
34
|
+
|| p.includes('deepseek') || p.includes('ollama') || p.includes('lmstudio') || p.includes('groq') || p.includes('openrouter');
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
/**
|