@link-assistant/hive-mind 2.13.4 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.hi.md +2 -0
- package/README.md +2 -0
- package/README.ru.md +2 -0
- package/README.zh.md +2 -0
- package/package.json +1 -1
- package/src/agent.lib.mjs +5 -1
- package/src/claude.lib.mjs +5 -158
- package/src/claude.session-tokens.lib.mjs +180 -0
- package/src/codex.diagnostics.lib.mjs +135 -0
- package/src/codex.lib.mjs +8 -121
- package/src/config.lib.mjs +9 -0
- package/src/docker-sidecar.lib.mjs +276 -0
- package/src/formal-ai-maintenance.lib.mjs +2 -14
- package/src/formal-ai-sidecar.lib.mjs +17 -137
- package/src/gemini.lib.mjs +5 -1
- package/src/git-push-guard.lib.mjs +230 -0
- package/src/git-retry.lib.mjs +97 -0
- package/src/github-pr-idempotency.lib.mjs +83 -0
- package/src/github-rate-limit.lib.mjs +44 -41
- package/src/hive.mjs +8 -150
- package/src/hive.repository-fallback.lib.mjs +125 -0
- package/src/hive.startup-checks.lib.mjs +57 -0
- package/src/isolation-runner.lib.mjs +94 -287
- package/src/isolation-runner.parsers.lib.mjs +292 -0
- package/src/lib.mjs +79 -18
- package/src/opencode.lib.mjs +5 -1
- package/src/qwen.lib.mjs +5 -1
- package/src/router-isolation.lib.mjs +496 -0
- package/src/router-logs.lib.mjs +143 -0
- package/src/router-maintenance.lib.mjs +77 -0
- package/src/router-session-drain.lib.mjs +153 -0
- package/src/router-sidecar.lib.mjs +516 -0
- package/src/router-task-isolation.lib.mjs +121 -0
- package/src/session-monitor.lib.mjs +12 -272
- package/src/session-monitor.queries.lib.mjs +304 -0
- package/src/solve.auto-pr-push-sync.lib.mjs +176 -0
- package/src/solve.auto-pr.lib.mjs +40 -154
- package/src/solve.config.lib.mjs +11 -0
- package/src/solve.mjs +8 -158
- package/src/solve.mode.lib.mjs +191 -0
- package/src/task.config.lib.mjs +5 -0
- package/src/task.mjs +1 -0
- package/src/telegram-bot.mjs +18 -0
- package/src/telegram-solve-queue.lib.mjs +19 -272
- package/src/telegram-solve-queue.throttling.lib.mjs +323 -0
- package/src/transient-errors.lib.mjs +238 -0
package/src/codex.lib.mjs
CHANGED
|
@@ -7,7 +7,11 @@ import { ensureUseM } from './use-m-bootstrap.lib.mjs';
|
|
|
7
7
|
if (typeof globalThis.use === 'undefined') {
|
|
8
8
|
await ensureUseM();
|
|
9
9
|
}
|
|
10
|
-
const { $ } = await use('command-stream');
|
|
10
|
+
const { $: __rawDollar$ } = await use('command-stream');
|
|
11
|
+
// Issue #2168: retry transient git network failures (push/fetch/pull) the same
|
|
12
|
+
// way `gh` calls are retried, for every command run through this module's `$`.
|
|
13
|
+
const { wrapDollarWithGitRetry } = await import('./git-retry.lib.mjs');
|
|
14
|
+
const $ = wrapDollarWithGitRetry(__rawDollar$);
|
|
11
15
|
const fs = (await use('fs')).promises;
|
|
12
16
|
const path = (await use('path')).default;
|
|
13
17
|
const os = (await use('os')).default;
|
|
@@ -44,127 +48,10 @@ import Decimal from 'decimal.js-light';
|
|
|
44
48
|
import { ensureAiToolScratchIgnored, filterAiToolScratchFromStatus } from './ai-tool-scratch.lib.mjs';
|
|
45
49
|
import { CODEX_CACHE_READ_USAGE_PATHS, CODEX_CACHE_WRITE_USAGE_PATHS, CODEX_MODEL_DIAGNOSTIC_PATHS, CODEX_REASONING_USAGE_PATHS, CODEX_USAGE_FIELD_NAMES, createCodexTokenFieldAvailability, getFirstObservedNumber, hasAnyObservedPath, hasOwnPath } from './codex.usage-fields.lib.mjs';
|
|
46
50
|
const CODEX_LONG_CONTEXT_PRICE_THRESHOLD = 272000;
|
|
47
|
-
const CODEX_COMPACT_API_ENDPOINT = '/responses/compact';
|
|
48
51
|
const getCodexExecEnv = (verbose = false) => (verbose ? { ...process.env, RUST_LOG: 'debug' } : { ...process.env });
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const match = line.match(new RegExp(`${escapeRegExp(key)}=(?:"([^"]*)"|([^\\s")]+))`));
|
|
53
|
-
return match?.[1] ?? match?.[2] ?? null;
|
|
54
|
-
};
|
|
55
|
-
const getCodexDiagnosticInteger = (line, key) => {
|
|
56
|
-
const value = getCodexDiagnosticValue(line, key);
|
|
57
|
-
if (value === null) return null;
|
|
58
|
-
const parsed = Number.parseInt(value, 10);
|
|
59
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
60
|
-
};
|
|
61
|
-
const getCodexDiagnosticTimestamp = line => {
|
|
62
|
-
const eventTimestamp = getCodexDiagnosticValue(line, 'event.timestamp');
|
|
63
|
-
if (eventTimestamp) return eventTimestamp;
|
|
64
|
-
const logPrefixMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+Z)\]/u);
|
|
65
|
-
return logPrefixMatch?.[1] ?? null;
|
|
66
|
-
};
|
|
67
|
-
const isSuccessfulCodexCompactRequestLine = line => {
|
|
68
|
-
if (!line.includes('codex_otel.log_only:')) return false;
|
|
69
|
-
if (!line.includes('event.name="codex.api_request"')) return false;
|
|
70
|
-
if (!line.includes(`endpoint="${CODEX_COMPACT_API_ENDPOINT}"`)) return false;
|
|
71
|
-
const statusCode = getCodexDiagnosticInteger(line, 'http.response.status_code');
|
|
72
|
-
return statusCode === null || (statusCode >= 200 && statusCode < 300);
|
|
73
|
-
};
|
|
74
|
-
|
|
75
|
-
const splitTokenCountEvenly = (total, partCount) => {
|
|
76
|
-
const safeTotal = Math.max(0, Math.round(total || 0));
|
|
77
|
-
const safePartCount = Math.max(1, Math.round(partCount || 1));
|
|
78
|
-
const base = Math.floor(safeTotal / safePartCount);
|
|
79
|
-
let remainder = safeTotal % safePartCount;
|
|
80
|
-
return Array.from({ length: safePartCount }, () => {
|
|
81
|
-
const value = base + (remainder > 0 ? 1 : 0);
|
|
82
|
-
if (remainder > 0) remainder--;
|
|
83
|
-
return value;
|
|
84
|
-
});
|
|
85
|
-
};
|
|
86
|
-
const splitCodexSubSessionInputTokens = (total, partCount, autoCompactTokenLimit = null) => {
|
|
87
|
-
const safeTotal = Math.max(0, Math.round(total || 0));
|
|
88
|
-
const safePartCount = Math.max(1, Math.round(partCount || 1));
|
|
89
|
-
const safeLimit = Number.isFinite(autoCompactTokenLimit) && autoCompactTokenLimit > 0 ? Math.round(autoCompactTokenLimit) : null;
|
|
90
|
-
if (safePartCount <= 1) return [safeTotal];
|
|
91
|
-
if (safeLimit && safeTotal > safeLimit * (safePartCount - 1)) {
|
|
92
|
-
const chunks = [];
|
|
93
|
-
let remaining = safeTotal;
|
|
94
|
-
for (let i = 0; i < safePartCount - 1; i++) {
|
|
95
|
-
const chunk = Math.min(safeLimit, remaining);
|
|
96
|
-
chunks.push(chunk);
|
|
97
|
-
remaining -= chunk;
|
|
98
|
-
}
|
|
99
|
-
chunks.push(Math.max(0, remaining));
|
|
100
|
-
return chunks;
|
|
101
|
-
}
|
|
102
|
-
return splitTokenCountEvenly(safeTotal, safePartCount);
|
|
103
|
-
};
|
|
104
|
-
const splitTokenCountByWeights = (total, weights) => {
|
|
105
|
-
const safeTotal = Math.max(0, Math.round(total || 0));
|
|
106
|
-
const safeWeights = Array.isArray(weights) && weights.length > 0 ? weights.map(weight => Math.max(0, weight || 0)) : [1];
|
|
107
|
-
const weightTotal = safeWeights.reduce((sum, weight) => sum + weight, 0);
|
|
108
|
-
if (weightTotal <= 0) return splitTokenCountEvenly(safeTotal, safeWeights.length);
|
|
109
|
-
let allocated = 0;
|
|
110
|
-
return safeWeights.map((weight, index) => {
|
|
111
|
-
if (index === safeWeights.length - 1) return Math.max(0, safeTotal - allocated);
|
|
112
|
-
const value = Math.floor((safeTotal * weight) / weightTotal);
|
|
113
|
-
allocated += value;
|
|
114
|
-
return value;
|
|
115
|
-
});
|
|
116
|
-
};
|
|
117
|
-
const rebuildCodexSubSessionsFromCompactifications = tokenUsage => {
|
|
118
|
-
const compactifications = Array.isArray(tokenUsage.compactifications) ? tokenUsage.compactifications : [];
|
|
119
|
-
if (compactifications.length === 0 || (tokenUsage.stepCount || 0) === 0) {
|
|
120
|
-
tokenUsage.subSessions = Array.isArray(tokenUsage.subSessions) ? tokenUsage.subSessions : [];
|
|
121
|
-
return;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
const subSessionCount = compactifications.length + 1;
|
|
125
|
-
const inputChunks = splitCodexSubSessionInputTokens(tokenUsage.inputTokens || 0, subSessionCount, tokenUsage.autoCompactTokenLimit);
|
|
126
|
-
const cacheWriteChunks = splitTokenCountByWeights(tokenUsage.cacheWriteTokens || 0, inputChunks);
|
|
127
|
-
const cacheReadChunks = splitTokenCountByWeights(tokenUsage.cacheReadTokens || 0, inputChunks);
|
|
128
|
-
const outputChunks = splitTokenCountByWeights(tokenUsage.outputTokens || 0, inputChunks);
|
|
129
|
-
tokenUsage.subSessions = inputChunks.map((inputTokens, index) => {
|
|
130
|
-
const cacheCreationTokens = cacheWriteChunks[index] || 0;
|
|
131
|
-
const outputTokens = outputChunks[index] || 0;
|
|
132
|
-
return {
|
|
133
|
-
inputTokens,
|
|
134
|
-
cacheCreationTokens,
|
|
135
|
-
cacheReadTokens: cacheReadChunks[index] || 0,
|
|
136
|
-
outputTokens,
|
|
137
|
-
messageCount: null,
|
|
138
|
-
peakContextUsage: getCumulativeContextInputTokens({ inputTokens, cacheCreationTokens }),
|
|
139
|
-
peakOutputUsage: outputTokens,
|
|
140
|
-
estimated: true,
|
|
141
|
-
source: 'codex.compact-diagnostics',
|
|
142
|
-
compactBoundaryBefore: index === 0 ? null : compactifications[index - 1] || null,
|
|
143
|
-
};
|
|
144
|
-
});
|
|
145
|
-
};
|
|
146
|
-
const recordCodexCompactification = (line, tokenUsage) => {
|
|
147
|
-
if (!isSuccessfulCodexCompactRequestLine(line)) return;
|
|
148
|
-
const timestamp = getCodexDiagnosticTimestamp(line);
|
|
149
|
-
const conversationId = getCodexDiagnosticValue(line, 'conversation.id');
|
|
150
|
-
const existing = tokenUsage.compactifications.find(compact => compact.timestamp === timestamp && compact.conversationId === conversationId);
|
|
151
|
-
if (existing) return;
|
|
152
|
-
tokenUsage.compactifications.push({
|
|
153
|
-
timestamp,
|
|
154
|
-
preTokens: null,
|
|
155
|
-
trigger: 'auto',
|
|
156
|
-
source: 'codex.responses.compact',
|
|
157
|
-
conversationId: conversationId || null,
|
|
158
|
-
});
|
|
159
|
-
};
|
|
160
|
-
const parseCodexDiagnosticLine = (line, tokenUsage) => {
|
|
161
|
-
const contextLimit = getCodexDiagnosticInteger(line, 'context_window') ?? getCodexDiagnosticInteger(line, 'model_context_window');
|
|
162
|
-
if (contextLimit !== null) tokenUsage.contextLimit = contextLimit;
|
|
163
|
-
|
|
164
|
-
const autoCompactTokenLimit = getCodexDiagnosticInteger(line, 'auto_compact_token_limit') ?? getCodexDiagnosticInteger(line, 'model_auto_compact_token_limit');
|
|
165
|
-
if (autoCompactTokenLimit !== null) tokenUsage.autoCompactTokenLimit = autoCompactTokenLimit;
|
|
166
|
-
recordCodexCompactification(line, tokenUsage);
|
|
167
|
-
};
|
|
52
|
+
// Issue #2175: diagnostic-line parsing lives in its own module to keep this file
|
|
53
|
+
// under the 1350-line warning threshold.
|
|
54
|
+
import { parseCodexDiagnosticLine, rebuildCodexSubSessionsFromCompactifications } from './codex.diagnostics.lib.mjs';
|
|
168
55
|
export const createCodexTokenUsage = requestedModelId => ({
|
|
169
56
|
inputTokens: 0,
|
|
170
57
|
outputTokens: 0,
|
package/src/config.lib.mjs
CHANGED
|
@@ -115,6 +115,15 @@ export const retryLimits = {
|
|
|
115
115
|
maxForkRetries: parseIntWithDefault('HIVE_MIND_MAX_FORK_RETRIES', 5),
|
|
116
116
|
maxVerifyRetries: parseIntWithDefault('HIVE_MIND_MAX_VERIFY_RETRIES', 5),
|
|
117
117
|
maxApiRetries: parseIntWithDefault('HIVE_MIND_MAX_API_RETRIES', 3),
|
|
118
|
+
// Issue #2168: GitHub's own 5xx / GraphQL-internal failures ("Something went
|
|
119
|
+
// wrong while executing your query") are usually over within seconds, but 3
|
|
120
|
+
// attempts at 1s+2s was too tight to ride one out. These budgets are used by
|
|
121
|
+
// `ghWithRateLimitRetry` for the transient (non-rate-limit) branch.
|
|
122
|
+
maxGitHubTransientRetries: parseIntWithDefault('HIVE_MIND_MAX_GITHUB_TRANSIENT_RETRIES', 6),
|
|
123
|
+
initialGitHubTransientDelayMs: parseIntWithDefault('HIVE_MIND_INITIAL_GITHUB_TRANSIENT_DELAY_MS', 2000),
|
|
124
|
+
// Issue #2168: network-facing git operations (push/fetch/clone) get the same
|
|
125
|
+
// treatment via `gitCmdRetry` in src/lib.mjs.
|
|
126
|
+
maxGitRetries: parseIntWithDefault('HIVE_MIND_MAX_GIT_RETRIES', 5),
|
|
118
127
|
retryBackoffMultiplier: parseFloatWithDefault('HIVE_MIND_RETRY_BACKOFF_MULTIPLIER', 2),
|
|
119
128
|
// Unified retry config for all transient API errors (Overloaded, 503, Internal Server Error)
|
|
120
129
|
// Issue #2169: count backstop only. With the defaults below (3 min → 30 min backoff) the 12-hour
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mechanics shared by every on-demand Docker sidecar Hive Mind runs.
|
|
3
|
+
*
|
|
4
|
+
* The Formal AI sidecar (issue #2146) and the router sidecar (issue #2164) are
|
|
5
|
+
* different services with different reasons to exist, but their *mechanics* are
|
|
6
|
+
* the same problem solved twice: one container shared by concurrent tasks, a
|
|
7
|
+
* lease per task, a durable JSON record that is only ever a cache of what
|
|
8
|
+
* Docker actually reports, and an exclusive lock so a launch and an update can
|
|
9
|
+
* never interleave.
|
|
10
|
+
*
|
|
11
|
+
* Those mechanics live here so the two lifecycles stay in step — a fix to lease
|
|
12
|
+
* reconciliation or to the network guard applies to both — and so each sidecar
|
|
13
|
+
* module is left holding only the part that is genuinely its own: which image,
|
|
14
|
+
* which mounts, which readiness check, and what the task is handed.
|
|
15
|
+
*
|
|
16
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2146
|
|
17
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2164
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { execFile } from 'node:child_process';
|
|
21
|
+
import fs from 'node:fs';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
import { promisify } from 'node:util';
|
|
24
|
+
|
|
25
|
+
import { resolveBotStateDir } from './session-store.lib.mjs';
|
|
26
|
+
|
|
27
|
+
const execFileAsync = promisify(execFile);
|
|
28
|
+
|
|
29
|
+
export const DEFAULT_DOCKER_TIMEOUT_MS = 120_000;
|
|
30
|
+
// Pulling a sidecar image is the one Docker call that legitimately takes many
|
|
31
|
+
// minutes, so it gets its own budget instead of the general command timeout.
|
|
32
|
+
export const DEFAULT_IMAGE_TIMEOUT_MS = 600_000;
|
|
33
|
+
|
|
34
|
+
export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
35
|
+
|
|
36
|
+
/** Run `docker …` and return trimmed stdout. Throws on a non-zero exit. */
|
|
37
|
+
export const dockerText = async (run, args, { timeoutMs = DEFAULT_DOCKER_TIMEOUT_MS } = {}) => {
|
|
38
|
+
const result = await run('docker', args, { encoding: 'utf8', timeout: timeoutMs });
|
|
39
|
+
return String(result?.stdout ?? '').trim();
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/** Run `docker …` for its effect only, reporting success as a boolean. */
|
|
43
|
+
export const dockerOk = async (run, args, options) => {
|
|
44
|
+
try {
|
|
45
|
+
await dockerText(run, args, options);
|
|
46
|
+
return true;
|
|
47
|
+
} catch {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
/** The message a failed `docker` invocation should be reported with. */
|
|
53
|
+
export const dockerErrorMessage = error => error?.stderr?.toString?.().trim() || error?.message || String(error);
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Inspect a container without treating "absent" as an error.
|
|
57
|
+
*
|
|
58
|
+
* @returns {Promise<{exists: boolean, running: boolean, image: string|null, imageDigest: string|null}>}
|
|
59
|
+
*/
|
|
60
|
+
export const inspectDockerContainer = async (name, { run = execFileAsync, timeoutMs } = {}) => {
|
|
61
|
+
try {
|
|
62
|
+
const raw = await dockerText(run, ['inspect', name, '--format', '{{.State.Running}}|{{.Config.Image}}|{{.Image}}'], { timeoutMs });
|
|
63
|
+
const [running, image, imageDigest] = raw.split('|');
|
|
64
|
+
return { exists: true, running: running === 'true', image: image || null, imageDigest: imageDigest || null };
|
|
65
|
+
} catch {
|
|
66
|
+
return { exists: false, running: false, image: null, imageDigest: null };
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
/** Resolve the local content digest of an image reference, or null when it is absent. */
|
|
71
|
+
export const readDockerImageDigest = async (image, { run = execFileAsync, timeoutMs } = {}) => {
|
|
72
|
+
try {
|
|
73
|
+
return (await dockerText(run, ['image', 'inspect', image, '--format', '{{.Id}}'], { timeoutMs })) || null;
|
|
74
|
+
} catch {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/** A container's IPv4 address on one named network, or null when it has none. */
|
|
80
|
+
export const readDockerContainerAddress = async (containerName, network, { run = execFileAsync, timeoutMs } = {}) => {
|
|
81
|
+
try {
|
|
82
|
+
return (await dockerText(run, ['inspect', containerName, '--format', `{{with index .NetworkSettings.Networks "${network}"}}{{.IPAddress}}{{end}}`], { timeoutMs })) || null;
|
|
83
|
+
} catch {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Create the private network a sidecar and its tasks share.
|
|
90
|
+
*
|
|
91
|
+
* `--internal` is the security requirement: the sidecar endpoint must not be
|
|
92
|
+
* published to the host and must not be reachable from any other network. An
|
|
93
|
+
* existing network that is *not* internal is a stale artifact from an older
|
|
94
|
+
* deployment and is replaced — but only while nothing is attached to it, since
|
|
95
|
+
* removing a network out from under a running container would break it.
|
|
96
|
+
*/
|
|
97
|
+
export const ensureInternalDockerNetwork = async ({ name, label, run = execFileAsync, timeoutMs, log = null, verbose = false, logPrefix = 'docker-sidecar' } = {}) => {
|
|
98
|
+
// `null` means "absent", which is different from "present but not internal".
|
|
99
|
+
let internal = null;
|
|
100
|
+
let containers = 0;
|
|
101
|
+
try {
|
|
102
|
+
const raw = await dockerText(run, ['network', 'inspect', name, '--format', '{{.Internal}}|{{len .Containers}}'], { timeoutMs });
|
|
103
|
+
const [internalFlag, containerCount] = raw.split('|');
|
|
104
|
+
internal = internalFlag === 'true';
|
|
105
|
+
containers = Number(containerCount) || 0;
|
|
106
|
+
} catch {
|
|
107
|
+
// Absent; fall through to creation.
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (internal === true) return { created: false, internal: true };
|
|
111
|
+
|
|
112
|
+
if (internal === false) {
|
|
113
|
+
if (containers > 0) {
|
|
114
|
+
if (log) await log(`⚠️ Network '${name}' is not internal but still has ${containers} attached container(s); leaving it in place`);
|
|
115
|
+
return { created: false, internal: false };
|
|
116
|
+
}
|
|
117
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: replacing non-internal network '${name}'`);
|
|
118
|
+
await dockerOk(run, ['network', 'rm', name], { timeoutMs });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
await dockerText(run, ['network', 'create', '--internal', '--label', `${label}=network`, name], { timeoutMs });
|
|
122
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: created internal network '${name}'`);
|
|
123
|
+
return { created: true, internal: true };
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Add a network to an already-created container.
|
|
128
|
+
*
|
|
129
|
+
* A single `docker run --network` *replaces* the container's default bridge, so
|
|
130
|
+
* an `--internal` network passed that way would also cut the container off from
|
|
131
|
+
* GitHub and the package registries. Attaching afterwards is additive, which is
|
|
132
|
+
* what both sidecars need — for the task container, and for the router sidecar
|
|
133
|
+
* itself, which must keep its outbound route to the vendor APIs.
|
|
134
|
+
*/
|
|
135
|
+
export const attachDockerNetwork = async ({ network, container, alias = null, run = execFileAsync, timeoutMs, log = null, verbose = false, logPrefix = 'docker-sidecar' } = {}) => {
|
|
136
|
+
if (!container) return { attached: false, error: 'no container' };
|
|
137
|
+
const args = ['network', 'connect'];
|
|
138
|
+
if (alias) args.push('--alias', alias);
|
|
139
|
+
args.push(network, container);
|
|
140
|
+
try {
|
|
141
|
+
await dockerText(run, args, { timeoutMs });
|
|
142
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: attached '${container}' to '${network}'${alias ? ` as '${alias}'` : ''}`);
|
|
143
|
+
return { attached: true, error: null };
|
|
144
|
+
} catch (error) {
|
|
145
|
+
const message = dockerErrorMessage(error);
|
|
146
|
+
// Docker reports an already-attached container as an error; that is success.
|
|
147
|
+
if (/already exists in network/i.test(message)) return { attached: true, error: null };
|
|
148
|
+
if (log) await log(`⚠️ Could not attach '${container}' to network '${network}': ${message}`);
|
|
149
|
+
return { attached: false, error: message };
|
|
150
|
+
}
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
/** Create a named volume if it is missing. Never removed by any caller: it holds the data the sidecar exists to keep. */
|
|
154
|
+
export const ensureDockerVolume = async ({ name, label, role = 'data', run = execFileAsync, timeoutMs, log = null, verbose = false, logPrefix = 'docker-sidecar' } = {}) => {
|
|
155
|
+
if (await dockerOk(run, ['volume', 'inspect', name], { timeoutMs })) return { created: false };
|
|
156
|
+
await dockerText(run, ['volume', 'create', '--label', `${label}=${role}`, name], { timeoutMs });
|
|
157
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: created volume '${name}'`);
|
|
158
|
+
return { created: true };
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
/** Path of a sidecar's durable record inside the bot state directory. */
|
|
162
|
+
export const resolveSidecarStatePath = (fileName, env = process.env) => path.join(resolveBotStateDir(env), fileName);
|
|
163
|
+
|
|
164
|
+
/** Read a durable sidecar record. A missing or corrupt file is an empty record, never a throw. */
|
|
165
|
+
export const readSidecarState = ({ fileName, emptyState, env = process.env, fsImpl = fs } = {}) => {
|
|
166
|
+
try {
|
|
167
|
+
const parsed = JSON.parse(fsImpl.readFileSync(resolveSidecarStatePath(fileName, env), 'utf8'));
|
|
168
|
+
return { ...emptyState, ...parsed, leases: Array.isArray(parsed?.leases) ? parsed.leases : [] };
|
|
169
|
+
} catch {
|
|
170
|
+
return { ...emptyState, leases: [] };
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Persist a sidecar record atomically so a crash mid-write cannot corrupt it.
|
|
176
|
+
*
|
|
177
|
+
* `mode` exists because the router's record holds its JWT signing secret, which
|
|
178
|
+
* mints subscription access: that file must not be world-readable (issue #2164).
|
|
179
|
+
*/
|
|
180
|
+
export const writeSidecarState = (state, { fileName, env = process.env, fsImpl = fs, mode = 0o600 } = {}) => {
|
|
181
|
+
const target = resolveSidecarStatePath(fileName, env);
|
|
182
|
+
fsImpl.mkdirSync(path.dirname(target), { recursive: true });
|
|
183
|
+
const temporary = `${target}.tmp`;
|
|
184
|
+
fsImpl.writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { encoding: 'utf8', mode });
|
|
185
|
+
fsImpl.renameSync(temporary, target);
|
|
186
|
+
return state;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* How long a lease whose container has never been seen running is kept.
|
|
191
|
+
*
|
|
192
|
+
* A lease is taken *before* start-command creates the task container, because
|
|
193
|
+
* the endpoint and the token have to be known when the task's environment is
|
|
194
|
+
* built. During that window the container legitimately does not exist yet — and
|
|
195
|
+
* creating it can take a long time when the isolation image still has to be
|
|
196
|
+
* pulled.
|
|
197
|
+
*/
|
|
198
|
+
export const LEASE_START_GRACE_MS = 60 * 60 * 1000;
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Drop leases whose task container no longer runs, so a crashed run cannot pin
|
|
202
|
+
* a sidecar up forever.
|
|
203
|
+
*
|
|
204
|
+
* Liveness is re-derived from Docker on every call rather than trusted from the
|
|
205
|
+
* store, which is what lets a restarted bot converge instead of orphaning.
|
|
206
|
+
*
|
|
207
|
+
* @param {Array<object>} leases
|
|
208
|
+
* @param {{onDropped?: (lease: object) => Promise<void>|void}} options
|
|
209
|
+
* `onDropped` lets a sidecar clean up what the lease owned — the router
|
|
210
|
+
* revokes the task's token there.
|
|
211
|
+
*/
|
|
212
|
+
export const reconcileSidecarLeases = async (leases, { run = execFileAsync, timeoutMs, log = null, verbose = false, now = () => Date.now(), onDropped = null, logPrefix = 'docker-sidecar' } = {}) => {
|
|
213
|
+
const live = [];
|
|
214
|
+
for (const lease of leases) {
|
|
215
|
+
if (!lease?.sessionId) continue;
|
|
216
|
+
const container = await inspectDockerContainer(lease.sessionId, { run, timeoutMs });
|
|
217
|
+
if (container.exists && container.running) {
|
|
218
|
+
live.push(lease.containerSeen ? lease : { ...lease, containerSeen: true });
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
if (!lease.containerSeen) {
|
|
222
|
+
const age = now() - (Date.parse(lease.acquiredAt ?? '') || 0);
|
|
223
|
+
if (age < LEASE_START_GRACE_MS) {
|
|
224
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: keeping lease '${lease.sessionId}' whose container has not appeared yet (${Math.round(age / 1000)}s into the ${Math.round(LEASE_START_GRACE_MS / 1000)}s launch grace)`);
|
|
225
|
+
live.push(lease);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (verbose && log) await log(`[VERBOSE] ${logPrefix}: dropping stale lease '${lease.sessionId}' (container exists=${container.exists} running=${container.running})`);
|
|
230
|
+
if (onDropped) await onDropped(lease);
|
|
231
|
+
}
|
|
232
|
+
return live;
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Periodic best-effort maintenance timer shared by the sidecar lifecycles.
|
|
237
|
+
*
|
|
238
|
+
* Both the Formal AI and the router sidecar need the same thing: a tick that
|
|
239
|
+
* runs immediately, repeats on an interval, never keeps the process alive on
|
|
240
|
+
* shutdown, and never rejects into the bot's event loop. Keeping one
|
|
241
|
+
* implementation means a fix to that shape applies to both.
|
|
242
|
+
*
|
|
243
|
+
* @returns {{stop: () => void}}
|
|
244
|
+
*/
|
|
245
|
+
export const startSidecarMaintenance = ({ runTick, logPrefix = 'sidecar-maintenance', env = process.env, log = null, verbose = false, intervalMs, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval } = {}) => {
|
|
246
|
+
const tick = () => {
|
|
247
|
+
runTick({ env, log, verbose }).catch(error => {
|
|
248
|
+
console.error(`[${logPrefix}] tick failed: ${error?.message || error}`);
|
|
249
|
+
});
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
const timer = setIntervalImpl(tick, intervalMs);
|
|
253
|
+
timer?.unref?.();
|
|
254
|
+
tick();
|
|
255
|
+
return {
|
|
256
|
+
stop: () => clearIntervalImpl(timer),
|
|
257
|
+
};
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export default {
|
|
261
|
+
attachDockerNetwork,
|
|
262
|
+
dockerErrorMessage,
|
|
263
|
+
dockerOk,
|
|
264
|
+
dockerText,
|
|
265
|
+
ensureDockerVolume,
|
|
266
|
+
ensureInternalDockerNetwork,
|
|
267
|
+
inspectDockerContainer,
|
|
268
|
+
readDockerContainerAddress,
|
|
269
|
+
readDockerImageDigest,
|
|
270
|
+
readSidecarState,
|
|
271
|
+
reconcileSidecarLeases,
|
|
272
|
+
resolveSidecarStatePath,
|
|
273
|
+
sleep,
|
|
274
|
+
startSidecarMaintenance,
|
|
275
|
+
writeSidecarState,
|
|
276
|
+
};
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import { updateAgenticClisWhenIdle } from './agentic-cli-updater.lib.mjs';
|
|
24
|
+
import { startSidecarMaintenance } from './docker-sidecar.lib.mjs';
|
|
24
25
|
import { reconcileFormalAiSidecar, stopFormalAiSidecar, withFormalAiSidecarLock } from './formal-ai-sidecar.lib.mjs';
|
|
25
26
|
import { updateFormalAiSidecarWhenIdle } from './formal-ai-updater.lib.mjs';
|
|
26
27
|
|
|
@@ -88,19 +89,6 @@ export const runFormalAiMaintenanceTick = async ({ env = process.env, run, log =
|
|
|
88
89
|
*
|
|
89
90
|
* @returns {{stop: () => void}}
|
|
90
91
|
*/
|
|
91
|
-
export const startFormalAiMaintenance = ({ env = process.env, log = null, verbose = false, intervalMs = DEFAULT_FORMAL_AI_MAINTENANCE_INTERVAL_MS, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, runTick = runFormalAiMaintenanceTick } = {}) => {
|
|
92
|
-
const tick = () => {
|
|
93
|
-
runTick({ env, log, verbose }).catch(error => {
|
|
94
|
-
console.error(`[formal-ai-maintenance] tick failed: ${error?.message || error}`);
|
|
95
|
-
});
|
|
96
|
-
};
|
|
97
|
-
|
|
98
|
-
const timer = setIntervalImpl(tick, intervalMs);
|
|
99
|
-
timer?.unref?.();
|
|
100
|
-
tick();
|
|
101
|
-
return {
|
|
102
|
-
stop: () => clearIntervalImpl(timer),
|
|
103
|
-
};
|
|
104
|
-
};
|
|
92
|
+
export const startFormalAiMaintenance = ({ env = process.env, log = null, verbose = false, intervalMs = DEFAULT_FORMAL_AI_MAINTENANCE_INTERVAL_MS, setIntervalImpl = setInterval, clearIntervalImpl = clearInterval, runTick = runFormalAiMaintenanceTick } = {}) => startSidecarMaintenance({ runTick, logPrefix: 'formal-ai-maintenance', env, log, verbose, intervalMs, setIntervalImpl, clearIntervalImpl });
|
|
105
93
|
|
|
106
94
|
export default { DEFAULT_FORMAL_AI_MAINTENANCE_INTERVAL_MS, runFormalAiMaintenanceTick, startFormalAiMaintenance, stopIdleFormalAiSidecar };
|