@link-assistant/hive-mind 2.12.2 → 2.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/formal-ai-image.lib.mjs +222 -0
- package/src/formal-ai-isolation.lib.mjs +8 -1
- package/src/formal-ai-sidecar.lib.mjs +36 -13
- package/src/formal-ai-updater.lib.mjs +14 -2
- package/src/hive-mind-image.lib.mjs +56 -0
- package/src/isolation-runner.lib.mjs +71 -54
- package/src/locales/en.lino +3 -0
- package/src/locales/hi.lino +3 -0
- package/src/locales/ru.lino +3 -0
- package/src/locales/zh.lino +3 -0
- package/src/session-monitor.lib.mjs +35 -4
- package/src/session-store.lib.mjs +7 -2
- package/src/telegram-command-execution.lib.mjs +26 -7
- package/src/telegram-solve-queue.lib.mjs +30 -5
- package/src/work-session-formatting.lib.mjs +56 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @link-assistant/hive-mind
|
|
2
2
|
|
|
3
|
+
## 2.12.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 97b034f: Make a refused work session explain itself. A Formal AI sidecar (or any isolation) launch that never produced a container now logs the session UUID, backend, tool and reason to stderr instead of failing silently, keeps the UUID in the Telegram failure reply together with a sentence saying the session has no log and is not listed by `--list`, records the reason on the `session_untracked` event and in the durable session store, and is reported by `/queue` as a failed item rather than as `Finished: … (started)`. Registry pull refusals are classified, so a permanent `unauthorized`/`denied`/`not-found` escalates with remediation instead of repeating the same bland warning, and a task image that cannot be pulled falls back to a locally present one. Telegram replies also show start-command's execution UUID — the identifier `$ --list` prints — next to the session UUID, so a running or finished task can finally be found in the session list.
|
|
8
|
+
|
|
3
9
|
## 2.12.2
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve — and prove available — the image the Formal AI sidecar boots from
|
|
3
|
+
* (issue #2154).
|
|
4
|
+
*
|
|
5
|
+
* ## Why this module exists
|
|
6
|
+
*
|
|
7
|
+
* Before this module, `acquireFormalAiSidecar` handed
|
|
8
|
+
* `ghcr.io/link-assistant/formal-ai:<version>` straight to `docker run` and
|
|
9
|
+
* relied on Docker's implicit pull. The published package is private, so every
|
|
10
|
+
* Formal AI task died with the raw daemon dump:
|
|
11
|
+
*
|
|
12
|
+
* ```text
|
|
13
|
+
* Command failed: docker run --detach --name hive-mind-formal-ai …
|
|
14
|
+
* Unable to find image 'ghcr.io/link-assistant/formal-ai:0.339.1' locally
|
|
15
|
+
* docker: Error response from daemon: error from registry: unauthorized
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* Three separate defects made that fatal:
|
|
19
|
+
*
|
|
20
|
+
* 1. **No preflight.** The registry problem surfaced as a failed *container
|
|
21
|
+
* launch* rather than as a failed *image resolution*, so nothing could tell
|
|
22
|
+
* the operator what to do about it.
|
|
23
|
+
* 2. **No diagnosis.** `unauthorized` from a registry has exactly three causes
|
|
24
|
+
* (private package, missing/insufficient credentials, wrong reference) and
|
|
25
|
+
* each has a different fix. The raw dump named none of them.
|
|
26
|
+
* 3. **No alternative.** The Hive Mind images themselves bake
|
|
27
|
+
* `/usr/local/bin/formal-ai` at the same pinned version (`Dockerfile`,
|
|
28
|
+
* `Dockerfile.dind`: `cargo install formal-ai --version ${FORMAL_AI_VERSION}
|
|
29
|
+
* --locked`), and that image is already present on the host because every
|
|
30
|
+
* isolated task runs from it. A registry outage therefore never had to stop
|
|
31
|
+
* Formal AI work at all.
|
|
32
|
+
*
|
|
33
|
+
* ## Contract
|
|
34
|
+
*
|
|
35
|
+
* - `HIVE_MIND_FORMAL_AI_IMAGE` is an **exact** operator pin: it is the only
|
|
36
|
+
* candidate, and if it cannot be resolved the acquire fails. An operator who
|
|
37
|
+
* names an image means that image.
|
|
38
|
+
* - Otherwise the published image is preferred, and the local Hive Mind image is
|
|
39
|
+
* the fallback. The fallback is only *used*; it is never pulled, because its
|
|
40
|
+
* whole point is that it is already on the host.
|
|
41
|
+
* - Nothing here downgrades a Formal AI task to another model. Issue #2146's
|
|
42
|
+
* fail-closed rule still holds: when no candidate resolves, the task is
|
|
43
|
+
* refused with an actionable message.
|
|
44
|
+
*
|
|
45
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2154
|
|
46
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2146
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
import { execFile } from 'node:child_process';
|
|
50
|
+
import { promisify } from 'node:util';
|
|
51
|
+
|
|
52
|
+
import { FORMAL_AI_BOOTSTRAP_VERSION } from './formal-ai-version.lib.mjs';
|
|
53
|
+
import { getDockerIsolationImage } from './hive-mind-image.lib.mjs';
|
|
54
|
+
|
|
55
|
+
const execFileAsync = promisify(execFile);
|
|
56
|
+
|
|
57
|
+
/** Image published by every Formal AI release (`:latest` plus the bare version). */
|
|
58
|
+
export const FORMAL_AI_IMAGE_REPOSITORY = 'ghcr.io/link-assistant/formal-ai';
|
|
59
|
+
|
|
60
|
+
const DEFAULT_DOCKER_TIMEOUT_MS = 600_000;
|
|
61
|
+
|
|
62
|
+
/** Where each candidate came from, so logs and errors can explain themselves. */
|
|
63
|
+
export const FORMAL_AI_IMAGE_SOURCES = Object.freeze({
|
|
64
|
+
PINNED: 'operator-pin',
|
|
65
|
+
PUBLISHED: 'published-image',
|
|
66
|
+
HIVE_MIND: 'hive-mind-image',
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const errorText = error => error?.stderr?.toString?.().trim() || error?.stdout?.toString?.().trim() || error?.message || String(error);
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Classify a `docker pull` failure into the operator action that fixes it.
|
|
73
|
+
*
|
|
74
|
+
* Registry errors are famously interchangeable-looking; the distinction that
|
|
75
|
+
* matters in practice is that GHCR answers `unauthorized` for a package that
|
|
76
|
+
* *exists but is private* and `denied` for one that does not exist or that the
|
|
77
|
+
* presented token may not see.
|
|
78
|
+
*
|
|
79
|
+
* @param {string} message - Raw stderr from the failed pull.
|
|
80
|
+
* @returns {{kind: string, reason: string, remediation: string[]}}
|
|
81
|
+
*/
|
|
82
|
+
export const classifyDockerRegistryError = message => {
|
|
83
|
+
const text = String(message || '');
|
|
84
|
+
if (/unauthorized|authentication required|requires authentication/i.test(text)) {
|
|
85
|
+
return {
|
|
86
|
+
kind: 'unauthorized',
|
|
87
|
+
reason: 'the registry refused an anonymous or under-scoped pull',
|
|
88
|
+
remediation: ['make the package public (GHCR packages published by GITHUB_TOKEN are private by default), or', 'authenticate the daemon with a token carrying the `read:packages` scope: `echo $TOKEN | docker login ghcr.io -u <user> --password-stdin`, or', 'point HIVE_MIND_FORMAL_AI_IMAGE at an image this host can already pull'],
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (/denied|forbidden|insufficient_scope|permission_denied/i.test(text)) {
|
|
92
|
+
return {
|
|
93
|
+
kind: 'denied',
|
|
94
|
+
reason: 'the registry denied access to that reference',
|
|
95
|
+
remediation: ['check the repository name and tag exist', 'check the credentials in use carry the `read:packages` scope'],
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
if (/manifest unknown|not found|no such (?:image|manifest)/i.test(text)) {
|
|
99
|
+
return { kind: 'not-found', reason: 'the registry has no such tag', remediation: ['check the tag exists in the registry', 'pin a published tag with HIVE_MIND_FORMAL_AI_IMAGE'] };
|
|
100
|
+
}
|
|
101
|
+
if (/no space left on device/i.test(text)) {
|
|
102
|
+
return { kind: 'disk-full', reason: 'the daemon ran out of disk while unpacking the image', remediation: ['free disk space on the Docker data root, then retry'] };
|
|
103
|
+
}
|
|
104
|
+
if (/timeout|timed out|temporary failure|dial tcp|i\/o timeout|connection refused|network is unreachable|EOF/i.test(text)) {
|
|
105
|
+
return { kind: 'network', reason: 'the registry was unreachable', remediation: ['check network/proxy access to the registry, then retry'] };
|
|
106
|
+
}
|
|
107
|
+
return { kind: 'unknown', reason: 'the pull failed', remediation: ['inspect the daemon error above'] };
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Resolve the local Hive Mind image, which bakes `formal-ai` at the same pinned
|
|
112
|
+
* version as the published sidecar image and is already present on any host
|
|
113
|
+
* that runs Docker-isolated tasks.
|
|
114
|
+
*/
|
|
115
|
+
export const resolveFormalAiFallbackImage = (env = process.env) => String(env.HIVE_MIND_FORMAL_AI_FALLBACK_IMAGE || '').trim() || getDockerIsolationImage({ env });
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Ordered list of images to try, most preferred first.
|
|
119
|
+
*
|
|
120
|
+
* @param {object} [env]
|
|
121
|
+
* @returns {Array<{image: string, source: string, pullable: boolean}>}
|
|
122
|
+
*/
|
|
123
|
+
export const resolveFormalAiSidecarImageCandidates = (env = process.env) => {
|
|
124
|
+
const pinned = String(env.HIVE_MIND_FORMAL_AI_IMAGE || '').trim();
|
|
125
|
+
if (pinned) return [{ image: pinned, source: FORMAL_AI_IMAGE_SOURCES.PINNED, pullable: true }];
|
|
126
|
+
const candidates = [{ image: `${FORMAL_AI_IMAGE_REPOSITORY}:${FORMAL_AI_BOOTSTRAP_VERSION}`, source: FORMAL_AI_IMAGE_SOURCES.PUBLISHED, pullable: true }];
|
|
127
|
+
const fallback = resolveFormalAiFallbackImage(env);
|
|
128
|
+
// Never pulled: the fallback's value is that it is already on the host. A
|
|
129
|
+
// deployment that has to pull it would be pulling the *bigger* image.
|
|
130
|
+
if (fallback && fallback !== candidates[0].image) candidates.push({ image: fallback, source: FORMAL_AI_IMAGE_SOURCES.HIVE_MIND, pullable: false });
|
|
131
|
+
return candidates;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
/** The preferred image, kept for callers and logs that only need the headline reference. */
|
|
135
|
+
export const resolveFormalAiSidecarImage = (env = process.env) => resolveFormalAiSidecarImageCandidates(env)[0].image;
|
|
136
|
+
|
|
137
|
+
const inspectLocalImage = async (image, { run, timeoutMs }) => {
|
|
138
|
+
try {
|
|
139
|
+
const result = await run('docker', ['image', 'inspect', image, '--format', '{{.Id}}'], { encoding: 'utf8', timeout: timeoutMs });
|
|
140
|
+
return String(result?.stdout ?? '').trim() || null;
|
|
141
|
+
} catch {
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Render the aggregated failure so an operator reading the Telegram reply or the
|
|
148
|
+
* bot log knows the cause and the fix without opening a shell.
|
|
149
|
+
*/
|
|
150
|
+
const describeFailure = attempts => {
|
|
151
|
+
const lines = ['No Formal AI image could be resolved, so the sidecar was not started.'];
|
|
152
|
+
for (const attempt of attempts) {
|
|
153
|
+
if (attempt.source === FORMAL_AI_IMAGE_SOURCES.HIVE_MIND && attempt.kind === 'absent') {
|
|
154
|
+
lines.push(`• ${attempt.image} (local Hive Mind image, fallback): not present on this host — Docker-isolated tasks would have to pull it too.`);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
lines.push(`• ${attempt.image} (${attempt.source}): ${attempt.reason}${attempt.error ? ` — ${attempt.error}` : ''}`);
|
|
158
|
+
for (const step of attempt.remediation ?? []) lines.push(` → ${step}`);
|
|
159
|
+
}
|
|
160
|
+
lines.push('Formal AI tasks fail closed by design (issue #2146): Hive Mind will not silently run them on another model.');
|
|
161
|
+
return lines.join('\n');
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Return the first candidate image that is usable on this host, pulling only the
|
|
166
|
+
* pullable ones and never throwing for a candidate that simply is not there.
|
|
167
|
+
*
|
|
168
|
+
* @param {object} params
|
|
169
|
+
* @param {object} [params.env]
|
|
170
|
+
* @param {Function} [params.run] - `execFile`-shaped seam, so tests can drive a fake daemon.
|
|
171
|
+
* @param {number} [params.timeoutMs]
|
|
172
|
+
* @param {Function|null} [params.log]
|
|
173
|
+
* @param {boolean} [params.verbose]
|
|
174
|
+
* @param {boolean} [params.pull] - Set false to accept only images already on the host.
|
|
175
|
+
* @returns {Promise<{image: string, source: string, digest: string|null, pulled: boolean, attempts: object[]}>}
|
|
176
|
+
* @throws {Error} When no candidate resolves; the message names every attempt and its fix.
|
|
177
|
+
*/
|
|
178
|
+
export const ensureFormalAiSidecarImage = async ({ env = process.env, run = execFileAsync, timeoutMs = DEFAULT_DOCKER_TIMEOUT_MS, log = null, verbose = false, pull = true, candidates = resolveFormalAiSidecarImageCandidates(env) } = {}) => {
|
|
179
|
+
const attempts = [];
|
|
180
|
+
|
|
181
|
+
for (const candidate of candidates) {
|
|
182
|
+
const localDigest = await inspectLocalImage(candidate.image, { run, timeoutMs });
|
|
183
|
+
if (localDigest) {
|
|
184
|
+
if (verbose && log) await log(`[VERBOSE] formal-ai-image: using '${candidate.image}' (${candidate.source}) already present locally, digest=${localDigest}`);
|
|
185
|
+
return { image: candidate.image, source: candidate.source, digest: localDigest, pulled: false, attempts };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (!candidate.pullable || !pull) {
|
|
189
|
+
attempts.push({ ...candidate, kind: 'absent', reason: 'not present locally and not pulled', error: null, remediation: [] });
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
if (log) await log(`⬇️ Pulling the Formal AI sidecar image ${candidate.image}`);
|
|
194
|
+
try {
|
|
195
|
+
await run('docker', ['pull', candidate.image], { encoding: 'utf8', timeout: timeoutMs });
|
|
196
|
+
} catch (error) {
|
|
197
|
+
const message = errorText(error);
|
|
198
|
+
const diagnosis = classifyDockerRegistryError(message);
|
|
199
|
+
attempts.push({ ...candidate, kind: diagnosis.kind, reason: diagnosis.reason, error: message, remediation: diagnosis.remediation });
|
|
200
|
+
if (log) await log(`⚠️ Could not pull ${candidate.image}: ${diagnosis.reason} (${diagnosis.kind}). ${message}`);
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const digest = await inspectLocalImage(candidate.image, { run, timeoutMs });
|
|
205
|
+
if (verbose && log) await log(`[VERBOSE] formal-ai-image: pulled '${candidate.image}' (${candidate.source}), digest=${digest ?? 'unknown'}`);
|
|
206
|
+
return { image: candidate.image, source: candidate.source, digest, pulled: true, attempts };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
const error = new Error(describeFailure(attempts));
|
|
210
|
+
error.formalAiImageAttempts = attempts;
|
|
211
|
+
throw error;
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
export default {
|
|
215
|
+
FORMAL_AI_IMAGE_REPOSITORY,
|
|
216
|
+
FORMAL_AI_IMAGE_SOURCES,
|
|
217
|
+
classifyDockerRegistryError,
|
|
218
|
+
ensureFormalAiSidecarImage,
|
|
219
|
+
resolveFormalAiFallbackImage,
|
|
220
|
+
resolveFormalAiSidecarImage,
|
|
221
|
+
resolveFormalAiSidecarImageCandidates,
|
|
222
|
+
};
|
|
@@ -32,7 +32,14 @@ export const acquireFormalAiSidecarForTask = async ({ backend, args = [], model
|
|
|
32
32
|
try {
|
|
33
33
|
return { sidecar: await acquire({ sessionId, tool, model, env, verbose, log }), error: null };
|
|
34
34
|
} catch (error) {
|
|
35
|
-
|
|
35
|
+
const message = `Formal AI sidecar could not be started, so the task was not launched (issue #2146): ${error?.message || error}`;
|
|
36
|
+
// Issue #2154: this used to be returned and nothing else. The reply went to
|
|
37
|
+
// Telegram, the session was untracked, and the bot log showed only the
|
|
38
|
+
// untracking — so the operator could see that a task had vanished but never
|
|
39
|
+
// why. Every refusal is now on the record, with the session UUID that names
|
|
40
|
+
// the task in `$ --list` and in the session store.
|
|
41
|
+
console.error(`[formal-ai-isolation] Session ${sessionId}: ${message}`);
|
|
42
|
+
return { sidecar: null, error: message };
|
|
36
43
|
}
|
|
37
44
|
};
|
|
38
45
|
|
|
@@ -46,7 +46,8 @@ import fs from 'node:fs';
|
|
|
46
46
|
import path from 'node:path';
|
|
47
47
|
import { promisify } from 'node:util';
|
|
48
48
|
|
|
49
|
-
import {
|
|
49
|
+
import { FORMAL_AI_MINIMUM_VERSION, isFormalAiVersionAtLeast } from './formal-ai-version.lib.mjs';
|
|
50
|
+
import { ensureFormalAiSidecarImage, resolveFormalAiSidecarImage } from './formal-ai-image.lib.mjs';
|
|
50
51
|
import { isFormalAiModel } from './formal-ai-model.lib.mjs';
|
|
51
52
|
import { getModelFromArgs } from './model-args.lib.mjs';
|
|
52
53
|
import { resolveBotStateDir } from './session-store.lib.mjs';
|
|
@@ -77,8 +78,14 @@ export const FORMAL_AI_SIDECAR_PORT = 8080;
|
|
|
77
78
|
export const FORMAL_AI_MEMORY_MOUNT = '/home/box/.formal-ai';
|
|
78
79
|
export const FORMAL_AI_MEMORY_PATH = `${FORMAL_AI_MEMORY_MOUNT}/memory.lino`;
|
|
79
80
|
|
|
80
|
-
/**
|
|
81
|
-
|
|
81
|
+
/**
|
|
82
|
+
* Image published by every Formal AI release (`:latest` plus the bare version).
|
|
83
|
+
*
|
|
84
|
+
* Re-exported from `formal-ai-image.lib.mjs`, which owns image resolution since
|
|
85
|
+
* issue #2154 taught us that "which image" and "is it actually pullable" are the
|
|
86
|
+
* same question.
|
|
87
|
+
*/
|
|
88
|
+
export { FORMAL_AI_IMAGE_REPOSITORY, resolveFormalAiSidecarImage, resolveFormalAiSidecarImageCandidates } from './formal-ai-image.lib.mjs';
|
|
82
89
|
|
|
83
90
|
/** Applied to the sidecar, its network and its volume so reconciliation can find them. */
|
|
84
91
|
export const FORMAL_AI_SIDECAR_LABEL = 'com.link-assistant.hive-mind.formal-ai';
|
|
@@ -86,6 +93,9 @@ export const FORMAL_AI_SIDECAR_LABEL = 'com.link-assistant.hive-mind.formal-ai';
|
|
|
86
93
|
const STATE_FILE_NAME = 'formal-ai-sidecar.json';
|
|
87
94
|
const SIDECAR_LOCK_NAME = 'formal-ai-sidecar';
|
|
88
95
|
const DEFAULT_DOCKER_TIMEOUT_MS = 120_000;
|
|
96
|
+
// Pulling a sidecar image is the one Docker call that legitimately takes many
|
|
97
|
+
// minutes, so it gets its own budget instead of the general command timeout.
|
|
98
|
+
const DEFAULT_IMAGE_TIMEOUT_MS = 600_000;
|
|
89
99
|
const DEFAULT_HEALTH_ATTEMPTS = 60;
|
|
90
100
|
const DEFAULT_HEALTH_DELAY_MS = 1000;
|
|
91
101
|
|
|
@@ -107,9 +117,6 @@ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
|
107
117
|
*/
|
|
108
118
|
export const isFormalAiTask = ({ args = [], model = null } = {}) => isFormalAiModel(model || getModelFromArgs(args));
|
|
109
119
|
|
|
110
|
-
/** Resolve the image the sidecar boots with. Operators may pin their own build. */
|
|
111
|
-
export const resolveFormalAiSidecarImage = (env = process.env) => String(env.HIVE_MIND_FORMAL_AI_IMAGE || '').trim() || `${FORMAL_AI_IMAGE_REPOSITORY}:${FORMAL_AI_BOOTSTRAP_VERSION}`;
|
|
112
|
-
|
|
113
120
|
/** Build the endpoint origin for a host name or address. */
|
|
114
121
|
export const buildFormalAiSidecarBaseUrl = (host = FORMAL_AI_SIDECAR_NETWORK_ALIAS) => `http://${host}:${FORMAL_AI_SIDECAR_PORT}`;
|
|
115
122
|
|
|
@@ -433,18 +440,14 @@ export const stopFormalAiSidecar = async ({ env = process.env, fsImpl = fs, run
|
|
|
433
440
|
* Must be called *before* the task container's command is allowed to run.
|
|
434
441
|
* Returns the endpoint the task should be pointed at.
|
|
435
442
|
*/
|
|
436
|
-
export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = null, env = process.env, fsImpl = fs, run = execFileAsync, timeoutMs, log = null, verbose = false, now = () => new Date(), healthAttempts, healthDelayMs, sleepImpl = sleep, lockOptions = {} } = {}) => {
|
|
443
|
+
export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = null, env = process.env, fsImpl = fs, run = execFileAsync, timeoutMs, imageTimeoutMs = DEFAULT_IMAGE_TIMEOUT_MS, log = null, verbose = false, now = () => new Date(), healthAttempts, healthDelayMs, sleepImpl = sleep, lockOptions = {} } = {}) => {
|
|
437
444
|
if (!sessionId) throw new Error('acquireFormalAiSidecar requires a sessionId');
|
|
438
445
|
|
|
439
446
|
return withFormalAiSidecarLock(
|
|
440
447
|
async () => {
|
|
441
|
-
const image = resolveFormalAiSidecarImage(env);
|
|
442
448
|
const state = readFormalAiSidecarState({ env, fsImpl });
|
|
443
449
|
const leases = await reconcileLeases(state.leases, { run, timeoutMs, log, verbose });
|
|
444
450
|
|
|
445
|
-
await ensureFormalAiNetwork({ run, timeoutMs, log, verbose });
|
|
446
|
-
await ensureFormalAiMemoryVolume({ image, run, timeoutMs, log, verbose });
|
|
447
|
-
|
|
448
451
|
let container = await inspectDockerContainer(FORMAL_AI_SIDECAR_CONTAINER_NAME, { run, timeoutMs });
|
|
449
452
|
if (container.exists && !container.running) {
|
|
450
453
|
// A stopped container may predate an image change; recreate instead of
|
|
@@ -453,8 +456,18 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
|
|
|
453
456
|
container = { exists: false, running: false, image: null, imageDigest: null };
|
|
454
457
|
}
|
|
455
458
|
|
|
459
|
+
// Resolve the image *before* anything shells out with it. Until issue
|
|
460
|
+
// #2154 the reference went straight into `docker run`, so a registry that
|
|
461
|
+
// refused the pull surfaced as an unreadable `Command failed: docker run …`
|
|
462
|
+
// dump and the task died even though a usable image sat on the host.
|
|
463
|
+
const resolved = container.exists && container.image ? { image: container.image, source: 'running-sidecar', pulled: false } : await ensureFormalAiSidecarImage({ env, run, timeoutMs: imageTimeoutMs, log, verbose });
|
|
464
|
+
const image = resolved.image;
|
|
465
|
+
|
|
466
|
+
await ensureFormalAiNetwork({ run, timeoutMs, log, verbose });
|
|
467
|
+
await ensureFormalAiMemoryVolume({ image, run, timeoutMs, log, verbose });
|
|
468
|
+
|
|
456
469
|
if (!container.exists) {
|
|
457
|
-
if (log) await log(`🧠 Starting the Formal AI sidecar (${image}) on the internal network '${FORMAL_AI_SIDECAR_NETWORK_NAME}'`);
|
|
470
|
+
if (log) await log(`🧠 Starting the Formal AI sidecar (${image}, ${resolved.source}) on the internal network '${FORMAL_AI_SIDECAR_NETWORK_NAME}'`);
|
|
458
471
|
await dockerText(run, buildFormalAiSidecarRunArgs({ image, env }), { timeoutMs });
|
|
459
472
|
container = await inspectDockerContainer(FORMAL_AI_SIDECAR_CONTAINER_NAME, { run, timeoutMs });
|
|
460
473
|
}
|
|
@@ -466,12 +479,21 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
|
|
|
466
479
|
throw new Error(`Formal AI sidecar '${FORMAL_AI_SIDECAR_CONTAINER_NAME}' did not become healthy: ${health.error}`);
|
|
467
480
|
}
|
|
468
481
|
|
|
482
|
+
// The sidecar may now boot from any of several images (issue #2154), so
|
|
483
|
+
// the version floor is enforced against what the process actually reports
|
|
484
|
+
// rather than assumed from the tag. A too-old binary would answer /health
|
|
485
|
+
// and then fail on the agent-mode API the tasks depend on.
|
|
486
|
+
const reportedVersion = health.health?.version ?? null;
|
|
487
|
+
if (reportedVersion && !isFormalAiVersionAtLeast(reportedVersion, FORMAL_AI_MINIMUM_VERSION)) {
|
|
488
|
+
throw new Error(`Formal AI sidecar image ${image} (${resolved.source}) runs formal-ai ${reportedVersion}, but Hive Mind requires >= ${FORMAL_AI_MINIMUM_VERSION}. Rebuild or repin the image (HIVE_MIND_FORMAL_AI_IMAGE) before running Formal AI tasks.`);
|
|
489
|
+
}
|
|
490
|
+
|
|
469
491
|
const address = await readFormalAiSidecarAddress({ run, timeoutMs });
|
|
470
492
|
const acquiredAt = now().toISOString();
|
|
471
493
|
const nextLeases = [...leases.filter(lease => lease.sessionId !== sessionId), { sessionId, tool, model, acquiredAt }];
|
|
472
494
|
writeFormalAiSidecarState({ ...state, image: container.image || image, imageDigest: container.imageDigest, startedAt: state.startedAt || acquiredAt, leases: nextLeases }, { env, fsImpl });
|
|
473
495
|
|
|
474
|
-
if (verbose && log) await log(`[VERBOSE] formal-ai-sidecar: lease '${sessionId}' acquired (${nextLeases.length} active), image=${container.image || image}, digest=${container.imageDigest ?? 'unknown'}, address=${address ?? 'unknown'}, formal-ai=${health.health?.version ?? 'unknown'}, memory schema=${health.health?.memory?.schema_version ?? 'unknown'}`);
|
|
496
|
+
if (verbose && log) await log(`[VERBOSE] formal-ai-sidecar: lease '${sessionId}' acquired (${nextLeases.length} active), image=${container.image || image} (${resolved.source}), digest=${container.imageDigest ?? 'unknown'}, address=${address ?? 'unknown'}, formal-ai=${health.health?.version ?? 'unknown'}, memory schema=${health.health?.memory?.schema_version ?? 'unknown'}`);
|
|
475
497
|
|
|
476
498
|
return {
|
|
477
499
|
address,
|
|
@@ -482,6 +504,7 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
|
|
|
482
504
|
containerName: FORMAL_AI_SIDECAR_CONTAINER_NAME,
|
|
483
505
|
memoryVolume: FORMAL_AI_MEMORY_VOLUME_NAME,
|
|
484
506
|
image: container.image || image,
|
|
507
|
+
imageSource: resolved.source,
|
|
485
508
|
imageDigest: container.imageDigest,
|
|
486
509
|
health: health.health,
|
|
487
510
|
leaseCount: nextLeases.length,
|
|
@@ -33,6 +33,7 @@ import { execFile } from 'node:child_process';
|
|
|
33
33
|
import fs from 'node:fs';
|
|
34
34
|
import { promisify } from 'node:util';
|
|
35
35
|
|
|
36
|
+
import { classifyDockerRegistryError } from './formal-ai-image.lib.mjs';
|
|
36
37
|
import { FORMAL_AI_IMAGE_REPOSITORY, FORMAL_AI_MEMORY_MOUNT, FORMAL_AI_MEMORY_PATH, FORMAL_AI_MEMORY_VOLUME_NAME, FORMAL_AI_SIDECAR_CONTAINER_NAME, buildFormalAiSidecarRunArgs, ensureFormalAiMemoryVolume, ensureFormalAiNetwork, inspectDockerContainer, readDockerImageDigest, readFormalAiSidecarState, reconcileFormalAiSidecar, stopFormalAiSidecar, waitForFormalAiSidecarHealth, withFormalAiSidecarLock, writeFormalAiSidecarState } from './formal-ai-sidecar.lib.mjs';
|
|
37
38
|
|
|
38
39
|
const execFileAsync = promisify(execFile);
|
|
@@ -207,8 +208,19 @@ export const updateFormalAiSidecarWhenIdle = async ({ env = process.env, fsImpl
|
|
|
207
208
|
await dockerText(run, ['pull', '--quiet', image], { timeoutMs: pullTimeoutMs });
|
|
208
209
|
} catch (error) {
|
|
209
210
|
const message = error?.stderr?.toString?.().trim() || error?.message || String(error);
|
|
210
|
-
|
|
211
|
-
|
|
211
|
+
// Issue #2154: this warning fired every ~5 minutes for hours ("Could not
|
|
212
|
+
// pull … unauthorized") without ever saying that the registry was
|
|
213
|
+
// refusing us, and the operator only learned of it when a Formal AI task
|
|
214
|
+
// failed to start. A permanent refusal (unauthorized/denied/not-found)
|
|
215
|
+
// is a configuration fault, not a transient blip: name it, say how to
|
|
216
|
+
// fix it, and report the classification to the caller.
|
|
217
|
+
const classification = classifyDockerRegistryError(message);
|
|
218
|
+
const permanent = ['unauthorized', 'denied', 'not-found'].includes(classification.kind);
|
|
219
|
+
if (log) {
|
|
220
|
+
const remediation = classification.remediation.length ? ` Fix it by one of: ${classification.remediation.join('; ')}.` : '';
|
|
221
|
+
await log(`${permanent ? '🚨' : '⚠️'} Could not pull ${image} — ${classification.reason} (${classification.kind}); keeping the current Formal AI image: ${message}${permanent ? remediation : ''}`);
|
|
222
|
+
}
|
|
223
|
+
return { status: 'failed', stage: 'pull', image, error: message, classification: classification.kind, permanent, remediation: classification.remediation };
|
|
212
224
|
}
|
|
213
225
|
|
|
214
226
|
const pulledDigest = await readDockerImageDigest(image, { run, timeoutMs });
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the Hive Mind container image references shared by the Docker
|
|
3
|
+
* isolation runner and the Formal AI sidecar.
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `isolation-runner.lib.mjs` for issue #2154: the Formal AI
|
|
6
|
+
* sidecar needs the very same reference to fall back to the locally present
|
|
7
|
+
* Hive Mind image (which bakes `/usr/local/bin/formal-ai`) when the published
|
|
8
|
+
* Formal AI image cannot be pulled, and importing the isolation runner from the
|
|
9
|
+
* sidecar would create an import cycle.
|
|
10
|
+
*
|
|
11
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2154
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export const HIVE_MIND_IMAGE_REPO = 'konard/hive-mind';
|
|
15
|
+
export const HIVE_MIND_DIND_IMAGE_REPO = 'konard/hive-mind-dind';
|
|
16
|
+
export const DEFAULT_HIVE_MIND_IMAGE_TAG = 'latest';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Resolve the tag used for the Docker isolation image.
|
|
20
|
+
*
|
|
21
|
+
* Release Docker images bake this env var from `HIVE_MIND_VERSION`, so a parent
|
|
22
|
+
* container started via `:latest` still launches child isolation containers from
|
|
23
|
+
* the same immutable release tag. Local/PR builds fall back to `latest`, and
|
|
24
|
+
* operators can override the tag explicitly when using custom images. Pinning
|
|
25
|
+
* matters for Docker-in-Docker deployments: the nested daemon starts with an
|
|
26
|
+
* empty image store, so a `:latest` digest drift from the host copy forces a
|
|
27
|
+
* fresh multi-gigabyte pull. See issue #1879.
|
|
28
|
+
*/
|
|
29
|
+
export function resolveDockerIsolationImageTag({ env = process.env } = {}) {
|
|
30
|
+
const explicit = String(env.HIVE_MIND_DOCKER_ISOLATION_IMAGE_TAG || '').trim();
|
|
31
|
+
return explicit || DEFAULT_HIVE_MIND_IMAGE_TAG;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Pick the Docker image used for `--isolation docker`.
|
|
36
|
+
*
|
|
37
|
+
* start-command defaults its Docker backend to a base OS image. Hive Mind needs
|
|
38
|
+
* an image with the same CLI/tooling baseline as the parent process instead.
|
|
39
|
+
*
|
|
40
|
+
* `HIVE_MIND_DOCKER_ISOLATION_IMAGE` is a full override (repo:tag). Otherwise
|
|
41
|
+
* the repo is chosen by image variant and the tag by
|
|
42
|
+
* `resolveDockerIsolationImageTag()`.
|
|
43
|
+
*/
|
|
44
|
+
export function getDockerIsolationImage({ env = process.env } = {}) {
|
|
45
|
+
if (env.HIVE_MIND_DOCKER_ISOLATION_IMAGE) return env.HIVE_MIND_DOCKER_ISOLATION_IMAGE;
|
|
46
|
+
const repo = String(env.HIVE_MIND_IMAGE_VARIANT || '').toLowerCase() === 'dind' ? HIVE_MIND_DIND_IMAGE_REPO : HIVE_MIND_IMAGE_REPO;
|
|
47
|
+
return `${repo}:${resolveDockerIsolationImageTag({ env })}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export default {
|
|
51
|
+
DEFAULT_HIVE_MIND_IMAGE_TAG,
|
|
52
|
+
HIVE_MIND_DIND_IMAGE_REPO,
|
|
53
|
+
HIVE_MIND_IMAGE_REPO,
|
|
54
|
+
getDockerIsolationImage,
|
|
55
|
+
resolveDockerIsolationImageTag,
|
|
56
|
+
};
|
|
@@ -21,6 +21,12 @@ import os from 'node:os';
|
|
|
21
21
|
import path from 'node:path';
|
|
22
22
|
import { isExecutingSessionStatus, isTerminalSessionStatus } from './session-status.lib.mjs';
|
|
23
23
|
import { acquireFormalAiSidecarForTask, attachFormalAiTaskContainer, releaseFormalAiSidecarForTask } from './formal-ai-isolation.lib.mjs';
|
|
24
|
+
// The image references live in their own module so the Formal AI sidecar can
|
|
25
|
+
// resolve the locally present Hive Mind image (which bakes `formal-ai`) without
|
|
26
|
+
// importing this runner and creating a cycle. Re-exported here because callers
|
|
27
|
+
// and tests have always reached them through the isolation runner. See #2154.
|
|
28
|
+
import { getDockerIsolationImage } from './hive-mind-image.lib.mjs';
|
|
29
|
+
export { getDockerIsolationImage, resolveDockerIsolationImageTag } from './hive-mind-image.lib.mjs';
|
|
24
30
|
let commandStreamDollarPromise = null;
|
|
25
31
|
async function getCommandStreamDollar() {
|
|
26
32
|
if (!commandStreamDollarPromise) {
|
|
@@ -43,9 +49,6 @@ async function getCommandStreamDollar() {
|
|
|
43
49
|
export { isExecutingSessionStatus, isTerminalSessionStatus, isKilledSessionStatus } from './session-status.lib.mjs';
|
|
44
50
|
// Valid isolation backends
|
|
45
51
|
const VALID_ISOLATION_BACKENDS = ['screen', 'tmux', 'docker'];
|
|
46
|
-
const HIVE_MIND_IMAGE_REPO = 'konard/hive-mind';
|
|
47
|
-
const HIVE_MIND_DIND_IMAGE_REPO = 'konard/hive-mind-dind';
|
|
48
|
-
const DEFAULT_HIVE_MIND_IMAGE_TAG = 'latest';
|
|
49
52
|
const DOCKER_CONTAINER_HOME = '/home/box';
|
|
50
53
|
const FORMAL_AI_COMPOSE_HOSTNAME = 'link-assistant-formal-ai';
|
|
51
54
|
// Default path where the host Docker socket is bind-mounted inside a DinD container so box's host-image passthrough can copy host images into the nested daemon. Matches box's own DIND_HOST_DOCKER_SOCK default. The deploy must mount it (`-v /var/run/docker.sock:/var/run/host-docker.sock:ro`) or the nested daemon starts empty and the first isolated task pulls the full, multi-gigabyte image. See issue #1914.
|
|
@@ -96,36 +99,6 @@ function maybeAddMount(mounts, source, target, existsSync) {
|
|
|
96
99
|
if (!existsSync(source)) return;
|
|
97
100
|
mounts.push({ source, target });
|
|
98
101
|
}
|
|
99
|
-
/**
|
|
100
|
-
* Resolve the tag used for the Docker isolation image.
|
|
101
|
-
*
|
|
102
|
-
* Release Docker images bake this env var from `HIVE_MIND_VERSION`, so a parent
|
|
103
|
-
* container started via `:latest` still launches child isolation containers from
|
|
104
|
-
* the same immutable release tag. Local/PR builds fall back to `latest`, and
|
|
105
|
-
* operators can override the tag explicitly when using custom images. Pinning
|
|
106
|
-
* matters for Docker-in-Docker deployments: the nested daemon starts with an
|
|
107
|
-
* empty image store, so a `:latest` digest drift from the host copy forces a
|
|
108
|
-
* fresh multi-gigabyte pull. See issue #1879.
|
|
109
|
-
*/
|
|
110
|
-
export function resolveDockerIsolationImageTag({ env = process.env } = {}) {
|
|
111
|
-
const explicit = String(env.HIVE_MIND_DOCKER_ISOLATION_IMAGE_TAG || '').trim();
|
|
112
|
-
return explicit || DEFAULT_HIVE_MIND_IMAGE_TAG;
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Pick the Docker image used for `--isolation docker`.
|
|
116
|
-
*
|
|
117
|
-
* start-command defaults its Docker backend to a base OS image. Hive Mind needs
|
|
118
|
-
* an image with the same CLI/tooling baseline as the parent process instead.
|
|
119
|
-
*
|
|
120
|
-
* `HIVE_MIND_DOCKER_ISOLATION_IMAGE` is a full override (repo:tag). Otherwise
|
|
121
|
-
* the repo is chosen by image variant and the tag by
|
|
122
|
-
* `resolveDockerIsolationImageTag()`.
|
|
123
|
-
*/
|
|
124
|
-
export function getDockerIsolationImage({ env = process.env } = {}) {
|
|
125
|
-
if (env.HIVE_MIND_DOCKER_ISOLATION_IMAGE) return env.HIVE_MIND_DOCKER_ISOLATION_IMAGE;
|
|
126
|
-
const repo = String(env.HIVE_MIND_IMAGE_VARIANT || '').toLowerCase() === 'dind' ? HIVE_MIND_DIND_IMAGE_REPO : HIVE_MIND_IMAGE_REPO;
|
|
127
|
-
return `${repo}:${resolveDockerIsolationImageTag({ env })}`;
|
|
128
|
-
}
|
|
129
102
|
/**
|
|
130
103
|
* Resolve the path where the host Docker socket is expected to be mounted inside
|
|
131
104
|
* a DinD container. box's entrypoint reads this socket to copy host images into
|
|
@@ -296,6 +269,48 @@ async function runStartCommand(binPath, startCommandArgs) {
|
|
|
296
269
|
export function generateSessionId() {
|
|
297
270
|
return crypto.randomUUID();
|
|
298
271
|
}
|
|
272
|
+
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
273
|
+
/**
|
|
274
|
+
* Extract start-command's own execution UUID from a launch banner.
|
|
275
|
+
*
|
|
276
|
+
* Issue #2154: an isolated task has two UUIDs. Hive Mind generates the session
|
|
277
|
+
* name and passes it as `--session` (it also becomes the container name);
|
|
278
|
+
* start-command mints a separate execution UUID and prints it as the `session`
|
|
279
|
+
* field of its launch banner:
|
|
280
|
+
*
|
|
281
|
+
* ```
|
|
282
|
+
* │ session edc7b051-e12f-4f7b-b677-c885f3208407
|
|
283
|
+
* │ container 0a3627ef-f1f1-4801-a073-3678b9453db7
|
|
284
|
+
* ```
|
|
285
|
+
*
|
|
286
|
+
* `$ --list` shows the execution UUID, while Telegram and the logs showed the
|
|
287
|
+
* session UUID, so the two views could not be joined — which is why three
|
|
288
|
+
* refused tasks and two healthy ones looked equally unaccounted for. Returning
|
|
289
|
+
* it lets the caller record both.
|
|
290
|
+
*
|
|
291
|
+
* Only a well-formed UUID is returned; a banner we do not recognise yields
|
|
292
|
+
* null rather than a guess, because a wrong correlation is worse than none.
|
|
293
|
+
*
|
|
294
|
+
* @param {string} output - Raw stdout from the detached `$` launch
|
|
295
|
+
* @returns {string|null}
|
|
296
|
+
*/
|
|
297
|
+
export function parseStartCommandExecutionUuid(output) {
|
|
298
|
+
const raw = (output || '').trim();
|
|
299
|
+
if (!raw) return null;
|
|
300
|
+
try {
|
|
301
|
+
const parsed = JSON.parse(raw);
|
|
302
|
+
const data = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
303
|
+
const uuid = data?.uuid || data?.session || null;
|
|
304
|
+
if (typeof uuid === 'string' && UUID_PATTERN.test(uuid.trim())) return uuid.trim();
|
|
305
|
+
} catch {
|
|
306
|
+
// Human-readable banner — fall through.
|
|
307
|
+
}
|
|
308
|
+
// The banner is box-drawn (`│ session <uuid>`); tolerate the prefix, an
|
|
309
|
+
// ASCII `|`, or no prefix at all.
|
|
310
|
+
const match = raw.match(/^[\s│|]*session\s+([^\s]+)\s*$/im);
|
|
311
|
+
const candidate = match?.[1]?.trim();
|
|
312
|
+
return candidate && UUID_PATTERN.test(candidate) ? candidate : null;
|
|
313
|
+
}
|
|
299
314
|
/**
|
|
300
315
|
* Parse output from `$ --status <session>`.
|
|
301
316
|
*
|
|
@@ -544,23 +559,22 @@ async function logDockerIsolationPostLaunchDiagnostics(sessionId, env = process.
|
|
|
544
559
|
export async function executeWithIsolation(command, args, options = {}) {
|
|
545
560
|
const { backend, verbose = false } = options;
|
|
546
561
|
const sessionId = options.sessionId || generateSessionId();
|
|
562
|
+
// Issue #2154: a launch that never produced a container left no trace in the
|
|
563
|
+
// bot log — the reply went to Telegram and the log jumped straight to
|
|
564
|
+
// "session untracked", so an operator could see that tasks were disappearing
|
|
565
|
+
// but not why, and could not even name them. Every unsuccessful return from
|
|
566
|
+
// this function now records the session UUID (the same one `$ --list` and the
|
|
567
|
+
// session store use) together with the reason.
|
|
568
|
+
const failLaunch = (error, extra = {}) => {
|
|
569
|
+
console.error(`[isolation-runner] Session ${sessionId} was not launched (backend=${backend}, tool=${options.tool ?? 'claude'}, model=${options.model ?? 'default'}): ${error}`);
|
|
570
|
+
return { success: false, sessionId, output: '', error, ...extra };
|
|
571
|
+
};
|
|
547
572
|
if (!VALID_ISOLATION_BACKENDS.includes(backend)) {
|
|
548
|
-
return {
|
|
549
|
-
success: false,
|
|
550
|
-
sessionId,
|
|
551
|
-
output: '',
|
|
552
|
-
error: `Invalid isolation backend: '${backend}'. Must be one of: ${VALID_ISOLATION_BACKENDS.join(', ')}`,
|
|
553
|
-
};
|
|
573
|
+
return failLaunch(`Invalid isolation backend: '${backend}'. Must be one of: ${VALID_ISOLATION_BACKENDS.join(', ')}`);
|
|
554
574
|
}
|
|
555
575
|
const binPath = await findStartCommandBinary();
|
|
556
576
|
if (!binPath) {
|
|
557
|
-
return {
|
|
558
|
-
success: false,
|
|
559
|
-
sessionId,
|
|
560
|
-
output: '',
|
|
561
|
-
warning: '⚠️ WARNING: start-command ($) not found in PATH\nPlease install: npm install -g start-command',
|
|
562
|
-
error: 'start-command ($) not found',
|
|
563
|
-
};
|
|
577
|
+
return failLaunch('start-command ($) not found', { warning: '⚠️ WARNING: start-command ($) not found in PATH\nPlease install: npm install -g start-command' });
|
|
564
578
|
}
|
|
565
579
|
if (verbose) {
|
|
566
580
|
console.log(`[VERBOSE] isolation-runner: Using $ binary at: ${binPath}`);
|
|
@@ -573,7 +587,7 @@ export async function executeWithIsolation(command, args, options = {}) {
|
|
|
573
587
|
// fails. Fail closed — a Formal AI task must never start without Formal AI.
|
|
574
588
|
const hostEnv = options.env || process.env;
|
|
575
589
|
const { sidecar, error: sidecarError } = await acquireFormalAiSidecarForTask({ backend, args, model: options.model ?? null, tool: options.tool ?? null, sessionId, env: hostEnv, verbose });
|
|
576
|
-
if (sidecarError) return
|
|
590
|
+
if (sidecarError) return failLaunch(sidecarError);
|
|
577
591
|
const taskEnv = sidecar ? { ...hostEnv, HIVE_MIND_FORMAL_AI_BASE_URL: sidecar.baseUrl } : hostEnv;
|
|
578
592
|
const effectiveOptions =
|
|
579
593
|
backend === 'docker'
|
|
@@ -628,7 +642,7 @@ export async function executeWithIsolation(command, args, options = {}) {
|
|
|
628
642
|
if (formalAiAttachError) await removeDockerContainer(sessionId, verbose);
|
|
629
643
|
await releaseFormalAiSidecarForTask({ sidecar, sessionId, env: hostEnv, verbose });
|
|
630
644
|
if (formalAiAttachError) {
|
|
631
|
-
return
|
|
645
|
+
return failLaunch(`Formal AI task container could not be attached to the internal Formal AI network, so the task was stopped instead of falling back to another model (issue #2146): ${formalAiAttachError}`, { output: result.output });
|
|
632
646
|
}
|
|
633
647
|
}
|
|
634
648
|
// Issue #1939: capture the freshly-launched docker session's reported status
|
|
@@ -639,19 +653,22 @@ export async function executeWithIsolation(command, args, options = {}) {
|
|
|
639
653
|
await logDockerIsolationPostLaunchDiagnostics(sessionId, options.env || process.env);
|
|
640
654
|
}
|
|
641
655
|
if (result.success) {
|
|
656
|
+
// Issue #2154: hand the caller start-command's own execution UUID as well.
|
|
657
|
+
// It is the identifier `$ --list` prints, so without it the bot and the
|
|
658
|
+
// session list cannot be joined by an operator.
|
|
659
|
+
const executionUuid = parseStartCommandExecutionUuid(result.output);
|
|
660
|
+
if (verbose) {
|
|
661
|
+
console.log(executionUuid ? `[VERBOSE] isolation-runner: start-command execution UUID for session ${sessionId}: ${executionUuid} (this is what '$ --list' shows)` : `[VERBOSE] isolation-runner: start-command reported no execution UUID for session ${sessionId}; '$ --list' cannot be correlated for this session`);
|
|
662
|
+
}
|
|
642
663
|
return {
|
|
643
664
|
success: true,
|
|
644
665
|
sessionId,
|
|
666
|
+
executionUuid,
|
|
645
667
|
output: result.output,
|
|
646
668
|
containerFilesystemStartBytes,
|
|
647
669
|
};
|
|
648
670
|
}
|
|
649
|
-
return {
|
|
650
|
-
success: false,
|
|
651
|
-
sessionId,
|
|
652
|
-
output: result.output,
|
|
653
|
-
error: result.error,
|
|
654
|
-
};
|
|
671
|
+
return failLaunch(result.error, { output: result.output });
|
|
655
672
|
}
|
|
656
673
|
/**
|
|
657
674
|
* Query the status of an isolated session via `$ --status <uuid>`
|
package/src/locales/en.lino
CHANGED
|
@@ -654,6 +654,7 @@ en
|
|
|
654
654
|
runner_also_failed "The runner also failed; its exit code is preserved for investigation."
|
|
655
655
|
killed "Work session {{reason}}{{exitSuffix}}"
|
|
656
656
|
stopped "Work session stopped by user{{requestedBy}}{{exitSuffix}}"
|
|
657
|
+
not_launched "The work session was not launched, so it has no log and is not listed by `--list`."
|
|
657
658
|
duration
|
|
658
659
|
label "Duration"
|
|
659
660
|
session
|
|
@@ -670,6 +671,8 @@ en
|
|
|
670
671
|
resumed_attempt "🔄 A new working session was started to recover from this kill (attempt {{attempt}}): {{sessionId}}"
|
|
671
672
|
isolation
|
|
672
673
|
label "Isolation"
|
|
674
|
+
execution
|
|
675
|
+
label "Execution"
|
|
673
676
|
error
|
|
674
677
|
executing
|
|
675
678
|
command "❌ Error executing {{commandName}} command"
|
package/src/locales/hi.lino
CHANGED
|
@@ -654,6 +654,7 @@ hi
|
|
|
654
654
|
runner_also_failed "रनर भी विफल हुआ; जाँच के लिए उसका exit code सुरक्षित रखा गया है।"
|
|
655
655
|
killed "कार्य सत्र रोका गया: {{reason}}{{exitSuffix}}"
|
|
656
656
|
stopped "कार्य सत्र उपयोगकर्ता द्वारा रोका गया{{requestedBy}}{{exitSuffix}}"
|
|
657
|
+
not_launched "कार्य सत्र शुरू ही नहीं हुआ, इसलिए उसका कोई log नहीं है और वह `--list` में नहीं दिखता।"
|
|
657
658
|
duration
|
|
658
659
|
label "अवधि"
|
|
659
660
|
session
|
|
@@ -670,6 +671,8 @@ hi
|
|
|
670
671
|
resumed_attempt "🔄 इस समाप्ति से पुनर्प्राप्ति हेतु नया कार्य सत्र शुरू किया गया (प्रयास {{attempt}}): {{sessionId}}"
|
|
671
672
|
isolation
|
|
672
673
|
label "Isolation"
|
|
674
|
+
execution
|
|
675
|
+
label "निष्पादन"
|
|
673
676
|
error
|
|
674
677
|
executing
|
|
675
678
|
command "❌ {{commandName}} command चलाने में त्रुटि"
|
package/src/locales/ru.lino
CHANGED
|
@@ -654,6 +654,7 @@ ru
|
|
|
654
654
|
runner_also_failed "Средство запуска также завершилось с ошибкой; код выхода сохранён для расследования."
|
|
655
655
|
killed "Рабочий сеанс остановлен: {{reason}}{{exitSuffix}}"
|
|
656
656
|
stopped "Рабочий сеанс остановлен пользователем{{requestedBy}}{{exitSuffix}}"
|
|
657
|
+
not_launched "Рабочий сеанс не был запущен, поэтому у него нет журнала и он не отображается в `--list`."
|
|
657
658
|
duration
|
|
658
659
|
label "Длительность"
|
|
659
660
|
session
|
|
@@ -670,6 +671,8 @@ ru
|
|
|
670
671
|
resumed_attempt "🔄 Запущена новая рабочая сессия для восстановления после этого завершения (попытка {{attempt}}): {{sessionId}}"
|
|
671
672
|
isolation
|
|
672
673
|
label "Изоляция"
|
|
674
|
+
execution
|
|
675
|
+
label "Запуск"
|
|
673
676
|
error
|
|
674
677
|
executing
|
|
675
678
|
command "❌ Ошибка выполнения команды {{commandName}}"
|
package/src/locales/zh.lino
CHANGED
|
@@ -654,6 +654,7 @@ zh
|
|
|
654
654
|
runner_also_failed "运行器也失败了;其退出代码已保留以供调查。"
|
|
655
655
|
killed "工作会话已终止:{{reason}}{{exitSuffix}}"
|
|
656
656
|
stopped "工作会话已由用户停止{{requestedBy}}{{exitSuffix}}"
|
|
657
|
+
not_launched "工作会话未启动,因此没有日志,也不会出现在 `--list` 中。"
|
|
657
658
|
duration
|
|
658
659
|
label "耗时"
|
|
659
660
|
session
|
|
@@ -670,6 +671,8 @@ zh
|
|
|
670
671
|
resumed_attempt "🔄 已启动新的工作会话以从此次终止中恢复(第 {{attempt}} 次尝试):{{sessionId}}"
|
|
671
672
|
isolation
|
|
672
673
|
label "隔离"
|
|
674
|
+
execution
|
|
675
|
+
label "执行"
|
|
673
676
|
error
|
|
674
677
|
executing
|
|
675
678
|
command "❌ 执行 {{commandName}} 命令时出错"
|
|
@@ -164,6 +164,9 @@ export function trackSession(sessionName, sessionInfo, verbose = false) {
|
|
|
164
164
|
url: sessionInfo.url || null,
|
|
165
165
|
command: sessionInfo.command || null,
|
|
166
166
|
sessionId: sessionInfo.sessionId || null,
|
|
167
|
+
// Issue #2154: `$ --list` prints start-command's execution UUID, not the
|
|
168
|
+
// session name. Logging both is what makes the two views joinable.
|
|
169
|
+
executionUuid: sessionInfo.executionUuid || null,
|
|
167
170
|
startTime: sessionInfo.startTime instanceof Date ? sessionInfo.startTime.toISOString() : sessionInfo.startTime || null,
|
|
168
171
|
});
|
|
169
172
|
}
|
|
@@ -230,24 +233,40 @@ export function markSessionStopRequested(sessionId, { requestedBy = null, verbos
|
|
|
230
233
|
* map and the durable store without emitting a `session_completed` audit event —
|
|
231
234
|
* the session never ran, so it has no exit code to record (issue #1946).
|
|
232
235
|
*
|
|
236
|
+
* Issue #2154: the durable structured log recorded only
|
|
237
|
+
* `session_untracked {"sessionName":…}`. Why the session disappeared a second
|
|
238
|
+
* after it was announced lived on untimestamped console lines, so an incident
|
|
239
|
+
* could not be reconstructed from the timestamped log alone. Callers now pass
|
|
240
|
+
* the reason and it is recorded with the event.
|
|
241
|
+
*
|
|
233
242
|
* @param {string} sessionName - Name/UUID of the session to drop
|
|
234
243
|
* @param {boolean} verbose - Whether to log verbose output
|
|
244
|
+
* @param {object} [details] - Extra context recorded with the `session_untracked` event
|
|
245
|
+
* @param {string} [details.reason] - Why the session was dropped (e.g. the launch error)
|
|
235
246
|
*/
|
|
236
|
-
export function untrackSession(sessionName, verbose = false) {
|
|
247
|
+
export function untrackSession(sessionName, verbose = false, details = {}) {
|
|
237
248
|
if (!sessionName) return;
|
|
238
249
|
const sessionInfo = activeSessions.get(sessionName) || null;
|
|
239
250
|
const existed = activeSessions.delete(sessionName);
|
|
251
|
+
const reason = typeof details?.reason === 'string' && details.reason.trim() ? details.reason.trim() : null;
|
|
240
252
|
if (verbose && existed) {
|
|
241
|
-
console.log(`[VERBOSE] Session ${sessionName} untracked (launch failed before it started)`);
|
|
253
|
+
console.log(`[VERBOSE] Session ${sessionName} untracked (launch failed before it started)${reason ? `: ${reason}` : ''}`);
|
|
242
254
|
}
|
|
243
255
|
if (sessionStore && isPersistableSession(sessionInfo)) {
|
|
244
256
|
try {
|
|
245
|
-
sessionStore.remove(sessionName, { status: 'launch-failed', exitCode: null });
|
|
257
|
+
sessionStore.remove(sessionName, { status: 'launch-failed', exitCode: null, reason });
|
|
246
258
|
} catch (error) {
|
|
247
259
|
console.error(`[session-monitor] Could not remove untracked session ${sessionName}: ${error.message}`);
|
|
248
260
|
}
|
|
249
261
|
}
|
|
250
|
-
logEvent('session_untracked', {
|
|
262
|
+
logEvent('session_untracked', {
|
|
263
|
+
sessionName,
|
|
264
|
+
reason,
|
|
265
|
+
url: sessionInfo?.url || null,
|
|
266
|
+
command: sessionInfo?.command || null,
|
|
267
|
+
tool: sessionInfo?.tool || null,
|
|
268
|
+
isolationBackend: sessionInfo?.isolationBackend || null,
|
|
269
|
+
});
|
|
251
270
|
}
|
|
252
271
|
/**
|
|
253
272
|
* Get the number of active sessions being tracked
|
|
@@ -646,6 +665,18 @@ export async function monitorSessions(bot, verbose = false, options = {}) {
|
|
|
646
665
|
sessionInfo.logPath = statusResult.logPath;
|
|
647
666
|
persistSessionSnapshot(sessionName, sessionInfo);
|
|
648
667
|
}
|
|
668
|
+
// Issue #2154: the same status record carries start-command's *execution*
|
|
669
|
+
// UUID — the only identifier `$ --list` prints. A session launched before
|
|
670
|
+
// this fix (or by a start-command whose banner we could not parse) has
|
|
671
|
+
// none, so backfill it here; otherwise that session stays impossible to
|
|
672
|
+
// find in the session list for its whole lifetime.
|
|
673
|
+
if (statusResult?.uuid && sessionInfo.executionUuid !== statusResult.uuid) {
|
|
674
|
+
if (verbose) {
|
|
675
|
+
console.log(`[VERBOSE] Session ${sessionName}: recorded start-command execution UUID ${statusResult.uuid} (this is what '$ --list' shows)`);
|
|
676
|
+
}
|
|
677
|
+
sessionInfo.executionUuid = statusResult.uuid;
|
|
678
|
+
persistSessionSnapshot(sessionName, sessionInfo);
|
|
679
|
+
}
|
|
649
680
|
} else {
|
|
650
681
|
// Issue #1586: Non-isolation screen sessions cannot reliably detect
|
|
651
682
|
// completion because start-screen keeps the screen alive via `exec bash`.
|
|
@@ -35,7 +35,10 @@ import path from 'node:path';
|
|
|
35
35
|
// with its exact original invocation plus `--resume <lastSessionId>`.
|
|
36
36
|
// `commandAlias` (#2109) preserves the Telegram spelling (`solve`, `codex`,
|
|
37
37
|
// `claude`, etc.) so a bot notification never suggests a terminal-only command.
|
|
38
|
-
|
|
38
|
+
// `executionUuid` (#2154) is start-command's own identifier for the execution —
|
|
39
|
+
// the one `$ --list` prints. It differs from `sessionId`, so persisting it is
|
|
40
|
+
// what lets a restarted bot still correlate its sessions with the session list.
|
|
41
|
+
const PERSISTABLE_FIELDS = ['chatId', 'messageId', 'startTime', 'url', 'command', 'commandAlias', 'isolationBackend', 'sessionId', 'executionUuid', 'containerFilesystemStartBytes', 'containerFilesystemLastBytes', 'containerFilesystemLastObservedAt', 'tool', 'infoBlock', 'urlContext', 'requesterUserId', 'showLimits', 'locale', 'logPath', 'args'];
|
|
39
42
|
|
|
40
43
|
/**
|
|
41
44
|
* Resolve the directory durable bot state is written to. Honors
|
|
@@ -213,7 +216,9 @@ export function createSessionStore(options = {}) {
|
|
|
213
216
|
delete sessions[sessionName];
|
|
214
217
|
writeSnapshotMap(sessions);
|
|
215
218
|
}
|
|
216
|
-
|
|
219
|
+
// Issue #2154: carry the reason (when the caller knows it) so the durable
|
|
220
|
+
// event log says why a session ended, not just that it did.
|
|
221
|
+
appendEvent('complete', sessionName, { status: meta.status ?? null, exitCode: meta.exitCode ?? null, reason: meta.reason ?? null });
|
|
217
222
|
log('debug', `Removed session ${sessionName} from snapshot`, meta);
|
|
218
223
|
},
|
|
219
224
|
|
|
@@ -2,7 +2,7 @@ import { spawn } from 'child_process';
|
|
|
2
2
|
import { describeChildExit } from './child-exit.lib.mjs';
|
|
3
3
|
import { promisify } from 'util';
|
|
4
4
|
import { exec as execCallback } from 'child_process';
|
|
5
|
-
import {
|
|
5
|
+
import { formatFailedLaunchMessage as defaultFormatFailedLaunchMessage } from './work-session-formatting.lib.mjs';
|
|
6
6
|
|
|
7
7
|
const exec = promisify(execCallback);
|
|
8
8
|
|
|
@@ -101,7 +101,7 @@ function executeWithCommand(startScreenCmd, command, args, verbose = false) {
|
|
|
101
101
|
* @returns {Function} executeAndUpdateMessage(ctx, startingMessage, commandName, args, infoBlock, perCommandIsolation, tool, urlContext, sessionExtras)
|
|
102
102
|
*/
|
|
103
103
|
export function buildExecuteAndUpdateMessage(deps) {
|
|
104
|
-
const { resolveIsolation, ISOLATION_BACKEND, isolationRunner, VERBOSE, executeStartScreen, trackSession, untrackSession, AUTO_WATCH_MESSAGE, startAutoTerminalWatchForSession, bot, formatExecutingWorkSessionMessage, formatStartingWorkSessionMessage } = deps;
|
|
104
|
+
const { resolveIsolation, ISOLATION_BACKEND, isolationRunner, VERBOSE, executeStartScreen, trackSession, untrackSession, AUTO_WATCH_MESSAGE, startAutoTerminalWatchForSession, bot, formatExecutingWorkSessionMessage, formatStartingWorkSessionMessage, formatFailedLaunchMessage = defaultFormatFailedLaunchMessage } = deps;
|
|
105
105
|
return async function executeAndUpdateMessage(ctx, startingMessage, commandName, args, infoBlock, perCommandIsolation = null, tool = 'claude', urlContext = null, { showLimits = false, limitsAtStart = null, locale = null, commandAlias = null } = {}) {
|
|
106
106
|
const { chat, message_id: msgId } = startingMessage;
|
|
107
107
|
const safeEdit = async text => {
|
|
@@ -130,14 +130,27 @@ export function buildExecuteAndUpdateMessage(deps) {
|
|
|
130
130
|
trackSession(session, sessionInfo, VERBOSE);
|
|
131
131
|
await safeEdit(formatStartingWorkSessionMessage({ sessionName: session, isolationBackend: iso.backend, infoBlock, locale }));
|
|
132
132
|
result = await iso.runner.executeWithIsolation(commandName, args, { backend: iso.backend, sessionId: session, tool, verbose: VERBOSE });
|
|
133
|
-
if (result.success && sessionInfo && Number.isFinite(result.containerFilesystemStartBytes)) {
|
|
134
|
-
sessionInfo.containerFilesystemStartBytes = result.containerFilesystemStartBytes;
|
|
133
|
+
if (result.success && sessionInfo && (Number.isFinite(result.containerFilesystemStartBytes) || result.executionUuid)) {
|
|
134
|
+
if (Number.isFinite(result.containerFilesystemStartBytes)) sessionInfo.containerFilesystemStartBytes = result.containerFilesystemStartBytes;
|
|
135
|
+
// Issue #2154: `$ --list` identifies executions by start-command's own
|
|
136
|
+
// UUID, not by the session name the bot shows. Keep both on the session
|
|
137
|
+
// so the two views can be joined — in the reply, in the structured log
|
|
138
|
+
// and in the durable snapshot after a restart.
|
|
139
|
+
if (result.executionUuid) sessionInfo.executionUuid = result.executionUuid;
|
|
135
140
|
trackSession(session, sessionInfo, VERBOSE);
|
|
136
141
|
}
|
|
137
142
|
if (!result.success) {
|
|
138
143
|
// The launch never produced a live container — drop the optimistic
|
|
139
144
|
// tracking so a phantom session is not monitored or resumed.
|
|
140
|
-
|
|
145
|
+
// Issue #2154: the untracking used to be the *only* trace of the
|
|
146
|
+
// failure anywhere outside Telegram. Record the UUID, the backend and
|
|
147
|
+
// the reason first, so the bot log explains why a session that was
|
|
148
|
+
// announced a second ago is gone and absent from `--list`.
|
|
149
|
+
const launchError = result.error || result.output || 'unknown error';
|
|
150
|
+
console.error(`[telegram-bot] ${commandName} session ${session} was not launched (isolation=${iso.backend}, tool=${tool}): ${launchError}`);
|
|
151
|
+
// The reason also goes into the structured `session_untracked` event, so
|
|
152
|
+
// the timestamped log explains the disappearance on its own.
|
|
153
|
+
if (typeof untrackSession === 'function') untrackSession(session, VERBOSE, { reason: launchError });
|
|
141
154
|
sessionInfo = undefined;
|
|
142
155
|
}
|
|
143
156
|
} else {
|
|
@@ -152,9 +165,15 @@ export function buildExecuteAndUpdateMessage(deps) {
|
|
|
152
165
|
}
|
|
153
166
|
if (result.warning) return safeEdit(`⚠️ ${result.warning}`);
|
|
154
167
|
if (result.success) {
|
|
155
|
-
await safeEdit(formatExecutingWorkSessionMessage({ sessionName: session, isolationBackend: iso?.backend || null, infoBlock, locale }));
|
|
168
|
+
await safeEdit(formatExecutingWorkSessionMessage({ sessionName: session, executionUuid: result.executionUuid || null, isolationBackend: iso?.backend || null, infoBlock, locale }));
|
|
156
169
|
if (AUTO_WATCH_MESSAGE && commandName === 'solve' && sessionInfo?.isolationBackend) await startAutoTerminalWatchForSession({ bot, ctx, sessionId: session, sessionInfo, verbose: VERBOSE });
|
|
157
|
-
} else
|
|
170
|
+
} else {
|
|
171
|
+
// Issue #2154: keep the session UUID in the failure reply. It is the only
|
|
172
|
+
// handle the operator has on the attempt, and this edit replaces the
|
|
173
|
+
// "🔄 Starting..." message that used to carry it.
|
|
174
|
+
if (!iso) console.error(`[telegram-bot] ${commandName} command failed to start (no isolation, tool=${tool}): ${result.error || result.output || 'unknown error'}`);
|
|
175
|
+
await safeEdit(formatFailedLaunchMessage({ commandName, sessionName: iso ? session : null, isolationBackend: iso?.backend || null, infoBlock, error: result.error || result.output, locale }));
|
|
176
|
+
}
|
|
158
177
|
};
|
|
159
178
|
}
|
|
160
179
|
|
|
@@ -15,7 +15,7 @@ import { collectExecutingItems, formatDuration, formatQueueToolSection, formatWa
|
|
|
15
15
|
export { QUEUE_CONFIG, THRESHOLD_STRATEGIES } from './queue-config.lib.mjs';
|
|
16
16
|
import { QUEUE_CONFIG } from './queue-config.lib.mjs';
|
|
17
17
|
import { reserveStartSlotForQueue } from './queue-start-reservation.lib.mjs';
|
|
18
|
-
import { formatExecutingWorkSessionMessage, formatStartingWorkSessionMessage } from './work-session-formatting.lib.mjs';
|
|
18
|
+
import { formatExecutingWorkSessionMessage, formatFailedLaunchMessage, formatStartingWorkSessionMessage } from './work-session-formatting.lib.mjs';
|
|
19
19
|
import { t } from './i18n.lib.mjs';
|
|
20
20
|
import { lt } from './limits-i18n.lib.mjs';
|
|
21
21
|
export const QueueItemStatus = {
|
|
@@ -1070,9 +1070,24 @@ export class SolveQueue {
|
|
|
1070
1070
|
// This was a bug where the final message update never happened because messageInfo was null
|
|
1071
1071
|
// See: https://github.com/link-assistant/hive-mind/issues/1062
|
|
1072
1072
|
const savedMessageInfo = item.messageInfo;
|
|
1073
|
-
//
|
|
1074
|
-
|
|
1075
|
-
|
|
1073
|
+
// Issue #2154: a launch that never produced a container was still marked
|
|
1074
|
+
// STARTED, counted in `totalCompleted`, pushed onto `completed` and
|
|
1075
|
+
// logged as `Finished: […] (started)`. Three Formal AI tasks that never
|
|
1076
|
+
// ran therefore appear in the bot log as successful starts — a false
|
|
1077
|
+
// positive that hid the incident and contradicted `$ --list`, which had
|
|
1078
|
+
// no such sessions. A refused launch is a failure of the queue item.
|
|
1079
|
+
const launchFailed = Boolean(result) && result.success === false;
|
|
1080
|
+
if (launchFailed) {
|
|
1081
|
+
item.setFailed(result.error || result.output || 'the launch was refused before the container started');
|
|
1082
|
+
item.sessionName = sessionName;
|
|
1083
|
+
item.messageInfo = null; // terminal status — stop tracking the message
|
|
1084
|
+
this.stats.totalFailed++;
|
|
1085
|
+
console.error(`[solve_queue] Item ${item.id} was not launched (session ${sessionName}): ${item.error}`);
|
|
1086
|
+
} else {
|
|
1087
|
+
// Update to Started status (terminal - forgets message tracking)
|
|
1088
|
+
item.setStarted(sessionName);
|
|
1089
|
+
this.stats.totalCompleted++;
|
|
1090
|
+
}
|
|
1076
1091
|
// Final message update using saved messageInfo
|
|
1077
1092
|
if (item.ctx && result && savedMessageInfo) {
|
|
1078
1093
|
const { chatId, messageId } = savedMessageInfo;
|
|
@@ -1089,7 +1104,17 @@ export class SolveQueue {
|
|
|
1089
1104
|
});
|
|
1090
1105
|
await item.ctx.telegram.editMessageText(chatId, messageId, undefined, response, { parse_mode: 'Markdown' });
|
|
1091
1106
|
} else {
|
|
1092
|
-
|
|
1107
|
+
// Issue #2154: a queued /solve that fails to launch reports the
|
|
1108
|
+
// same way as a direct one — with its session UUID and a note
|
|
1109
|
+
// that nothing was started, instead of a bare error dump.
|
|
1110
|
+
const response = formatFailedLaunchMessage({
|
|
1111
|
+
commandName: 'solve',
|
|
1112
|
+
sessionName: sessionName === 'unknown' ? null : sessionName,
|
|
1113
|
+
isolationBackend: result.isolationBackend || null,
|
|
1114
|
+
infoBlock: item.infoBlock,
|
|
1115
|
+
error: result.error || result.output,
|
|
1116
|
+
locale: item.locale,
|
|
1117
|
+
});
|
|
1093
1118
|
await item.ctx.telegram.editMessageText(chatId, messageId, undefined, response, { parse_mode: 'Markdown' });
|
|
1094
1119
|
}
|
|
1095
1120
|
} catch (error) {
|
|
@@ -80,12 +80,58 @@ export function formatStartingWorkSessionMessage({ sessionName = null, isolation
|
|
|
80
80
|
return `${header}\n\n📊 ${sessionLabel}: \`${sessionName}\`${isolationInfo}${details}`;
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
export function formatExecutingWorkSessionMessage({ sessionName = 'unknown', isolationBackend = null, infoBlock = '', locale = null } = {}) {
|
|
83
|
+
export function formatExecutingWorkSessionMessage({ sessionName = 'unknown', executionUuid = null, isolationBackend = null, infoBlock = '', locale = null } = {}) {
|
|
84
84
|
const sessionLabel = text(locale, 'telegram.session_label', 'Session');
|
|
85
85
|
const isolationLabel = text(locale, 'telegram.isolation_label', 'Isolation');
|
|
86
86
|
const isolationInfo = isolationBackend ? `\n🔒 ${isolationLabel}: \`${isolationBackend}\`` : '';
|
|
87
87
|
const details = infoBlock ? `\n\n${infoBlock}` : '';
|
|
88
|
-
|
|
88
|
+
// Issue #2154: `$ --list` identifies an execution by start-command's own UUID,
|
|
89
|
+
// which is *not* the session name shown above it. Printing only one of the two
|
|
90
|
+
// left the operator unable to match a running task to the session list. The
|
|
91
|
+
// line is omitted entirely when start-command reported no UUID, so a session
|
|
92
|
+
// never carries an empty label.
|
|
93
|
+
const executionLabel = text(locale, 'telegram.execution_label', 'Execution');
|
|
94
|
+
const executionInfo = executionUuid ? `\n🆔 ${executionLabel}: \`${executionUuid}\`` : '';
|
|
95
|
+
return `${text(locale, 'telegram.work_session_executing', '⏳ Executing...')}\n\n📊 ${sessionLabel}: \`${sessionName}\`${executionInfo}${isolationInfo}${details}`;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Render the reply for a work session that never started (issue #2154).
|
|
100
|
+
*
|
|
101
|
+
* The previous reply was the raw runner error in a code fence and nothing else.
|
|
102
|
+
* Two things were missing, and both were reported as separate symptoms of the
|
|
103
|
+
* same incident:
|
|
104
|
+
*
|
|
105
|
+
* - **The session UUID.** It was generated before the launch and shown in the
|
|
106
|
+
* "🔄 Starting..." message, but the failure reply *overwrote* that message,
|
|
107
|
+
* so the only identifier the task ever had was destroyed by its own error
|
|
108
|
+
* report. Nothing then connected the Telegram thread to the bot log lines,
|
|
109
|
+
* to the session store, or to a `--log <uuid>` lookup.
|
|
110
|
+
* - **Why the task is missing from `--list`.** A failed launch produces no
|
|
111
|
+
* container, so the session is untracked and never appears in the listing.
|
|
112
|
+
* Without saying so, the reply reads as if the task were running somewhere.
|
|
113
|
+
*
|
|
114
|
+
* @param {Object} params
|
|
115
|
+
* @param {string} [params.commandName] - Command the user invoked (`solve`, `hive`, …)
|
|
116
|
+
* @param {string|null} [params.sessionName] - Session UUID, when one was generated
|
|
117
|
+
* @param {string|null} [params.isolationBackend]
|
|
118
|
+
* @param {string} [params.infoBlock]
|
|
119
|
+
* @param {string} [params.error] - Runner error text
|
|
120
|
+
* @param {string|null} [params.locale]
|
|
121
|
+
* @returns {string} Markdown reply
|
|
122
|
+
*
|
|
123
|
+
* @see https://github.com/link-assistant/hive-mind/issues/2154
|
|
124
|
+
*/
|
|
125
|
+
export function formatFailedLaunchMessage({ commandName = 'command', sessionName = null, isolationBackend = null, infoBlock = '', error = '', locale = null } = {}) {
|
|
126
|
+
const header = text(locale, 'telegram.error_executing_command', `❌ Error executing ${commandName} command`, { commandName });
|
|
127
|
+
const sessionLabel = text(locale, 'telegram.session_label', 'Session');
|
|
128
|
+
const isolationLabel = text(locale, 'telegram.isolation_label', 'Isolation');
|
|
129
|
+
const sessionLine = sessionName ? `\n📊 ${sessionLabel}: \`${sessionName}\`` : '';
|
|
130
|
+
const isolationLine = isolationBackend ? `\n🔒 ${isolationLabel}: \`${isolationBackend}\`` : '';
|
|
131
|
+
const body = String(error ?? '').trim() || 'unknown error';
|
|
132
|
+
const notLaunched = sessionName ? `\n\n${text(locale, 'telegram.work_session_not_launched', 'The work session was not launched, so it has no log and is not listed by `--list`.')}` : '';
|
|
133
|
+
const details = infoBlock ? `\n\n${infoBlock}` : '';
|
|
134
|
+
return `${header}:${sessionLine}${isolationLine}\n\n\`\`\`\n${body}\n\`\`\`${notLaunched}${details}`;
|
|
89
135
|
}
|
|
90
136
|
|
|
91
137
|
/**
|
|
@@ -176,6 +222,13 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
176
222
|
const sessionLabel = text(messageLocale, 'telegram.session_label', 'Session');
|
|
177
223
|
const isolationLabel = text(messageLocale, 'telegram.isolation_label', 'Isolation');
|
|
178
224
|
const isolationInfo = sessionInfo?.isolationBackend ? `\n🔒 ${isolationLabel}: \`${sessionInfo.isolationBackend}\`` : '';
|
|
225
|
+
// Issue #2154: the completion reply is the last word the Telegram thread has
|
|
226
|
+
// on a task, and the handle an operator uses afterwards to fetch its log. Keep
|
|
227
|
+
// start-command's execution UUID (the one `$ --list` prints) next to the
|
|
228
|
+
// session UUID, so the finished task can still be found in the session list.
|
|
229
|
+
const executionLabel = text(messageLocale, 'telegram.execution_label', 'Execution');
|
|
230
|
+
const executionUuid = sessionInfo?.executionUuid || statusResult?.uuid || null;
|
|
231
|
+
const executionInfo = executionUuid ? `\n🆔 ${executionLabel}: \`${executionUuid}\`` : '';
|
|
179
232
|
const startTime = parseDateValue(statusResult?.startTime) || parseDateValue(sessionInfo?.startTime) || observedEndTime;
|
|
180
233
|
const endTime = parseDateValue(statusResult?.endTime) || observedEndTime;
|
|
181
234
|
const durationSeconds = Math.max(0, (endTime.getTime() - startTime.getTime()) / 1000);
|
|
@@ -188,7 +241,7 @@ export function formatSessionCompletionMessage({ sessionName, sessionInfo, statu
|
|
|
188
241
|
const statusEmoji = statusEmojiOverride || (failed ? '❌' : '✅');
|
|
189
242
|
let message = `${statusEmoji} *${statusText}*\n\n`;
|
|
190
243
|
message += `⏱️ ${durationLabel}: ${formatSessionDurationSeconds(durationSeconds)}\n`;
|
|
191
|
-
message += `📊 ${sessionLabel}: \`${sessionName || 'unknown'}\`${isolationInfo}${details}`;
|
|
244
|
+
message += `📊 ${sessionLabel}: \`${sessionName || 'unknown'}\`${executionInfo}${isolationInfo}${details}`;
|
|
192
245
|
|
|
193
246
|
// Issue #594: --show-limits virtual option appends snapshot/delta sections
|
|
194
247
|
// (Markdown code blocks) below the standard completion details.
|