@link-assistant/hive-mind 2.12.2 → 2.12.4

package/src/formal-ai-image.lib.mjs

@@ -0,0 +1,222 @@
+/**
+ * Resolve — and prove available — the image the Formal AI sidecar boots from
+ * (issue #2154).
+ *
+ * ## Why this module exists
+ *
+ * Before this module, `acquireFormalAiSidecar` handed
+ * `ghcr.io/link-assistant/formal-ai:<version>` straight to `docker run` and
+ * relied on Docker's implicit pull. The published package is private, so every
+ * Formal AI task died with the raw daemon dump:
+ *
+ * ```text
+ * Command failed: docker run --detach --name hive-mind-formal-ai …
+ * Unable to find image 'ghcr.io/link-assistant/formal-ai:0.339.1' locally
+ * docker: Error response from daemon: error from registry: unauthorized
+ * ```
+ *
+ * Three separate defects made that fatal:
+ *
+ *  1. **No preflight.** The registry problem surfaced as a failed *container
+ *     launch* rather than as a failed *image resolution*, so nothing could tell
+ *     the operator what to do about it.
+ *  2. **No diagnosis.** `unauthorized` from a registry has exactly three causes
+ *     (private package, missing/insufficient credentials, wrong reference) and
+ *     each has a different fix. The raw dump named none of them.
+ *  3. **No alternative.** The Hive Mind images themselves bake
+ *     `/usr/local/bin/formal-ai` at the same pinned version (`Dockerfile`,
+ *     `Dockerfile.dind`: `cargo install formal-ai --version ${FORMAL_AI_VERSION}
+ *     --locked`), and that image is already present on the host because every
+ *     isolated task runs from it. A registry outage therefore never had to stop
+ *     Formal AI work at all.
+ *
+ * ## Contract
+ *
+ * - `HIVE_MIND_FORMAL_AI_IMAGE` is an **exact** operator pin: it is the only
+ *   candidate, and if it cannot be resolved the acquire fails. An operator who
+ *   names an image means that image.
+ * - Otherwise the published image is preferred, and the local Hive Mind image is
+ *   the fallback. The fallback is only *used*; it is never pulled, because its
+ *   whole point is that it is already on the host.
+ * - Nothing here downgrades a Formal AI task to another model. Issue #2146's
+ *   fail-closed rule still holds: when no candidate resolves, the task is
+ *   refused with an actionable message.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2154
+ * @see https://github.com/link-assistant/hive-mind/issues/2146
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+import { FORMAL_AI_BOOTSTRAP_VERSION } from './formal-ai-version.lib.mjs';
+import { getDockerIsolationImage } from './hive-mind-image.lib.mjs';
+
+const execFileAsync = promisify(execFile);
+
+/** Image published by every Formal AI release (`:latest` plus the bare version). */
+export const FORMAL_AI_IMAGE_REPOSITORY = 'ghcr.io/link-assistant/formal-ai';
+
+const DEFAULT_DOCKER_TIMEOUT_MS = 600_000;
+
+/** Where each candidate came from, so logs and errors can explain themselves. */
+export const FORMAL_AI_IMAGE_SOURCES = Object.freeze({
+  PINNED: 'operator-pin',
+  PUBLISHED: 'published-image',
+  HIVE_MIND: 'hive-mind-image',
+});
+
+const errorText = error => error?.stderr?.toString?.().trim() || error?.stdout?.toString?.().trim() || error?.message || String(error);
+
+/**
+ * Classify a `docker pull` failure into the operator action that fixes it.
+ *
+ * Registry errors are famously interchangeable-looking; the distinction that
+ * matters in practice is that GHCR answers `unauthorized` for a package that
+ * *exists but is private* and `denied` for one that does not exist or that the
+ * presented token may not see.
+ *
+ * @param {string} message - Raw stderr from the failed pull.
+ * @returns {{kind: string, reason: string, remediation: string[]}}
+ */
+export const classifyDockerRegistryError = message => {
+  const text = String(message || '');
+  if (/unauthorized|authentication required|requires authentication/i.test(text)) {
+    return {
+      kind: 'unauthorized',
+      reason: 'the registry refused an anonymous or under-scoped pull',
+      remediation: ['make the package public (GHCR packages published by GITHUB_TOKEN are private by default), or', 'authenticate the daemon with a token carrying the `read:packages` scope: `echo $TOKEN | docker login ghcr.io -u <user> --password-stdin`, or', 'point HIVE_MIND_FORMAL_AI_IMAGE at an image this host can already pull'],
+    };
+  }
+  if (/denied|forbidden|insufficient_scope|permission_denied/i.test(text)) {
+    return {
+      kind: 'denied',
+      reason: 'the registry denied access to that reference',
+      remediation: ['check the repository name and tag exist', 'check the credentials in use carry the `read:packages` scope'],
+    };
+  }
+  if (/manifest unknown|not found|no such (?:image|manifest)/i.test(text)) {
+    return { kind: 'not-found', reason: 'the registry has no such tag', remediation: ['check the tag exists in the registry', 'pin a published tag with HIVE_MIND_FORMAL_AI_IMAGE'] };
+  }
+  if (/no space left on device/i.test(text)) {
+    return { kind: 'disk-full', reason: 'the daemon ran out of disk while unpacking the image', remediation: ['free disk space on the Docker data root, then retry'] };
+  }
+  if (/timeout|timed out|temporary failure|dial tcp|i\/o timeout|connection refused|network is unreachable|EOF/i.test(text)) {
+    return { kind: 'network', reason: 'the registry was unreachable', remediation: ['check network/proxy access to the registry, then retry'] };
+  }
+  return { kind: 'unknown', reason: 'the pull failed', remediation: ['inspect the daemon error above'] };
+};
+
+/**
+ * Resolve the local Hive Mind image, which bakes `formal-ai` at the same pinned
+ * version as the published sidecar image and is already present on any host
+ * that runs Docker-isolated tasks.
+ */
+export const resolveFormalAiFallbackImage = (env = process.env) => String(env.HIVE_MIND_FORMAL_AI_FALLBACK_IMAGE || '').trim() || getDockerIsolationImage({ env });
+
+/**
+ * Ordered list of images to try, most preferred first.
+ *
+ * @param {object} [env]
+ * @returns {Array<{image: string, source: string, pullable: boolean}>}
+ */
+export const resolveFormalAiSidecarImageCandidates = (env = process.env) => {
+  const pinned = String(env.HIVE_MIND_FORMAL_AI_IMAGE || '').trim();
+  if (pinned) return [{ image: pinned, source: FORMAL_AI_IMAGE_SOURCES.PINNED, pullable: true }];
+  const candidates = [{ image: `${FORMAL_AI_IMAGE_REPOSITORY}:${FORMAL_AI_BOOTSTRAP_VERSION}`, source: FORMAL_AI_IMAGE_SOURCES.PUBLISHED, pullable: true }];
+  const fallback = resolveFormalAiFallbackImage(env);
+  // Never pulled: the fallback's value is that it is already on the host. A
+  // deployment that has to pull it would be pulling the *bigger* image.
+  if (fallback && fallback !== candidates[0].image) candidates.push({ image: fallback, source: FORMAL_AI_IMAGE_SOURCES.HIVE_MIND, pullable: false });
+  return candidates;
+};
+
+/** The preferred image, kept for callers and logs that only need the headline reference. */
+export const resolveFormalAiSidecarImage = (env = process.env) => resolveFormalAiSidecarImageCandidates(env)[0].image;
+
+const inspectLocalImage = async (image, { run, timeoutMs }) => {
+  try {
+    const result = await run('docker', ['image', 'inspect', image, '--format', '{{.Id}}'], { encoding: 'utf8', timeout: timeoutMs });
+    return String(result?.stdout ?? '').trim() || null;
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * Render the aggregated failure so an operator reading the Telegram reply or the
+ * bot log knows the cause and the fix without opening a shell.
+ */
+const describeFailure = attempts => {
+  const lines = ['No Formal AI image could be resolved, so the sidecar was not started.'];
+  for (const attempt of attempts) {
+    if (attempt.source === FORMAL_AI_IMAGE_SOURCES.HIVE_MIND && attempt.kind === 'absent') {
+      lines.push(`• ${attempt.image} (local Hive Mind image, fallback): not present on this host — Docker-isolated tasks would have to pull it too.`);
+      continue;
+    }
+    lines.push(`• ${attempt.image} (${attempt.source}): ${attempt.reason}${attempt.error ? ` — ${attempt.error}` : ''}`);
+    for (const step of attempt.remediation ?? []) lines.push(`    → ${step}`);
+  }
+  lines.push('Formal AI tasks fail closed by design (issue #2146): Hive Mind will not silently run them on another model.');
+  return lines.join('\n');
+};
+
+/**
+ * Return the first candidate image that is usable on this host, pulling only the
+ * pullable ones and never throwing for a candidate that simply is not there.
+ *
+ * @param {object} params
+ * @param {object} [params.env]
+ * @param {Function} [params.run] - `execFile`-shaped seam, so tests can drive a fake daemon.
+ * @param {number} [params.timeoutMs]
+ * @param {Function|null} [params.log]
+ * @param {boolean} [params.verbose]
+ * @param {boolean} [params.pull] - Set false to accept only images already on the host.
+ * @returns {Promise<{image: string, source: string, digest: string|null, pulled: boolean, attempts: object[]}>}
+ * @throws {Error} When no candidate resolves; the message names every attempt and its fix.
+ */
+export const ensureFormalAiSidecarImage = async ({ env = process.env, run = execFileAsync, timeoutMs = DEFAULT_DOCKER_TIMEOUT_MS, log = null, verbose = false, pull = true, candidates = resolveFormalAiSidecarImageCandidates(env) } = {}) => {
+  const attempts = [];
+
+  for (const candidate of candidates) {
+    const localDigest = await inspectLocalImage(candidate.image, { run, timeoutMs });
+    if (localDigest) {
+      if (verbose && log) await log(`[VERBOSE] formal-ai-image: using '${candidate.image}' (${candidate.source}) already present locally, digest=${localDigest}`);
+      return { image: candidate.image, source: candidate.source, digest: localDigest, pulled: false, attempts };
+    }
+
+    if (!candidate.pullable || !pull) {
+      attempts.push({ ...candidate, kind: 'absent', reason: 'not present locally and not pulled', error: null, remediation: [] });
+      continue;
+    }
+
+    if (log) await log(`⬇️ Pulling the Formal AI sidecar image ${candidate.image}`);
+    try {
+      await run('docker', ['pull', candidate.image], { encoding: 'utf8', timeout: timeoutMs });
+    } catch (error) {
+      const message = errorText(error);
+      const diagnosis = classifyDockerRegistryError(message);
+      attempts.push({ ...candidate, kind: diagnosis.kind, reason: diagnosis.reason, error: message, remediation: diagnosis.remediation });
+      if (log) await log(`⚠️ Could not pull ${candidate.image}: ${diagnosis.reason} (${diagnosis.kind}). ${message}`);
+      continue;
+    }
+
+    const digest = await inspectLocalImage(candidate.image, { run, timeoutMs });
+    if (verbose && log) await log(`[VERBOSE] formal-ai-image: pulled '${candidate.image}' (${candidate.source}), digest=${digest ?? 'unknown'}`);
+    return { image: candidate.image, source: candidate.source, digest, pulled: true, attempts };
+  }
+
+  const error = new Error(describeFailure(attempts));
+  error.formalAiImageAttempts = attempts;
+  throw error;
+};
+
+export default {
+  FORMAL_AI_IMAGE_REPOSITORY,
+  FORMAL_AI_IMAGE_SOURCES,
+  classifyDockerRegistryError,
+  ensureFormalAiSidecarImage,
+  resolveFormalAiFallbackImage,
+  resolveFormalAiSidecarImage,
+  resolveFormalAiSidecarImageCandidates,
+};

package/src/formal-ai-isolation.lib.mjs

@@ -32,7 +32,14 @@ export const acquireFormalAiSidecarForTask = async ({ backend, args = [], model
   try {
     return { sidecar: await acquire({ sessionId, tool, model, env, verbose, log }), error: null };
   } catch (error) {
-    return { sidecar: null, error: `Formal AI sidecar could not be started, so the task was not launched (issue #2146): ${error?.message || error}` };
+    const message = `Formal AI sidecar could not be started, so the task was not launched (issue #2146): ${error?.message || error}`;
+    // Issue #2154: this used to be returned and nothing else. The reply went to
+    // Telegram, the session was untracked, and the bot log showed only the
+    // untracking — so the operator could see that a task had vanished but never
+    // why. Every refusal is now on the record, with the session UUID that names
+    // the task in `$ --list` and in the session store.
+    console.error(`[formal-ai-isolation] Session ${sessionId}: ${message}`);
+    return { sidecar: null, error: message };
   }
 };
 

package/src/formal-ai-sidecar.lib.mjs

@@ -46,7 +46,8 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { promisify } from 'node:util';
 
-import { FORMAL_AI_BOOTSTRAP_VERSION } from './formal-ai-version.lib.mjs';
+import { FORMAL_AI_MINIMUM_VERSION, isFormalAiVersionAtLeast } from './formal-ai-version.lib.mjs';
+import { ensureFormalAiSidecarImage, resolveFormalAiSidecarImage } from './formal-ai-image.lib.mjs';
 import { isFormalAiModel } from './formal-ai-model.lib.mjs';
 import { getModelFromArgs } from './model-args.lib.mjs';
 import { resolveBotStateDir } from './session-store.lib.mjs';
@@ -77,8 +78,14 @@ export const FORMAL_AI_SIDECAR_PORT = 8080;
 export const FORMAL_AI_MEMORY_MOUNT = '/home/box/.formal-ai';
 export const FORMAL_AI_MEMORY_PATH = `${FORMAL_AI_MEMORY_MOUNT}/memory.lino`;
 
-/** Image published by every Formal AI release (`:latest` plus the bare version). */
-export const FORMAL_AI_IMAGE_REPOSITORY = 'ghcr.io/link-assistant/formal-ai';
+/**
+ * Image published by every Formal AI release (`:latest` plus the bare version).
+ *
+ * Re-exported from `formal-ai-image.lib.mjs`, which owns image resolution since
+ * issue #2154 taught us that "which image" and "is it actually pullable" are the
+ * same question.
+ */
+export { FORMAL_AI_IMAGE_REPOSITORY, resolveFormalAiSidecarImage, resolveFormalAiSidecarImageCandidates } from './formal-ai-image.lib.mjs';
 
 /** Applied to the sidecar, its network and its volume so reconciliation can find them. */
 export const FORMAL_AI_SIDECAR_LABEL = 'com.link-assistant.hive-mind.formal-ai';
@@ -86,6 +93,9 @@ export const FORMAL_AI_SIDECAR_LABEL = 'com.link-assistant.hive-mind.formal-ai';
 const STATE_FILE_NAME = 'formal-ai-sidecar.json';
 const SIDECAR_LOCK_NAME = 'formal-ai-sidecar';
 const DEFAULT_DOCKER_TIMEOUT_MS = 120_000;
+// Pulling a sidecar image is the one Docker call that legitimately takes many
+// minutes, so it gets its own budget instead of the general command timeout.
+const DEFAULT_IMAGE_TIMEOUT_MS = 600_000;
 const DEFAULT_HEALTH_ATTEMPTS = 60;
 const DEFAULT_HEALTH_DELAY_MS = 1000;
 
@@ -107,9 +117,6 @@ const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
  */
 export const isFormalAiTask = ({ args = [], model = null } = {}) => isFormalAiModel(model || getModelFromArgs(args));
 
-/** Resolve the image the sidecar boots with. Operators may pin their own build. */
-export const resolveFormalAiSidecarImage = (env = process.env) => String(env.HIVE_MIND_FORMAL_AI_IMAGE || '').trim() || `${FORMAL_AI_IMAGE_REPOSITORY}:${FORMAL_AI_BOOTSTRAP_VERSION}`;
-
 /** Build the endpoint origin for a host name or address. */
 export const buildFormalAiSidecarBaseUrl = (host = FORMAL_AI_SIDECAR_NETWORK_ALIAS) => `http://${host}:${FORMAL_AI_SIDECAR_PORT}`;
 
@@ -433,18 +440,14 @@ export const stopFormalAiSidecar = async ({ env = process.env, fsImpl = fs, run
  * Must be called *before* the task container's command is allowed to run.
  * Returns the endpoint the task should be pointed at.
  */
-export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = null, env = process.env, fsImpl = fs, run = execFileAsync, timeoutMs, log = null, verbose = false, now = () => new Date(), healthAttempts, healthDelayMs, sleepImpl = sleep, lockOptions = {} } = {}) => {
+export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = null, env = process.env, fsImpl = fs, run = execFileAsync, timeoutMs, imageTimeoutMs = DEFAULT_IMAGE_TIMEOUT_MS, log = null, verbose = false, now = () => new Date(), healthAttempts, healthDelayMs, sleepImpl = sleep, lockOptions = {} } = {}) => {
   if (!sessionId) throw new Error('acquireFormalAiSidecar requires a sessionId');
 
   return withFormalAiSidecarLock(
     async () => {
-      const image = resolveFormalAiSidecarImage(env);
       const state = readFormalAiSidecarState({ env, fsImpl });
       const leases = await reconcileLeases(state.leases, { run, timeoutMs, log, verbose });
 
-      await ensureFormalAiNetwork({ run, timeoutMs, log, verbose });
-      await ensureFormalAiMemoryVolume({ image, run, timeoutMs, log, verbose });
-
       let container = await inspectDockerContainer(FORMAL_AI_SIDECAR_CONTAINER_NAME, { run, timeoutMs });
       if (container.exists && !container.running) {
         // A stopped container may predate an image change; recreate instead of
@@ -453,8 +456,18 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
         container = { exists: false, running: false, image: null, imageDigest: null };
       }
 
+      // Resolve the image *before* anything shells out with it. Until issue
+      // #2154 the reference went straight into `docker run`, so a registry that
+      // refused the pull surfaced as an unreadable `Command failed: docker run …`
+      // dump and the task died even though a usable image sat on the host.
+      const resolved = container.exists && container.image ? { image: container.image, source: 'running-sidecar', pulled: false } : await ensureFormalAiSidecarImage({ env, run, timeoutMs: imageTimeoutMs, log, verbose });
+      const image = resolved.image;
+
+      await ensureFormalAiNetwork({ run, timeoutMs, log, verbose });
+      await ensureFormalAiMemoryVolume({ image, run, timeoutMs, log, verbose });
+
       if (!container.exists) {
-        if (log) await log(`🧠 Starting the Formal AI sidecar (${image}) on the internal network '${FORMAL_AI_SIDECAR_NETWORK_NAME}'`);
+        if (log) await log(`🧠 Starting the Formal AI sidecar (${image}, ${resolved.source}) on the internal network '${FORMAL_AI_SIDECAR_NETWORK_NAME}'`);
         await dockerText(run, buildFormalAiSidecarRunArgs({ image, env }), { timeoutMs });
         container = await inspectDockerContainer(FORMAL_AI_SIDECAR_CONTAINER_NAME, { run, timeoutMs });
       }
@@ -466,12 +479,21 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
         throw new Error(`Formal AI sidecar '${FORMAL_AI_SIDECAR_CONTAINER_NAME}' did not become healthy: ${health.error}`);
       }
 
+      // The sidecar may now boot from any of several images (issue #2154), so
+      // the version floor is enforced against what the process actually reports
+      // rather than assumed from the tag. A too-old binary would answer /health
+      // and then fail on the agent-mode API the tasks depend on.
+      const reportedVersion = health.health?.version ?? null;
+      if (reportedVersion && !isFormalAiVersionAtLeast(reportedVersion, FORMAL_AI_MINIMUM_VERSION)) {
+        throw new Error(`Formal AI sidecar image ${image} (${resolved.source}) runs formal-ai ${reportedVersion}, but Hive Mind requires >= ${FORMAL_AI_MINIMUM_VERSION}. Rebuild or repin the image (HIVE_MIND_FORMAL_AI_IMAGE) before running Formal AI tasks.`);
+      }
+
       const address = await readFormalAiSidecarAddress({ run, timeoutMs });
       const acquiredAt = now().toISOString();
       const nextLeases = [...leases.filter(lease => lease.sessionId !== sessionId), { sessionId, tool, model, acquiredAt }];
       writeFormalAiSidecarState({ ...state, image: container.image || image, imageDigest: container.imageDigest, startedAt: state.startedAt || acquiredAt, leases: nextLeases }, { env, fsImpl });
 
-      if (verbose && log) await log(`[VERBOSE] formal-ai-sidecar: lease '${sessionId}' acquired (${nextLeases.length} active), image=${container.image || image}, digest=${container.imageDigest ?? 'unknown'}, address=${address ?? 'unknown'}, formal-ai=${health.health?.version ?? 'unknown'}, memory schema=${health.health?.memory?.schema_version ?? 'unknown'}`);
+      if (verbose && log) await log(`[VERBOSE] formal-ai-sidecar: lease '${sessionId}' acquired (${nextLeases.length} active), image=${container.image || image} (${resolved.source}), digest=${container.imageDigest ?? 'unknown'}, address=${address ?? 'unknown'}, formal-ai=${health.health?.version ?? 'unknown'}, memory schema=${health.health?.memory?.schema_version ?? 'unknown'}`);
 
       return {
         address,
@@ -482,6 +504,7 @@ export const acquireFormalAiSidecar = async ({ sessionId, tool = null, model = n
         containerName: FORMAL_AI_SIDECAR_CONTAINER_NAME,
         memoryVolume: FORMAL_AI_MEMORY_VOLUME_NAME,
         image: container.image || image,
+        imageSource: resolved.source,
         imageDigest: container.imageDigest,
         health: health.health,
         leaseCount: nextLeases.length,

package/src/formal-ai-updater.lib.mjs

@@ -33,6 +33,7 @@ import { execFile } from 'node:child_process';
 import fs from 'node:fs';
 import { promisify } from 'node:util';
 
+import { classifyDockerRegistryError } from './formal-ai-image.lib.mjs';
 import { FORMAL_AI_IMAGE_REPOSITORY, FORMAL_AI_MEMORY_MOUNT, FORMAL_AI_MEMORY_PATH, FORMAL_AI_MEMORY_VOLUME_NAME, FORMAL_AI_SIDECAR_CONTAINER_NAME, buildFormalAiSidecarRunArgs, ensureFormalAiMemoryVolume, ensureFormalAiNetwork, inspectDockerContainer, readDockerImageDigest, readFormalAiSidecarState, reconcileFormalAiSidecar, stopFormalAiSidecar, waitForFormalAiSidecarHealth, withFormalAiSidecarLock, writeFormalAiSidecarState } from './formal-ai-sidecar.lib.mjs';
 
 const execFileAsync = promisify(execFile);
@@ -207,8 +208,19 @@ export const updateFormalAiSidecarWhenIdle = async ({ env = process.env, fsImpl
         await dockerText(run, ['pull', '--quiet', image], { timeoutMs: pullTimeoutMs });
       } catch (error) {
         const message = error?.stderr?.toString?.().trim() || error?.message || String(error);
-        if (log) await log(`⚠️ Could not pull ${image}; keeping the current Formal AI image: ${message}`);
-        return { status: 'failed', stage: 'pull', image, error: message };
+        // Issue #2154: this warning fired every ~5 minutes for hours ("Could not
+        // pull … unauthorized") without ever saying that the registry was
+        // refusing us, and the operator only learned of it when a Formal AI task
+        // failed to start. A permanent refusal (unauthorized/denied/not-found)
+        // is a configuration fault, not a transient blip: name it, say how to
+        // fix it, and report the classification to the caller.
+        const classification = classifyDockerRegistryError(message);
+        const permanent = ['unauthorized', 'denied', 'not-found'].includes(classification.kind);
+        if (log) {
+          const remediation = classification.remediation.length ? ` Fix it by one of: ${classification.remediation.join('; ')}.` : '';
+          await log(`${permanent ? '🚨' : '⚠️'} Could not pull ${image} — ${classification.reason} (${classification.kind}); keeping the current Formal AI image: ${message}${permanent ? remediation : ''}`);
+        }
+        return { status: 'failed', stage: 'pull', image, error: message, classification: classification.kind, permanent, remediation: classification.remediation };
       }
 
       const pulledDigest = await readDockerImageDigest(image, { run, timeoutMs });

package/src/hive-mind-image.lib.mjs

@@ -0,0 +1,56 @@
+/**
+ * Resolve the Hive Mind container image references shared by the Docker
+ * isolation runner and the Formal AI sidecar.
+ *
+ * Extracted from `isolation-runner.lib.mjs` for issue #2154: the Formal AI
+ * sidecar needs the very same reference to fall back to the locally present
+ * Hive Mind image (which bakes `/usr/local/bin/formal-ai`) when the published
+ * Formal AI image cannot be pulled, and importing the isolation runner from the
+ * sidecar would create an import cycle.
+ *
+ * @see https://github.com/link-assistant/hive-mind/issues/2154
+ */
+
+export const HIVE_MIND_IMAGE_REPO = 'konard/hive-mind';
+export const HIVE_MIND_DIND_IMAGE_REPO = 'konard/hive-mind-dind';
+export const DEFAULT_HIVE_MIND_IMAGE_TAG = 'latest';
+
+/**
+ * Resolve the tag used for the Docker isolation image.
+ *
+ * Release Docker images bake this env var from `HIVE_MIND_VERSION`, so a parent
+ * container started via `:latest` still launches child isolation containers from
+ * the same immutable release tag. Local/PR builds fall back to `latest`, and
+ * operators can override the tag explicitly when using custom images. Pinning
+ * matters for Docker-in-Docker deployments: the nested daemon starts with an
+ * empty image store, so a `:latest` digest drift from the host copy forces a
+ * fresh multi-gigabyte pull. See issue #1879.
+ */
+export function resolveDockerIsolationImageTag({ env = process.env } = {}) {
+  const explicit = String(env.HIVE_MIND_DOCKER_ISOLATION_IMAGE_TAG || '').trim();
+  return explicit || DEFAULT_HIVE_MIND_IMAGE_TAG;
+}
+
+/**
+ * Pick the Docker image used for `--isolation docker`.
+ *
+ * start-command defaults its Docker backend to a base OS image. Hive Mind needs
+ * an image with the same CLI/tooling baseline as the parent process instead.
+ *
+ * `HIVE_MIND_DOCKER_ISOLATION_IMAGE` is a full override (repo:tag). Otherwise
+ * the repo is chosen by image variant and the tag by
+ * `resolveDockerIsolationImageTag()`.
+ */
+export function getDockerIsolationImage({ env = process.env } = {}) {
+  if (env.HIVE_MIND_DOCKER_ISOLATION_IMAGE) return env.HIVE_MIND_DOCKER_ISOLATION_IMAGE;
+  const repo = String(env.HIVE_MIND_IMAGE_VARIANT || '').toLowerCase() === 'dind' ? HIVE_MIND_DIND_IMAGE_REPO : HIVE_MIND_IMAGE_REPO;
+  return `${repo}:${resolveDockerIsolationImageTag({ env })}`;
+}
+
+export default {
+  DEFAULT_HIVE_MIND_IMAGE_TAG,
+  HIVE_MIND_DIND_IMAGE_REPO,
+  HIVE_MIND_IMAGE_REPO,
+  getDockerIsolationImage,
+  resolveDockerIsolationImageTag,
+};

package/src/isolation-runner.lib.mjs

@@ -21,6 +21,12 @@ import os from 'node:os';
 import path from 'node:path';
 import { isExecutingSessionStatus, isTerminalSessionStatus } from './session-status.lib.mjs';
 import { acquireFormalAiSidecarForTask, attachFormalAiTaskContainer, releaseFormalAiSidecarForTask } from './formal-ai-isolation.lib.mjs';
+// The image references live in their own module so the Formal AI sidecar can
+// resolve the locally present Hive Mind image (which bakes `formal-ai`) without
+// importing this runner and creating a cycle. Re-exported here because callers
+// and tests have always reached them through the isolation runner. See #2154.
+import { getDockerIsolationImage } from './hive-mind-image.lib.mjs';
+export { getDockerIsolationImage, resolveDockerIsolationImageTag } from './hive-mind-image.lib.mjs';
 let commandStreamDollarPromise = null;
 async function getCommandStreamDollar() {
   if (!commandStreamDollarPromise) {
@@ -43,9 +49,6 @@ async function getCommandStreamDollar() {
 export { isExecutingSessionStatus, isTerminalSessionStatus, isKilledSessionStatus } from './session-status.lib.mjs';
 // Valid isolation backends
 const VALID_ISOLATION_BACKENDS = ['screen', 'tmux', 'docker'];
-const HIVE_MIND_IMAGE_REPO = 'konard/hive-mind';
-const HIVE_MIND_DIND_IMAGE_REPO = 'konard/hive-mind-dind';
-const DEFAULT_HIVE_MIND_IMAGE_TAG = 'latest';
 const DOCKER_CONTAINER_HOME = '/home/box';
 const FORMAL_AI_COMPOSE_HOSTNAME = 'link-assistant-formal-ai';
 // Default path where the host Docker socket is bind-mounted inside a DinD container so box's host-image passthrough can copy host images into the nested daemon. Matches box's own DIND_HOST_DOCKER_SOCK default. The deploy must mount it (`-v /var/run/docker.sock:/var/run/host-docker.sock:ro`) or the nested daemon starts empty and the first isolated task pulls the full, multi-gigabyte image. See issue #1914.
@@ -96,36 +99,6 @@ function maybeAddMount(mounts, source, target, existsSync) {
   if (!existsSync(source)) return;
   mounts.push({ source, target });
 }
-/**
- * Resolve the tag used for the Docker isolation image.
- *
- * Release Docker images bake this env var from `HIVE_MIND_VERSION`, so a parent
- * container started via `:latest` still launches child isolation containers from
- * the same immutable release tag. Local/PR builds fall back to `latest`, and
- * operators can override the tag explicitly when using custom images. Pinning
- * matters for Docker-in-Docker deployments: the nested daemon starts with an
- * empty image store, so a `:latest` digest drift from the host copy forces a
- * fresh multi-gigabyte pull. See issue #1879.
- */
-export function resolveDockerIsolationImageTag({ env = process.env } = {}) {
-  const explicit = String(env.HIVE_MIND_DOCKER_ISOLATION_IMAGE_TAG || '').trim();
-  return explicit || DEFAULT_HIVE_MIND_IMAGE_TAG;
-}
-/**
- * Pick the Docker image used for `--isolation docker`.
- *
- * start-command defaults its Docker backend to a base OS image. Hive Mind needs
- * an image with the same CLI/tooling baseline as the parent process instead.
- *
- * `HIVE_MIND_DOCKER_ISOLATION_IMAGE` is a full override (repo:tag). Otherwise
- * the repo is chosen by image variant and the tag by
- * `resolveDockerIsolationImageTag()`.
- */
-export function getDockerIsolationImage({ env = process.env } = {}) {
-  if (env.HIVE_MIND_DOCKER_ISOLATION_IMAGE) return env.HIVE_MIND_DOCKER_ISOLATION_IMAGE;
-  const repo = String(env.HIVE_MIND_IMAGE_VARIANT || '').toLowerCase() === 'dind' ? HIVE_MIND_DIND_IMAGE_REPO : HIVE_MIND_IMAGE_REPO;
-  return `${repo}:${resolveDockerIsolationImageTag({ env })}`;
-}
 /**
  * Resolve the path where the host Docker socket is expected to be mounted inside
  * a DinD container. box's entrypoint reads this socket to copy host images into
@@ -296,6 +269,48 @@ async function runStartCommand(binPath, startCommandArgs) {
 export function generateSessionId() {
   return crypto.randomUUID();
 }
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/**
+ * Extract start-command's own execution UUID from a launch banner.
+ *
+ * Issue #2154: an isolated task has two UUIDs. Hive Mind generates the session
+ * name and passes it as `--session` (it also becomes the container name);
+ * start-command mints a separate execution UUID and prints it as the `session`
+ * field of its launch banner:
+ *
+ * ```
+ * │ session   edc7b051-e12f-4f7b-b677-c885f3208407
+ * │ container 0a3627ef-f1f1-4801-a073-3678b9453db7
+ * ```
+ *
+ * `$ --list` shows the execution UUID, while Telegram and the logs showed the
+ * session UUID, so the two views could not be joined — which is why three
+ * refused tasks and two healthy ones looked equally unaccounted for. Returning
+ * it lets the caller record both.
+ *
+ * Only a well-formed UUID is returned; a banner we do not recognise yields
+ * null rather than a guess, because a wrong correlation is worse than none.
+ *
+ * @param {string} output - Raw stdout from the detached `$` launch
+ * @returns {string|null}
+ */
+export function parseStartCommandExecutionUuid(output) {
+  const raw = (output || '').trim();
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw);
+    const data = Array.isArray(parsed) ? parsed[0] : parsed;
+    const uuid = data?.uuid || data?.session || null;
+    if (typeof uuid === 'string' && UUID_PATTERN.test(uuid.trim())) return uuid.trim();
+  } catch {
+    // Human-readable banner — fall through.
+  }
+  // The banner is box-drawn (`│ session   <uuid>`); tolerate the prefix, an
+  // ASCII `|`, or no prefix at all.
+  const match = raw.match(/^[\s│|]*session\s+([^\s]+)\s*$/im);
+  const candidate = match?.[1]?.trim();
+  return candidate && UUID_PATTERN.test(candidate) ? candidate : null;
+}
 /**
  * Parse output from `$ --status <session>`.
  *
@@ -544,23 +559,22 @@ async function logDockerIsolationPostLaunchDiagnostics(sessionId, env = process.
 export async function executeWithIsolation(command, args, options = {}) {
   const { backend, verbose = false } = options;
   const sessionId = options.sessionId || generateSessionId();
+  // Issue #2154: a launch that never produced a container left no trace in the
+  // bot log — the reply went to Telegram and the log jumped straight to
+  // "session untracked", so an operator could see that tasks were disappearing
+  // but not why, and could not even name them. Every unsuccessful return from
+  // this function now records the session UUID (the same one `$ --list` and the
+  // session store use) together with the reason.
+  const failLaunch = (error, extra = {}) => {
+    console.error(`[isolation-runner] Session ${sessionId} was not launched (backend=${backend}, tool=${options.tool ?? 'claude'}, model=${options.model ?? 'default'}): ${error}`);
+    return { success: false, sessionId, output: '', error, ...extra };
+  };
   if (!VALID_ISOLATION_BACKENDS.includes(backend)) {
-    return {
-      success: false,
-      sessionId,
-      output: '',
-      error: `Invalid isolation backend: '${backend}'. Must be one of: ${VALID_ISOLATION_BACKENDS.join(', ')}`,
-    };
+    return failLaunch(`Invalid isolation backend: '${backend}'. Must be one of: ${VALID_ISOLATION_BACKENDS.join(', ')}`);
   }
   const binPath = await findStartCommandBinary();
   if (!binPath) {
-    return {
-      success: false,
-      sessionId,
-      output: '',
-      warning: '⚠️ WARNING: start-command ($) not found in PATH\nPlease install: npm install -g start-command',
-      error: 'start-command ($) not found',
-    };
+    return failLaunch('start-command ($) not found', { warning: '⚠️ WARNING: start-command ($) not found in PATH\nPlease install: npm install -g start-command' });
   }
   if (verbose) {
     console.log(`[VERBOSE] isolation-runner: Using $ binary at: ${binPath}`);
@@ -573,7 +587,7 @@ export async function executeWithIsolation(command, args, options = {}) {
   // fails. Fail closed — a Formal AI task must never start without Formal AI.
   const hostEnv = options.env || process.env;
   const { sidecar, error: sidecarError } = await acquireFormalAiSidecarForTask({ backend, args, model: options.model ?? null, tool: options.tool ?? null, sessionId, env: hostEnv, verbose });
-  if (sidecarError) return { success: false, sessionId, output: '', error: sidecarError };
+  if (sidecarError) return failLaunch(sidecarError);
   const taskEnv = sidecar ? { ...hostEnv, HIVE_MIND_FORMAL_AI_BASE_URL: sidecar.baseUrl } : hostEnv;
   const effectiveOptions =
     backend === 'docker'
@@ -628,7 +642,7 @@ export async function executeWithIsolation(command, args, options = {}) {
     if (formalAiAttachError) await removeDockerContainer(sessionId, verbose);
     await releaseFormalAiSidecarForTask({ sidecar, sessionId, env: hostEnv, verbose });
     if (formalAiAttachError) {
-      return { success: false, sessionId, output: result.output, error: `Formal AI task container could not be attached to the internal Formal AI network, so the task was stopped instead of falling back to another model (issue #2146): ${formalAiAttachError}` };
+      return failLaunch(`Formal AI task container could not be attached to the internal Formal AI network, so the task was stopped instead of falling back to another model (issue #2146): ${formalAiAttachError}`, { output: result.output });
     }
   }
   // Issue #1939: capture the freshly-launched docker session's reported status
@@ -639,19 +653,22 @@ export async function executeWithIsolation(command, args, options = {}) {
     await logDockerIsolationPostLaunchDiagnostics(sessionId, options.env || process.env);
   }
   if (result.success) {
+    // Issue #2154: hand the caller start-command's own execution UUID as well.
+    // It is the identifier `$ --list` prints, so without it the bot and the
+    // session list cannot be joined by an operator.
+    const executionUuid = parseStartCommandExecutionUuid(result.output);
+    if (verbose) {
+      console.log(executionUuid ? `[VERBOSE] isolation-runner: start-command execution UUID for session ${sessionId}: ${executionUuid} (this is what '$ --list' shows)` : `[VERBOSE] isolation-runner: start-command reported no execution UUID for session ${sessionId}; '$ --list' cannot be correlated for this session`);
+    }
     return {
       success: true,
       sessionId,
+      executionUuid,
       output: result.output,
       containerFilesystemStartBytes,
     };
   }
-  return {
-    success: false,
-    sessionId,
-    output: result.output,
-    error: result.error,
-  };
+  return failLaunch(result.error, { output: result.output });
 }
 /**
  * Query the status of an isolated session via `$ --status <uuid>`

package/src/locales/en.lino

@@ -654,6 +654,7 @@ en
         runner_also_failed "The runner also failed; its exit code is preserved for investigation."
         killed "Work session {{reason}}{{exitSuffix}}"
         stopped "Work session stopped by user{{requestedBy}}{{exitSuffix}}"
+        not_launched "The work session was not launched, so it has no log and is not listed by `--list`."
     duration
       label "Duration"
     session
@@ -670,6 +671,8 @@ en
         resumed_attempt "🔄 A new working session was started to recover from this kill (attempt {{attempt}}): {{sessionId}}"
     isolation
       label "Isolation"
+    execution
+      label "Execution"
     error
       executing
         command "❌ Error executing {{commandName}} command"