@vantran-se/openchamber-web 1.24.4 → 2.0.2

package/server/lib/spaces/places/docker-engine.js

@@ -0,0 +1,126 @@
+// How the Docker place talks to the docker CLI: run, inspect, find by label, remove.
+// Shared by the space operations in docker.js and the tools volume in docker-tools.js.
+
+import { SpaceError } from '../errors.js';
+
+const QUERY_TIMEOUT_MS = 30_000;
+export const CHANGE_TIMEOUT_MS = 120_000;
+// After a timed-out step the daemon may still finish it. A rollback sweeps again after this pause.
+export const ROLLBACK_SETTLE_MS = 2_000;
+
+// The CLI died in the middle of a step, so nobody knows whether the daemon finished it.
+const INTERRUPTED_CODES = ['command_timeout', 'command_killed', 'command_output_too_large'];
+export const isInterrupted = (error) => INTERRUPTED_CODES.includes(error?.code);
+
+const NOT_FOUND_PATTERNS = [
+  /\bNo such (?:object|container|volume|network|image)\b/i,
+  /\b(?:container|volume|network)\b.*\bnot found\b/i,
+];
+
+const isNotFound = (result) => result.code !== 0 && NOT_FOUND_PATTERNS.some((pattern) => pattern.test(result.stderr));
+
+// For a removal, "someone else is removing it right now" is as good as gone.
+const REMOVAL_IN_PROGRESS = /removal of container .* is already in progress/i;
+const isAlreadyGone = (result) => isNotFound(result) || (result.code !== 0 && REMOVAL_IN_PROGRESS.test(result.stderr));
+
+const parseJson = (text) => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new SpaceError('docker_output_unreadable', 'Docker printed something that is not JSON');
+  }
+};
+
+const inspectArgs = (kind, names) => (kind === 'container' ? ['inspect', '--type', 'container', ...names] : [kind, 'inspect', ...names]);
+const listArgs = (kind, filters) => (kind === 'container'
+  ? ['ps', '--all', ...filters, '--format', '{{.Names}}']
+  : [kind, 'ls', ...filters, '--format', '{{.Name}}']);
+const removeArgs = (kind, name) => (kind === 'container' ? ['rm', '--force', name] : [kind, 'rm', name]);
+
+export const entryLabels = (kind, entry) => (kind === 'container' ? entry.Config?.Labels : entry.Labels);
+export const entryName = (entry) => String(entry.Name ?? '').replace(/^\//, '');
+
+export const pause = (milliseconds) => new Promise((resolve) => { setTimeout(resolve, milliseconds); });
+
+export function createDockerEngine({ runCommand, dockerPath }) {
+  const run = (args, timeoutMs, options = {}) => runCommand(dockerPath, args, { ...options, timeoutMs });
+
+  const failure = (args, result) => new SpaceError(
+    'docker_command_failed',
+    `docker ${args.slice(0, 2).join(' ')} failed: ${result.stderr.trim() || `exit code ${result.code}`}`,
+  );
+
+  const docker = async (args, timeoutMs) => {
+    const result = await run(args, timeoutMs);
+    if (result.code !== 0) {
+      throw failure(args, result);
+    }
+    return result.stdout;
+  };
+
+  /** The parsed inspect entry, or null when Docker has no such resource. */
+  const inspect = async (kind, name) => {
+    const args = inspectArgs(kind, [name]);
+    const result = await run(args, QUERY_TIMEOUT_MS);
+    if (isNotFound(result)) {
+      return null;
+    }
+    if (result.code !== 0) {
+      throw failure(args, result);
+    }
+    return parseJson(result.stdout)[0];
+  };
+
+  /** The inspect entries of every resource of one kind that the label filters select. */
+  const findByLabel = async (kind, filters) => {
+    const names = (await docker(listArgs(kind, filters), QUERY_TIMEOUT_MS)).split('\n').filter(Boolean);
+    if (names.length === 0) {
+      return [];
+    }
+    // A name can vanish between the listing and the inspect. Docker then exits 1 but
+    // still prints the entries it found, and the rest must not suffer.
+    const args = inspectArgs(kind, names);
+    const result = await run(args, QUERY_TIMEOUT_MS);
+    if (result.code !== 0 && !isNotFound(result)) {
+      throw failure(args, result);
+    }
+    return parseJson(result.stdout);
+  };
+
+  const removeWith = async (args, kind, name) => {
+    try {
+      const result = await run(args, CHANGE_TIMEOUT_MS);
+      return result.code === 0 || isAlreadyGone(result) ? null : { kind, name, message: result.stderr.trim() };
+    } catch (error) {
+      return { kind, name, message: error.message };
+    }
+  };
+
+  /** Null when the resource is gone afterwards, otherwise what went wrong. A container goes even while it runs. */
+  const removeOne = (kind, name) => removeWith(removeArgs(kind, name), kind, name);
+
+  /** The same for a container that must be stopped. Docker refuses a running one, and that refusal is the point. */
+  const removeStoppedContainer = (nameOrId) => removeWith(['rm', nameOrId], 'container', nameOrId);
+
+  /**
+   * Removes the named resources that exist, labelled or not. Only for names that the
+   * same call found absent a moment ago: a `docker create` or `docker run` that
+   * the daemon finishes late makes a missing `src=` volume again, without labels.
+   */
+  const removeByName = async (names) => {
+    const failed = [];
+    for (const [kind, name] of names) {
+      try {
+        if (await inspect(kind, name)) {
+          const problem = await removeOne(kind, name);
+          if (problem) failed.push(problem);
+        }
+      } catch (error) {
+        failed.push({ kind, name, message: error.message });
+      }
+    }
+    return failed;
+  };
+
+  return { run, docker, failure, inspect, findByLabel, removeOne, removeStoppedContainer, removeByName };
+}

package/server/lib/spaces/places/docker-live-support.js

@@ -0,0 +1,202 @@
+// Test support for the suites that talk to a real Docker daemon.
+// They run only with OPENCHAMBER_TEST_DOCKER=1.
+
+import crypto from 'node:crypto';
+
+import { expect } from 'vitest';
+
+import { LABEL_MARKER, LABEL_OWNER, ROLE_GATEKEEPER, ROLE_NETWORK, ROLE_OUTER_NETWORK, ROLE_SPACE, spaceResourceName } from '../labels.js';
+import { runCommand } from '../run-command.js';
+import { createRegistryToolsSource, readHostToolVersions } from '../tools.js';
+import { SPACE_BASE_IMAGE, createDockerPlace } from './docker.js';
+
+export const LIVE_DOCKER_ENABLED = process.env.OPENCHAMBER_TEST_DOCKER === '1';
+
+const LISTINGS = [
+  ['ps', '--all', '--format', '{{.Names}}'],
+  ['network', 'ls', '--format', '{{.Name}}'],
+  ['volume', 'ls', '--format', '{{.Name}}'],
+];
+
+const HOST_LISTENER = (port) => `
+const routes = require('node:fs').readFileSync('/proc/net/fib_trie', 'utf8');
+const local = [...routes.matchAll(/\\|-- (\\d+\\.\\d+\\.\\d+\\.\\d+)\\n\\s+\\/32 host LOCAL/g)].map((match) => match[1]);
+const addresses = [...new Set(local)].filter((address) => !address.startsWith('127.'));
+require('node:net').createServer((socket) => { socket.on('error', () => {}); socket.end('host'); }).listen(${port}, '0.0.0.0', () => console.log(JSON.stringify(addresses)));
+`;
+
+/**
+ * A Docker place with an owner id of its own, so parallel runs and real spaces stay apart.
+ * The owner also gets a tools volume of its own, so the first create of every live file pays for one fill.
+ * `placeWith(toolsSource)` makes another place for the same owner, as a host with other tools would be.
+ */
+export function createLiveDockerPlace({ toolsSource = createRegistryToolsSource(readHostToolVersions()) } = {}) {
+  const owner = `test-${crypto.randomBytes(6).toString('hex')}`;
+  const placeWith = (source) => createDockerPlace({ runCommand, dockerPath: 'docker', owner, toolsSource: source });
+  const place = placeWith(toolsSource);
+  const ownerFilter = ['--filter', `label=${LABEL_MARKER}`, '--filter', `label=${LABEL_OWNER}=${owner}`];
+  // Helper containers carry the marker and the owner, so the leftover check sees them. They have no space id.
+  const helperLabels = ['--label', `${LABEL_MARKER}=true`, '--label', `${LABEL_OWNER}=${owner}`];
+
+  const docker = async (args, options) => {
+    const result = await runCommand('docker', args, options);
+    expect(result.code, `docker ${args[0]}: ${result.stderr}`).toBe(0);
+    return result;
+  };
+
+  const leftovers = async () => {
+    const names = [];
+    for (const listing of LISTINGS) {
+      names.push(...(await docker([...listing, ...ownerFilter])).stdout.split('\n').filter(Boolean));
+    }
+    return names;
+  };
+
+  const host = {
+    runUnrestricted: (script) => docker(['run', '--rm', ...helperLabels, SPACE_BASE_IMAGE, 'node', '-e', script], { timeoutMs: 120_000 }),
+
+    // `--network host` puts the listener on the Docker host itself: the Colima VM here, the machine on Linux.
+    // It prints every IPv4 address of that host first, read from the kernel's local route table.
+    // `os.networkInterfaces()` would miss a bridge that has no running container yet. On an engine that ignores the isolated gateway mode,
+    // that list holds the gateway of the space's own bridge, which is the address the leak was measured on.
+    startHostListener: async () => {
+      const port = 20_000 + crypto.randomInt(20_000);
+      const name = `openchamber-test-listener-${owner}`;
+      await docker(['run', '--detach', '--name', name, ...helperLabels, '--network', 'host', SPACE_BASE_IMAGE, 'node', '-e', HOST_LISTENER(port)], { timeoutMs: 120_000 });
+      let addresses = [];
+      for (let attempt = 0; attempt < 40 && addresses.length === 0; attempt += 1) {
+        await new Promise((resolve) => { setTimeout(resolve, 250); });
+        const firstLine = (await docker(['logs', name])).stdout.split('\n')[0];
+        addresses = firstLine.startsWith('[') ? JSON.parse(firstLine) : [];
+      }
+      expect(addresses.length, 'the host listener reported no address').toBeGreaterThan(0);
+      // Docker Desktop injects the two names. They are tolerated when they do not resolve.
+      return { port, candidates: [...addresses, 'host.docker.internal', 'gateway.docker.internal'], stop: () => docker(['rm', '--force', name]) };
+    },
+
+    // An internal network as the space network was before the isolated gateway mode. A probe from it
+    // must reach the host, or the host-listener test could never fail and proves nothing.
+    createPlainInternalNetwork: async () => {
+      const name = `openchamber-test-plain-${owner}`;
+      await docker(['network', 'create', '--driver', 'bridge', '--internal', '--ipv6=false', ...helperLabels, name]);
+      return {
+        run: (script) => docker(['run', '--rm', ...helperLabels, '--network', name, SPACE_BASE_IMAGE, 'node', '-e', script], { timeoutMs: 120_000 }),
+        remove: () => docker(['network', 'rm', name]),
+      };
+    },
+
+    // Everything the place's own records show about the space container. A secret must not be in here.
+    spaceMetadata: async (spaceId) => (await docker(['inspect', spaceResourceName(spaceId, ROLE_SPACE)])).stdout,
+
+    // The same for the gatekeeper, which is where the secrets are.
+    gatekeeperMetadata: async (spaceId) => (await docker(['inspect', spaceResourceName(spaceId, ROLE_GATEKEEPER)])).stdout,
+
+    /** The space container's own address on its inner network, as Docker gave it. */
+    spaceAddress: async (spaceId) => {
+      const [entry] = JSON.parse((await docker(['inspect', spaceResourceName(spaceId, ROLE_SPACE)])).stdout);
+      return entry.NetworkSettings.Networks[spaceResourceName(spaceId, ROLE_NETWORK)]?.IPAddress ?? '';
+    },
+
+    /** The gatekeeper's own addresses, `{ inner, outer }`, as Docker gave them. */
+    gatekeeperAddresses: async (spaceId) => {
+      const [entry] = JSON.parse((await docker(['inspect', spaceResourceName(spaceId, ROLE_GATEKEEPER)])).stdout);
+      const networks = entry.NetworkSettings.Networks;
+      return {
+        inner: networks[spaceResourceName(spaceId, ROLE_NETWORK)]?.IPAddress ?? '',
+        outer: networks[spaceResourceName(spaceId, ROLE_OUTER_NETWORK)]?.IPAddress ?? '',
+      };
+    },
+
+    /**
+     * A stand-in model provider on the space's outer network, where a real one would be. It
+     * answers with the sha256 of the credentials it saw, never with the credentials, so that a
+     * test can prove the real key travelled without handing it to the space.
+     */
+    startWindowUpstream: async (spaceId) => {
+      const name = `openchamber-test-upstream-${owner}`;
+      const program = `
+const crypto = require('node:crypto');
+const hash = (value) => (value === undefined ? 'none' : 'sha256:' + crypto.createHash('sha256').update(String(value)).digest('hex'));
+require('node:http').createServer((request, response) => {
+  response.writeHead(200, { 'content-type': 'application/json' });
+  response.end(JSON.stringify({
+    path: request.url,
+    authorization: hash(request.headers.authorization),
+    apiKey: hash(request.headers['x-api-key']),
+    headerNames: Object.keys(request.headers).sort(),
+  }));
+}).listen(9000, '0.0.0.0', () => console.log('upstream listening'));
+`;
+      await docker([
+        'run', '--detach', '--name', name, ...helperLabels,
+        '--network', spaceResourceName(spaceId, ROLE_OUTER_NETWORK), '--network-alias', 'upstream',
+        SPACE_BASE_IMAGE, 'node', '-e', program,
+      ], { timeoutMs: 120_000 });
+      for (let attempt = 0; attempt < 40; attempt += 1) {
+        await new Promise((resolve) => { setTimeout(resolve, 250); });
+        if ((await docker(['logs', name])).stdout.includes('upstream listening')) break;
+      }
+      return { url: 'http://upstream:9000/v1', stop: () => docker(['rm', '--force', name]) };
+    },
+
+    /**
+     * A name of our own in the space's outer network, answered by Docker's embedded resolver with
+     * an address in that bridge's subnet — a private range, and therefore one the corridor must
+     * refuse. It exists so that the resolve-then-refuse path can be proved live without a
+     * third-party wildcard DNS service: when somebody else's resolver has a bad minute, a test
+     * that leans on it reports an inconclusive run and proves nothing.
+     */
+    startBlockedName: async (spaceId) => {
+      const name = `openchamber-test-blocked-${owner}`;
+      // A last label that is not a number, because the corridor takes names and this one has to
+      // get past the name rule to reach the address rule that is under test.
+      const alias = 'blocked-by-address.test';
+      const network = spaceResourceName(spaceId, ROLE_OUTER_NETWORK);
+      await docker([
+        'run', '--detach', '--name', name, ...helperLabels,
+        '--network', network, '--network-alias', alias,
+        SPACE_BASE_IMAGE, 'sleep', '3600',
+      ], { timeoutMs: 120_000 });
+      const [entry] = JSON.parse((await docker(['inspect', name])).stdout);
+      return {
+        alias,
+        address: entry.NetworkSettings.Networks[network]?.IPAddress ?? '',
+        stop: () => docker(['rm', '--force', name]),
+      };
+    },
+
+    /** The names of this owner's volumes. */
+    volumes: async () => (await docker([...LISTINGS[2], ...ownerFilter])).stdout.split('\n').filter(Boolean),
+
+    logBytes: async (spaceId) => {
+      const result = await docker(['logs', spaceResourceName(spaceId, ROLE_SPACE)], { maxOutputBytes: 128 * 1024 * 1024, timeoutMs: 120_000 });
+      return Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr);
+    },
+  };
+
+  /** Removes every space, helper and tools volume of this run, then asserts that Docker holds nothing with this owner label. */
+  const dispose = async () => {
+    // Helpers first: one of them sits on a space's outer network, and a network with a
+    // container on it cannot go.
+    const spaces = new Set((await place.list()).flatMap((space) => [spaceResourceName(space.id, ROLE_SPACE), spaceResourceName(space.id, ROLE_GATEKEEPER)]));
+    for (const helper of (await docker([...LISTINGS[0], ...ownerFilter])).stdout.split('\n').filter(Boolean)) {
+      if (!spaces.has(helper)) await docker(['rm', '--force', helper]);
+    }
+    for (const space of await place.list()) {
+      await place.remove(space.id);
+    }
+    for (const helper of (await docker([...LISTINGS[0], ...ownerFilter])).stdout.split('\n').filter(Boolean)) {
+      await docker(['rm', '--force', helper]);
+    }
+    for (const network of (await docker([...LISTINGS[1], ...ownerFilter])).stdout.split('\n').filter(Boolean)) {
+      await docker(['network', 'rm', network]);
+    }
+    // What is left now are the tools volumes. No space mounts them any more.
+    for (const volume of await host.volumes()) {
+      await docker(['volume', 'rm', volume]);
+    }
+    expect(await leftovers()).toEqual([]);
+  };
+
+  return { place, placeWith, dispose, host, owner };
+}

package/server/lib/spaces/places/docker-tools.js

@@ -0,0 +1,186 @@
+// The tools volume of the Docker place: one volume per owner and per tools content,
+// filled once by a one-shot container and mounted read-only into spaces.
+// A filled volume is never changed again. New content means a new key and a new volume.
+
+import { SpaceError } from '../errors.js';
+import { TOOLS_MARKER_MISSING_EXIT_CODE, buildToolsCheckRunArgs, buildToolsFillRunArgs } from '../hardening.js';
+import {
+  ROLE_TOOLS,
+  ROLE_TOOLS_CHECK,
+  ROLE_TOOLS_FILL,
+  buildToolsLabels,
+  labelArgs,
+  parseToolsLabels,
+  toolsLabelFilterArgs,
+  toolsResourceName,
+} from '../labels.js';
+import { buildFillInput } from '../tools-filler.js';
+import { requireToolsSource, toolsContentKey } from '../tools.js';
+import { CHANGE_TIMEOUT_MS, ROLLBACK_SETTLE_MS, entryLabels, entryName, isInterrupted } from './docker-engine.js';
+
+// A fill downloads about 440 MB. It took 36 seconds here and may take many minutes on a slow link.
+const FILL_TIMEOUT_MS = 30 * 60_000;
+// npm prints one line per warning, and a broken install prints a long error report.
+const FILL_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
+const ERROR_TAIL_CHARACTERS = 2_000;
+
+export function createDockerTools({ engine, owner, toolsSource, image, now, wait }) {
+  const source = requireToolsSource(toolsSource);
+  const key = toolsContentKey(source, image);
+  const volume = toolsResourceName(owner, key);
+  let inFlight = null;
+
+  const labelsFor = (role) => labelArgs(buildToolsLabels({ role, owner, key, description: source.description, created: now().toISOString() }));
+  const oneShotArgs = (role) => ({ containerName: toolsResourceName(owner, key, role), labelArguments: labelsFor(role), toolsVolume: volume, image });
+
+  /** This owner's tools resources of one kind, checked again on the inspect result. */
+  const findOurs = async (kind, filter) => (await engine.findByLabel(kind, toolsLabelFilterArgs({ owner, ...filter })))
+    .filter((entry) => parseToolsLabels(entryLabels(kind, entry))?.owner === owner)
+    .map(entryName);
+
+  /** One-shot containers of this key that a killed CLI left behind. They would block the container name. */
+  const removeOneShots = async () => {
+    const failed = [];
+    for (const name of await findOurs('container', { key })) {
+      const problem = await engine.removeOne('container', name);
+      if (problem) failed.push(problem);
+    }
+    return failed;
+  };
+
+  /**
+   * What the fill marker says: `filled`, `missing`, or `wrong` for a marker with other content.
+   * A docker CLI that cannot reach the daemon exits with 1, as `cat` does for a missing file, so
+   * "missing" has an exit code of its own. Every other failure rejects: it must never read as
+   * "not filled", because that would remove a filled volume.
+   */
+  const readMarker = async () => {
+    const args = buildToolsCheckRunArgs(oneShotArgs(ROLE_TOOLS_CHECK));
+    const result = await engine.run(args, CHANGE_TIMEOUT_MS);
+    if (result.code === TOOLS_MARKER_MISSING_EXIT_CODE) {
+      return 'missing';
+    }
+    if (result.code !== 0) {
+      throw engine.failure(args, result);
+    }
+    return result.stdout.trim() === key ? 'filled' : 'wrong';
+  };
+
+  /** Removes what a failed fill made. The volume goes by name: this call found the name free, or found it ours. */
+  const cleanUpFill = async () => {
+    try {
+      return [...(await removeOneShots()), ...(await engine.removeByName([['volume', volume]]))];
+    } catch (error) {
+      return [{ kind: 'volume', name: volume, message: error.message }];
+    }
+  };
+
+  const fill = async () => {
+    try {
+      await engine.docker(['volume', 'create', ...labelsFor(ROLE_TOOLS), volume], CHANGE_TIMEOUT_MS);
+      const result = await engine.run(buildToolsFillRunArgs(oneShotArgs(ROLE_TOOLS_FILL)), FILL_TIMEOUT_MS, {
+        stdin: buildFillInput({ key, packageJson: source.packageJson, files: source.files }),
+        maxOutputBytes: FILL_MAX_OUTPUT_BYTES,
+      });
+      if (result.code !== 0) {
+        throw new SpaceError(
+          'tools_fill_failed',
+          `Installing ${source.description} failed: ${result.stderr.trim().slice(-ERROR_TAIL_CHARACTERS) || `exit code ${result.code}`}. Common causes: the Docker machine cannot reach the npm registry, or this version was never published.`,
+        );
+      }
+      if ((await readMarker()) !== 'filled') {
+        throw new SpaceError('tools_fill_failed', `Installing ${source.description} ended without the fill marker, or the space user cannot read it.`);
+      }
+    } catch (error) {
+      let rollbackFailures = await cleanUpFill();
+      // The CLI was killed, the daemon may still run the filler. Sweep again after a pause.
+      const uncertain = isInterrupted(error);
+      if (uncertain) {
+        await wait(ROLLBACK_SETTLE_MS);
+        rollbackFailures = await cleanUpFill();
+      }
+      const leftovers = rollbackFailures.length > 0
+        ? ` Clean-up also failed for: ${rollbackFailures.map((item) => `${item.kind} ${item.name}`).join(', ')}.`
+        : '';
+      const wrapped = new SpaceError(error.code ?? 'tools_fill_failed', `Could not prepare the tools for spaces. ${error.message}${leftovers}`, { rollbackFailures, uncertain });
+      wrapped.cause = error;
+      throw wrapped;
+    }
+  };
+
+  /**
+   * Removes this owner's other tools volumes. Docker refuses a volume that a container
+   * still mounts, and that refusal means "a space still runs on it, keep it".
+   * Nothing here may fail the call that asked for it.
+   */
+  const prune = async () => {
+    try {
+      for (const name of await findOurs('volume', { role: ROLE_TOOLS })) {
+        if (name !== volume) await engine.removeOne('volume', name);
+      }
+    } catch {
+      // The next fill or the next recreate tries again.
+    }
+  };
+
+  const ensureOnce = async () => {
+    const existing = await engine.inspect('volume', volume);
+    const labels = parseToolsLabels(existing?.Labels);
+    // `docker volume create` succeeds silently on an existing name, so a stranger's volume is never adopted.
+    if (existing && (labels?.role !== ROLE_TOOLS || labels?.owner !== owner || labels?.key !== key)) {
+      throw new SpaceError('tools_volume_not_ours', `Docker already has a volume named ${volume} that this OpenChamber installation did not create. Remove it, then try again.`);
+    }
+    const leftovers = await removeOneShots();
+    if (leftovers.length > 0) {
+      throw new SpaceError('tools_cleanup_failed', `Could not remove an old tools container: ${leftovers.map((item) => `${item.name} (${item.message})`).join(', ')}`);
+    }
+    if (existing) {
+      const marker = await readMarker();
+      if (marker === 'filled') {
+        return volume;
+      }
+      // Ours by label. No marker means a fill was interrupted. A wrong or empty marker means the
+      // Docker machine went down right after a fill, or something else wrote here. Either way the
+      // content cannot be trusted, so start over. Docker refuses while any container mounts the volume.
+      const problem = await engine.removeOne('volume', volume);
+      if (problem && marker === 'wrong') {
+        throw new SpaceError(
+          'tools_marker_mismatch',
+          `The tools for spaces are damaged and must be installed again, but spaces still use them. Apply or discard the work in the spaces that were made with ${source.description}, remove those spaces, then try again.`,
+          { volume, reason: problem.message },
+        );
+      }
+      if (problem) {
+        throw new SpaceError('tools_volume_stuck', `The tools volume ${volume} was never filled to the end and could not be removed: ${problem.message}`);
+      }
+    }
+    await fill();
+    await prune();
+    return volume;
+  };
+
+  /** Resolves the name of the filled tools volume. Concurrent calls share one run, so one key is never filled twice at once. */
+  const ensure = () => {
+    inFlight ??= ensureOnce().finally(() => { inFlight = null; });
+    return inFlight;
+  };
+
+  /**
+   * For the failure path of a create that got `name` back from `ensure` as ours and filled.
+   * If another process pruned it in between, `docker create` made it again without labels, and
+   * every later call would refuse it as a stranger's. A volume with no labels at all is that
+   * leftover, and goes by name. A volume with labels, ours or anyone's, is never touched here.
+   */
+  const removeIfUnlabelled = async (name) => {
+    try {
+      const entry = await engine.inspect('volume', name);
+      const unlabelled = entry && Object.keys(entry.Labels ?? {}).length === 0;
+      const problem = unlabelled ? await engine.removeOne('volume', name) : null;
+      return problem ? [problem] : [];
+    } catch (error) {
+      return [{ kind: 'volume', name, message: error.message }];
+    }
+  };
+
+  return { ensure, prune, removeIfUnlabelled };
+}