@lakutata/containerd 0.0.0-stage → 3.0.0-beta.1

package/dist/esm/client/ContainerStore.js

@@ -0,0 +1,1339 @@
+import { randomBytes } from 'node:crypto';
+import { availableParallelism, constants as osConstants, hostname as osHostname, totalmem } from 'node:os';
+import { copyFile, lstat, mkdir, open, readdir, readFile, readlink, rm, stat, truncate, writeFile } from 'node:fs/promises';
+import { createReadStream, existsSync, readdirSync } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { pack as tarPack } from 'tar-stream';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { ContainerdClient, isGRPCError, PROCESS_TYPE_URL, SPEC_TYPE_URL, timestampDate } from './ContainerdClient.js';
+import { GRPCStatus, decode, encode } from '@lakutata/grpc';
+import * as M from './Messages.js';
+import { formatPlatform, hostPlatform, parsePlatform } from './Platforms.js';
+import { chainIds } from './ImageStore.js';
+import { ProcessFifo } from './Fifo.js';
+import { VolumeStore } from './Volumes.js';
+import { DEFAULT_NETWORK } from './Networks.js';
+import { DirectoryTree, layerDirectories, LayersTree, resolveUser } from './Rootfs.js';
+import { containerCapabilities, execProcessSpec, generateSpec, parseDeviceCgroupRule } from './Spec.js';
+import { defaultSeccompProfile, seccompSupported } from './Seccomp.js';
+/**
+ * The labels of the containers: the ones of nerdctl (nerdctl ps, nerdctl logs know them), the ones of the restart
+ * monitor of containerd
+ */
+export const ContainerLabels = {
+    NAME: 'nerdctl/name',
+    HOSTNAME: 'nerdctl/hostname',
+    PLATFORM: 'nerdctl/platform',
+    LOG_URI: 'nerdctl/log-uri',
+    STATE_DIR: 'nerdctl/state-dir',
+    NETWORKS: 'nerdctl/networks',
+    PORTS: 'nerdctl/ports',
+    MOUNTS: 'nerdctl/mounts',
+    STOP_SIGNAL: 'io.containerd.image.config.stop-signal',
+    RESTART_POLICY: 'containerd.io/restart.policy',
+    RESTART_STATUS: 'containerd.io/restart.status',
+    RESTART_LOG_URI: 'containerd.io/restart.loguri'
+};
+/**
+ * The resolv.conf of a container: the one of the host, without its local name servers on its own network (the ones of
+ * Google when it has no other one, as Docker does)
+ * @param content
+ * @param ownNetwork
+ */
+export function resolvConf(content, ownNetwork) {
+    if (!ownNetwork)
+        return content;
+    const lines = content.split('\n').filter((line) => !/^\s*nameserver\s+(127\.|::1\s*$)/.test(line));
+    while (lines.length && !lines[lines.length - 1].trim())
+        lines.pop();
+    if (!lines.some((line) => /^\s*nameserver\s+/.test(line)))
+        lines.push('nameserver 8.8.8.8', 'nameserver 8.8.4.4');
+    return lines.join('\n') + '\n';
+}
+/**
+ * An error of the runtime refusing the profile of seccomp of a container (a runc built without seccomp), with what to do
+ * @param error
+ */
+export function seccompHint(error) {
+    const message = error?.message ?? '';
+    if (!/seccomp/i.test(message) || !/not (supported|enabled)/i.test(message))
+        return error;
+    return Object.assign(error, { message: `${message} (the runtime of this machine does not filter the system calls with seccomp: create the container with the profile of seccomp unconfined)` });
+}
+/**
+ * The number of a signal (SIGTERM, TERM, 15)
+ * @param signal
+ */
+export function signalNumber(signal) {
+    if (typeof signal === 'number')
+        return signal;
+    if (/^\d+$/.test(signal))
+        return Number(signal);
+    const name = signal.toUpperCase().startsWith('SIG') ? signal.toUpperCase() : `SIG${signal.toUpperCase()}`;
+    const number = osConstants.signals[name];
+    if (number === undefined)
+        throw new Error(`The signal ${signal} is unknown`);
+    return number;
+}
+let swapLimitSupported;
+/**
+ * Whether the kernel limits the swap of the cgroups (Docker discards the swap limits otherwise): the files of the swap
+ * limits, in the cgroups below the root on cgroup v2 (the root has none)
+ */
+export function isSwapLimitSupported() {
+    if (swapLimitSupported !== undefined)
+        return swapLimitSupported;
+    if (existsSync('/sys/fs/cgroup/cgroup.controllers')) {
+        try {
+            swapLimitSupported = readdirSync('/sys/fs/cgroup', { withFileTypes: true }).some((entry) => entry.isDirectory() && existsSync(path.join('/sys/fs/cgroup', entry.name, 'memory.swap.max')));
+        }
+        catch {
+            swapLimitSupported = false;
+        }
+    }
+    else {
+        swapLimitSupported = existsSync('/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes');
+    }
+    return swapLimitSupported;
+}
+/**
+ * The swap limit of a container (its memory and its swap together), as Docker and nerdctl give it: twice its memory
+ * limit by default, -1 unlimited; none without a memory limit, or when the kernel does not limit the swap
+ * @param memory
+ * @param memorySwap
+ */
+export function swapLimit(memory, memorySwap) {
+    if (!memory || !isSwapLimitSupported())
+        return undefined;
+    return memorySwap === -1 || (memorySwap && memorySwap > 0) ? memorySwap : memory * 2;
+}
+/**
+ * The CPUs of the machine online, as a cpuset writes them (0-3,6)
+ */
+async function onlineCpuSet() {
+    return (await readFile('/sys/devices/system/cpu/online', 'utf8').catch(() => '')).trim() || `0-${availableParallelism() - 1}`;
+}
+/**
+ * The counters of a metrics message as numbers: the 64-bit ones beyond 2^53 (an unlimited limit, 2^64 - 1) are decoded
+ * as bigints, the statistics are numbers
+ * @param value
+ */
+function metricCounters(value) {
+    if (typeof value === 'bigint')
+        return Number(value);
+    if (Array.isArray(value))
+        return value.map(metricCounters);
+    if (value && typeof value === 'object' && !(value instanceof Uint8Array))
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, metricCounters(item)]));
+    return value;
+}
+/**
+ * The containers of containerd: created from the images (their specifications of OCI, their snapshots), their tasks
+ * started, stopped, executed in; their logs written by the shims of containerd (file://), kept by the restart monitor
+ * of containerd as their restart policies ask
+ */
+export class ContainerStore {
+    constructor(client, images, networks, options, volumes) {
+        this.client = client;
+        this.images = images;
+        this.networks = networks;
+        this.options = options;
+        this.volumes = volumes ?? new VolumeStore(path.join(options.dataRoot, 'volumes', client.namespace));
+    }
+    /**
+     * The names of the volumes of a container
+     * @param container
+     */
+    static Volumes(container) {
+        try {
+            return JSON.parse(container.labels?.[ContainerLabels.MOUNTS] ?? '[]')
+                .filter((mount) => mount.Type === 'volume' && !!mount.Name)
+                .map((mount) => mount.Name);
+        }
+        catch {
+            return [];
+        }
+    }
+    /**
+     * The networks of CNI of a container
+     * @param network
+     * @protected
+     */
+    async cniNetworks(network) {
+        if (!Array.isArray(network))
+            return [];
+        if (!network.length)
+            throw new Error('The networks of the container are empty');
+        const networks = [];
+        for (const item of network) {
+            if (item.name === 'host' || item.name === 'none')
+                throw new Error(`The network ${item.name} is not one of CNI: give it alone`);
+            const found = await this.networks.get(item.name);
+            if (!found)
+                throw Object.assign(new Error(`The network ${item.name} does not exist`), { code: 'NETWORK_NOT_FOUND' });
+            if (networks.some((known) => known.network.name === found.name))
+                throw new Error(`The network ${found.name} is given twice`);
+            networks.push({ network: found, ip: item.ip, ipv6: item.ipv6 });
+        }
+        return networks;
+    }
+    /**
+     * The profile of seccomp of a container, as nerdctl gives it: none for the privileged ones, unconfined ones and when
+     * the kernel does not filter the system calls; the default one by its capabilities; the one of a file
+     * @param options
+     * @protected
+     */
+    async seccomp(options) {
+        const profile = options.seccompProfile ?? 'default';
+        if (options.privileged || profile === 'unconfined')
+            return undefined;
+        if (profile !== 'default') {
+            try {
+                return JSON.parse(await readFile(profile, 'utf8'));
+            }
+            catch (error) {
+                throw new Error(`The profile of seccomp ${profile} is not read: ${error.message}`, { cause: error });
+            }
+        }
+        if (!seccompSupported())
+            return undefined;
+        return defaultSeccompProfile(containerCapabilities(options));
+    }
+    /**
+     * The interfaces of a container on its networks of CNI (their addresses while it runs)
+     * @param container
+     */
+    async interfaces(container) {
+        return await this.networks.interfaces(this.stateDir(container.id));
+    }
+    /**
+     * The directory of the state of a container
+     * @param id
+     */
+    stateDir(id) {
+        return path.join(this.options.dataRoot, 'containers', this.client.namespace, id);
+    }
+    /**
+     * The file of the logs of a container
+     * @param id
+     */
+    logFile(id) {
+        return path.join(this.stateDir(id), `${id}.log`);
+    }
+    /**
+     * The state written of a container
+     * @param id
+     */
+    async readState(id) {
+        try {
+            return JSON.parse(await readFile(path.join(this.stateDir(id), 'state.json'), 'utf8'));
+        }
+        catch {
+            return {};
+        }
+    }
+    async writeState(id, state) {
+        await mkdir(this.stateDir(id), { recursive: true });
+        await writeFile(path.join(this.stateDir(id), 'state.json'), JSON.stringify({ ...await this.readState(id), ...state }));
+    }
+    /**
+     * The settings a container was created with by the component (its update creates it again with them)
+     * @param id
+     */
+    async readSettings(id) {
+        try {
+            return JSON.parse(await readFile(path.join(this.stateDir(id), 'settings.json'), 'utf8'));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async writeSettings(id, settings) {
+        await mkdir(this.stateDir(id), { recursive: true });
+        await writeFile(path.join(this.stateDir(id), 'settings.json'), JSON.stringify(settings));
+    }
+    /**
+     * Name a container again (its label of nerdctl)
+     * @param container
+     * @param name
+     */
+    async rename(container, name) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(name))
+            throw new Error(`The name of a container "${name}" is not valid ([a-zA-Z0-9][a-zA-Z0-9_.-]*)`);
+        const used = await this.client.listContainers([`labels."${ContainerLabels.NAME}"==${name}`]);
+        if (used.some((other) => other.id !== container.id))
+            throw Object.assign(new Error(`The name ${name} is used by another container`), { code: 'CONFLICT' });
+        await this.label(container, { [ContainerLabels.NAME]: name });
+    }
+    /**
+     * Find a container: by its identifier, by its name, by a prefix of its identifier (only one)
+     * @param idOrName
+     */
+    async find(idOrName) {
+        const byId = await this.client.getContainer(idOrName);
+        if (byId)
+            return byId;
+        const byName = await this.client.listContainers([`labels."${ContainerLabels.NAME}"==${idOrName}`]);
+        if (byName.length)
+            return byName[0];
+        if (!/^[0-9a-f]{3,}$/.test(idOrName))
+            return undefined;
+        const byPrefix = (await this.client.listContainers()).filter((container) => container.id.startsWith(idOrName));
+        if (byPrefix.length > 1)
+            throw new Error(`The prefix ${idOrName} names ${byPrefix.length} containers`);
+        return byPrefix[0];
+    }
+    /**
+     * The containers
+     * @param filters the filters of containerd (labels."app"==web)
+     */
+    async list(filters = []) {
+        return await this.client.listContainers(filters);
+    }
+    /**
+     * The devices of a host path (a device, or the ones of a directory)
+     * @param mapping
+     */
+    async devices(mapping) {
+        const stats = await stat(mapping.hostPath, { bigint: true });
+        const containerPath = mapping.containerPath ?? mapping.hostPath;
+        if (stats.isDirectory()) {
+            const devices = [];
+            for (const entry of await readdir(mapping.hostPath)) {
+                devices.push(...await this.devices({ hostPath: path.join(mapping.hostPath, entry), containerPath: path.posix.join(containerPath, entry), permissions: mapping.permissions }).catch(() => []));
+            }
+            return devices;
+        }
+        if (!stats.isCharacterDevice() && !stats.isBlockDevice())
+            throw new Error(`${mapping.hostPath} is not a device`);
+        return [ContainerStore.Device(mapping.hostPath, containerPath, stats, mapping.permissions ?? 'rwm')];
+    }
+    /**
+     * A device of the host, by the stats of its node (its major and minor numbers of its rdev, as glibc splits it)
+     * @param hostPath
+     * @param containerPath
+     * @param stats
+     * @param permissions
+     */
+    static Device(hostPath, containerPath, stats, permissions) {
+        const rdev = stats.rdev;
+        return {
+            path: hostPath,
+            containerPath: containerPath,
+            type: stats.isCharacterDevice() ? 'c' : 'b',
+            major: Number(((rdev >> 8n) & 0xfffn) | ((rdev >> 32n) & ~0xfffn)),
+            minor: Number((rdev & 0xffn) | ((rdev >> 12n) & ~0xffn)),
+            fileMode: Number(stats.mode & 511n),
+            uid: Number(stats.uid),
+            gid: Number(stats.gid),
+            permissions: permissions
+        };
+    }
+    /**
+     * The devices of the host given to a privileged container, as containerd's WithHostDevices gives them: the
+     * character and block devices of /dev and of its subdirectories, without the links, the console and the directories
+     * of the pseudo terminals, the shared memory, the descriptors and the queues
+     * @param directory
+     */
+    async hostDevices(directory = '/dev') {
+        const devices = [];
+        const entries = await readdir(directory, { withFileTypes: true }).catch(() => []);
+        for (const entry of entries) {
+            const entryPath = path.join(directory, entry.name);
+            if (entry.isDirectory()) {
+                if (!['pts', 'shm', 'fd', 'mqueue', '.lxc', '.lxd-mounts', '.udev'].includes(entry.name))
+                    devices.push(...await this.hostDevices(entryPath));
+                continue;
+            }
+            if (entry.name === 'console' || (!entry.isCharacterDevice() && !entry.isBlockDevice()))
+                continue;
+            //Removed meanwhile
+            const stats = await lstat(entryPath, { bigint: true }).catch(() => undefined);
+            if (stats && (stats.isCharacterDevice() || stats.isBlockDevice()))
+                devices.push(ContainerStore.Device(entryPath, entryPath, stats, 'rwm'));
+        }
+        return devices;
+    }
+    /**
+     * The arguments of the process of a container: its entrypoint and its command, as Docker composes them
+     * @param resolved
+     * @param options
+     */
+    args(resolved, options) {
+        const config = resolved.config.config ?? {};
+        const entrypoint = options.entrypoint ?? config.Entrypoint ?? [];
+        //An entrypoint given resets the command of the image
+        const cmd = options.cmd ?? (options.entrypoint ? [] : config.Cmd ?? []);
+        const args = [...entrypoint.filter((part, index) => !(index === 0 && part === '' && entrypoint.length === 1)), ...cmd];
+        if (!args.length)
+            throw new Error(`The image ${resolved.name} has no command: give the command of the container`);
+        return args;
+    }
+    /**
+     * Create a container of an image: its snapshot (its root file system, the image unpacked), its specification, its
+     * state directory (its hosts, resolv.conf, hostname, logs)
+     * @param options
+     */
+    async create(options) {
+        const platform = options.platform ?? hostPlatform();
+        const resolved = await this.images.resolve(options.image, platform);
+        const id = randomBytes(32).toString('hex');
+        const name = options.name ?? `${resolved.name.split('/').pop().split(/[:@]/)[0].replace(/[^a-zA-Z0-9_.-]/g, '')}-${id.slice(0, 5)}`;
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(name))
+            throw new Error(`The name of a container "${name}" is not valid ([a-zA-Z0-9][a-zA-Z0-9_.-]*)`);
+        if ((await this.client.listContainers([`labels."${ContainerLabels.NAME}"==${name}`])).length)
+            throw Object.assign(new Error(`The name ${name} is used by another container`), { code: 'CONFLICT' });
+        const network = Array.isArray(options.network) ? 'cni' : options.network ?? 'host';
+        const cniNetworks = await this.cniNetworks(options.network);
+        const ports = options.ports ?? [];
+        if (ports.length && network !== 'cni')
+            throw new Error(`The ports of a container are published on its networks of CNI (${DEFAULT_NETWORK}), not on the network ${network}`);
+        const hostname = options.hostname ?? (network === 'host' ? osHostname() : id.slice(0, 12));
+        const lease = await this.client.createLease();
+        const stateDir = this.stateDir(id);
+        try {
+            const chain = chainIds(resolved.config.rootfs?.diff_ids ?? []);
+            const parent = chain[chain.length - 1] ?? '';
+            if (parent && !(await this.client.statSnapshot(this.options.snapshotter, parent)))
+                await this.images.unpack(resolved.manifest, lease);
+            const mounts = await this.client.prepareSnapshot(this.options.snapshotter, id, parent, {}, lease);
+            const user = await resolveUser(mounts, options.user ?? resolved.config.config?.User);
+            //The environment: the one of the image, PATH, HOSTNAME, HOME, the one given
+            const env = new Map([['PATH', '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin']]);
+            for (const entry of resolved.config.config?.Env ?? [])
+                env.set(entry.split('=')[0], entry.slice(entry.indexOf('=') + 1));
+            env.set('HOSTNAME', hostname);
+            if (!env.has('HOME'))
+                env.set('HOME', user.home);
+            for (const [key, value] of Object.entries(options.env ?? {}))
+                env.set(key, value);
+            if (options.terminal && !env.has('TERM'))
+                env.set('TERM', 'xterm');
+            //The files of the network of the container
+            await mkdir(stateDir, { recursive: true });
+            const hostHosts = network === 'host' ? await readFile('/etc/hosts', 'utf8').catch(() => '') : '';
+            const hosts = [hostHosts.trimEnd() || '127.0.0.1\tlocalhost localhost.localdomain\n::1\tlocalhost localhost.localdomain ip6-localhost ip6-loopback'];
+            if (network === 'none')
+                hosts.push(`127.0.0.1\t${hostname}`);
+            for (const [host, address] of Object.entries(options.extraHosts ?? {}))
+                hosts.push(`${address}\t${host}`);
+            //The hosts of the networks of CNI: the address of the container added by its hook
+            await writeFile(path.join(stateDir, 'hosts.base'), hosts.join('\n') + '\n');
+            await writeFile(path.join(stateDir, 'hosts'), hosts.join('\n') + '\n');
+            await writeFile(path.join(stateDir, 'resolv.conf'), options.dns?.length ? options.dns.map((server) => `nameserver ${server}`).join('\n') + '\n' : resolvConf(await readFile('/etc/resolv.conf', 'utf8').catch(() => ''), network !== 'host'));
+            await writeFile(path.join(stateDir, 'hostname'), hostname + '\n');
+            await writeFile(this.logFile(id), '');
+            const specMounts = [
+                { source: path.join(stateDir, 'hosts'), destination: '/etc/hosts', options: ['bind', 'rprivate'] },
+                { source: path.join(stateDir, 'resolv.conf'), destination: '/etc/resolv.conf', options: ['bind', 'rprivate'] },
+                { source: path.join(stateDir, 'hostname'), destination: '/etc/hostname', options: ['bind', 'rprivate'] }
+            ];
+            for (const mount of options.mounts ?? []) {
+                if (mount.type === 'tmpfs') {
+                    specMounts.push({ type: 'tmpfs', source: 'tmpfs', destination: mount.target, options: ['nosuid', 'nodev', 'noexec', ...mount.options ?? [], ...mount.readonly ? ['ro'] : []] });
+                }
+                else if (mount.type === 'volume') {
+                    if (!mount.source)
+                        throw new Error(`The volume mounted on ${mount.target} has no name`);
+                    const volume = await this.volumes.create(mount.source);
+                    specMounts.push({ type: 'bind', source: volume.mountpoint, destination: mount.target, options: ['rbind', 'rprivate', mount.readonly ? 'ro' : 'rw'] });
+                }
+                else {
+                    if (!mount.source)
+                        throw new Error(`The bind mount of ${mount.target} has no source`);
+                    const source = path.resolve(mount.source);
+                    await mkdir(source, { recursive: true }).catch(() => undefined);
+                    specMounts.push({ type: 'bind', source: source, destination: mount.target, options: ['rbind', ...mount.options?.length ? mount.options : ['rprivate'], mount.readonly ? 'ro' : 'rw'] });
+                }
+            }
+            const devices = [];
+            for (const device of options.devices ?? [])
+                devices.push(...await this.devices(device));
+            //The devices of the host in its /dev (its cgroup allows them all), the ones given kept
+            if (options.privileged)
+                devices.push(...(await this.hostDevices()).filter((device) => !devices.some((given) => given.containerPath === device.containerPath)));
+            let hooks;
+            if (network === 'cni') {
+                if (!this.networks.pluginDirectories().length)
+                    throw new Error(`The plugins of CNI are not installed (${this.networks.cniPath.join(', ')}): install them (the package containernetworking-plugins, or the CNI plugins release in /opt/cni/bin), or give the container the network 'host'`);
+                await this.networks.render(stateDir, id, cniNetworks, ports);
+                const hook = path.join(stateDir, 'network', 'hook.sh');
+                hooks = {
+                    createRuntime: [{ path: '/bin/sh', args: ['sh', hook, 'add', stateDir], timeout: 120 }],
+                    poststop: [{ path: '/bin/sh', args: ['sh', hook, 'del', stateDir], timeout: 120 }]
+                };
+            }
+            const spec = generateSpec({
+                id: id,
+                namespace: this.client.namespace,
+                args: this.args(resolved, options),
+                env: [...env].map(([key, value]) => `${key}=${value}`),
+                cwd: options.workingDir ?? resolved.config.config?.WorkingDir ?? '/',
+                uid: user.uid,
+                gid: user.gid,
+                additionalGids: user.additionalGids,
+                hostname: hostname,
+                terminal: options.terminal,
+                privileged: options.privileged,
+                readonlyRootfs: options.readonlyRootfs,
+                capAdd: options.capAdd,
+                capDrop: options.capDrop,
+                seccomp: await this.seccomp(options),
+                mounts: specMounts,
+                devices: devices,
+                deviceCgroupRules: (options.deviceCgroupRules ?? []).map(parseDeviceCgroupRule),
+                network: { mode: network },
+                hooks: hooks,
+                pidHost: options.pidHost,
+                ipcHost: options.ipcHost,
+                cgroupNamespace: (options.cgroupns ?? 'private') === 'private' && existsSync('/sys/fs/cgroup/cgroup.controllers'),
+                cgroupsPath: this.options.cgroupManager === 'systemd' ? `system.slice:lakutata:${id}` : `/${this.client.namespace}/${id}`,
+                memory: options.memory,
+                memorySwap: swapLimit(options.memory, options.memorySwap),
+                cpuQuota: options.cpus ? Math.round(options.cpus * 100000) : undefined,
+                cpuShares: options.cpuShares,
+                cpuset: options.cpuset,
+                pidsLimit: options.pidsLimit,
+                oomKillDisable: options.oomKillDisable,
+                sysctls: options.sysctls,
+                shmSize: options.shmSize
+            });
+            const logUri = `file://${this.logFile(id)}`;
+            const labels = {
+                ...resolved.config.config?.Labels ?? {},
+                ...options.labels ?? {},
+                [ContainerLabels.NAME]: name,
+                [ContainerLabels.HOSTNAME]: hostname,
+                [ContainerLabels.PLATFORM]: formatPlatform(resolved.platform),
+                [ContainerLabels.LOG_URI]: logUri,
+                [ContainerLabels.STATE_DIR]: stateDir,
+                [ContainerLabels.NETWORKS]: JSON.stringify(network === 'cni' ? cniNetworks.map((item) => item.network.name) : [network]),
+                [ContainerLabels.PORTS]: JSON.stringify(ports.map((port) => ({ HostPort: port.hostPort, ContainerPort: port.containerPort, Protocol: port.protocol, HostIP: port.hostIP || '0.0.0.0' }))),
+                [ContainerLabels.MOUNTS]: JSON.stringify((options.mounts ?? []).map((mount) => mount.type === 'volume'
+                    ? { Type: 'volume', Name: mount.source, Source: path.join(this.volumes.directory, mount.source, '_data'), Destination: mount.target, RW: !mount.readonly }
+                    : { Type: mount.type ?? 'bind', Source: mount.source ?? '', Destination: mount.target, RW: !mount.readonly }))
+            };
+            const stopSignal = options.stopSignal ?? resolved.config.config?.StopSignal;
+            if (stopSignal)
+                labels[ContainerLabels.STOP_SIGNAL] = stopSignal;
+            if (options.restart && options.restart !== 'no') {
+                labels[ContainerLabels.RESTART_POLICY] = options.restart;
+                labels[ContainerLabels.RESTART_LOG_URI] = logUri;
+                labels[ContainerLabels.RESTART_STATUS] = 'stopped';
+            }
+            const runtime = { name: this.options.runtime };
+            if (this.options.cgroupManager === 'systemd')
+                runtime.options = { typeUrl: 'containerd.runc.v1.Options', value: encode(M.RuncOptions, { systemdCgroup: true }) };
+            return await this.client.createContainer({
+                id: id,
+                labels: labels,
+                image: resolved.name,
+                runtime: runtime,
+                spec: ContainerdClient.Json(SPEC_TYPE_URL, spec),
+                snapshotter: this.options.snapshotter,
+                snapshotKey: id
+            }, lease);
+        }
+        catch (error) {
+            await this.client.removeSnapshot(this.options.snapshotter, id).catch(() => undefined);
+            await rm(stateDir, { recursive: true, force: true });
+            throw error;
+        }
+        finally {
+            await this.client.deleteLease(lease);
+        }
+    }
+    /**
+     * The state of a container: the one of its task, its last exit when it has none
+     * @param container
+     */
+    async state(container) {
+        const stored = await this.readState(container.id);
+        const task = await this.client.getTask(container.id);
+        const restarting = container.labels?.[ContainerLabels.RESTART_STATUS] === 'running';
+        const state = {
+            status: 'created',
+            running: false,
+            paused: false,
+            pid: 0,
+            exitCode: stored.exitCode ?? 0,
+            startedAt: stored.startedAt ? new Date(stored.startedAt) : null,
+            finishedAt: stored.finishedAt ? new Date(stored.finishedAt) : null
+        };
+        if (!task) {
+            state.status = stored.startedAt ? 'exited' : 'created';
+            //Its restart monitor recreates its task
+            if (restarting && stored.startedAt)
+                state.status = 'restarting';
+            return state;
+        }
+        state.pid = task.pid ?? 0;
+        switch (task.status) {
+            case M.ProcessStatus.RUNNING:
+                state.status = 'running';
+                state.running = true;
+                break;
+            case M.ProcessStatus.PAUSED:
+                state.status = 'paused';
+                state.paused = true;
+                break;
+            case M.ProcessStatus.PAUSING:
+                state.status = 'pausing';
+                break;
+            case M.ProcessStatus.STOPPED:
+                state.status = restarting ? 'restarting' : 'exited';
+                state.exitCode = task.exitStatus ?? 0;
+                state.finishedAt = timestampDate(task.exitedAt) ?? state.finishedAt;
+                break;
+            case M.ProcessStatus.CREATED:
+                state.status = 'created';
+                break;
+            default:
+                state.status = 'unknown';
+        }
+        return state;
+    }
+    /**
+     * Rotate the logs of a container once they are larger than the size of the options: copied to the file .1, the
+     * file truncated (the shim writes to its end)
+     * @param id
+     */
+    async rotateLogs(id) {
+        if (!this.options.logMaxSize)
+            return;
+        const file = this.logFile(id);
+        const size = (await stat(file).catch(() => undefined))?.size ?? 0;
+        if (size <= this.options.logMaxSize)
+            return;
+        await copyFile(file, `${file}.1`);
+        await truncate(file, 0);
+    }
+    /**
+     * Delete the task of a container (stopped), its exit kept
+     * @param id
+     */
+    async deleteTask(id) {
+        const deleted = await this.client.deleteTask(id).catch(async (error) => {
+            //A task still running (a stop of its restart monitor meanwhile): killed
+            if (!isGRPCError(error, GRPCStatus.FAILED_PRECONDITION))
+                throw error;
+            await this.client.kill(id, signalNumber('SIGKILL'), { all: true }).catch(() => undefined);
+            await this.client.wait(id).catch(() => undefined);
+            return await this.client.deleteTask(id);
+        });
+        if (deleted)
+            await this.writeState(id, { exitCode: deleted.exitStatus ?? 0, finishedAt: (timestampDate(deleted.exitedAt) ?? new Date()).toISOString() });
+    }
+    /**
+     * Set the labels of a container
+     * @param container
+     * @param labels
+     */
+    async label(container, labels) {
+        await this.client.updateContainer({ id: container.id, labels: labels }, Object.keys(labels).map((key) => `labels.${key}`));
+    }
+    /**
+     * Start a container: a new task of its snapshot (the one of its previous run deleted), its output written to its
+     * logs
+     * @param container
+     */
+    async start(container) {
+        const task = await this.client.getTask(container.id);
+        if (task?.status === M.ProcessStatus.RUNNING)
+            return task.pid;
+        if (task?.status === M.ProcessStatus.PAUSED)
+            throw Object.assign(new Error(`The container ${container.id} is paused: resume it`), { code: 'PAUSED' });
+        if (task)
+            await this.deleteTask(container.id);
+        await this.rotateLogs(container.id);
+        const logUri = container.labels?.[ContainerLabels.LOG_URI] ?? `file://${this.logFile(container.id)}`;
+        await mkdir(path.dirname(this.logFile(container.id)), { recursive: true });
+        const rootfs = await this.client.snapshotMounts(container.snapshotter || this.options.snapshotter, container.snapshotKey || container.id);
+        await this.client.createTask({ containerId: container.id, rootfs: rootfs, stdout: logUri, stderr: logUri }).catch((error) => {
+            throw seccompHint(error);
+        });
+        try {
+            const pid = await this.client.startTask(container.id).catch((error) => {
+                throw seccompHint(error);
+            });
+            await this.writeState(container.id, { startedAt: new Date().toISOString(), finishedAt: undefined, exitCode: undefined });
+            if (container.labels?.[ContainerLabels.RESTART_POLICY])
+                await this.label(container, { [ContainerLabels.RESTART_STATUS]: 'running' });
+            return pid;
+        }
+        catch (error) {
+            await this.client.deleteTask(container.id).catch(() => undefined);
+            throw error;
+        }
+    }
+    /**
+     * Wait for the task of a container to exit, at most a time
+     * @param id
+     * @param timeout milliseconds
+     */
+    async waitExit(id, timeout) {
+        const abort = new AbortController();
+        const timer = setTimeout(() => abort.abort(), timeout);
+        try {
+            await this.client.wait(id, '', abort.signal);
+            return true;
+        }
+        catch (error) {
+            if (abort.signal.aborted)
+                return false;
+            if (isGRPCError(error, GRPCStatus.NOT_FOUND))
+                return true;
+            throw error;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /**
+     * Stop a container: its stop signal (SIGTERM, the one of its image) sent, then SIGKILL to all its processes after a
+     * time; its task deleted, its restart monitor told not to restart it
+     * @param container
+     * @param options
+     */
+    async stop(container, options = {}) {
+        if (container.labels?.[ContainerLabels.RESTART_POLICY])
+            await this.label(container, { [ContainerLabels.RESTART_STATUS]: 'stopped' });
+        const task = await this.client.getTask(container.id);
+        if (!task)
+            return;
+        if (task.status === M.ProcessStatus.PAUSED)
+            await this.client.resume(container.id);
+        if (task.status === M.ProcessStatus.RUNNING || task.status === M.ProcessStatus.PAUSED) {
+            const signal = signalNumber(options.signal ?? container.labels?.[ContainerLabels.STOP_SIGNAL] ?? 'SIGTERM');
+            await this.client.kill(container.id, signal).catch((error) => {
+                if (!isGRPCError(error, GRPCStatus.NOT_FOUND))
+                    throw error;
+            });
+            if (!(await this.waitExit(container.id, (options.timeout ?? 10) * 1000))) {
+                await this.client.kill(container.id, signalNumber('SIGKILL'), { all: true }).catch(() => undefined);
+                await this.waitExit(container.id, 10000);
+            }
+        }
+        await this.deleteTask(container.id);
+    }
+    /**
+     * Send a signal to a container (its main process, or all its processes)
+     * @param container
+     * @param signal
+     * @param all
+     */
+    async kill(container, signal = 'SIGKILL', all = false) {
+        if (signalNumber(signal) === signalNumber('SIGKILL') && container.labels?.[ContainerLabels.RESTART_POLICY])
+            await this.label(container, { [ContainerLabels.RESTART_STATUS]: 'stopped' });
+        await this.client.kill(container.id, signalNumber(signal), { all: all }).catch((error) => {
+            throw this.taskError(container, error);
+        });
+    }
+    async pause(container) {
+        await this.running(container);
+        await this.client.pause(container.id).catch((error) => {
+            throw this.taskError(container, error);
+        });
+    }
+    async resume(container) {
+        await this.running(container);
+        await this.client.resume(container.id).catch((error) => {
+            throw this.taskError(container, error);
+        });
+    }
+    /**
+     * Check that a container has a task which did not exit (the shim refuses to pause or resume a stopped one with an
+     * error of no code)
+     * @param container
+     * @protected
+     */
+    async running(container) {
+        const task = await this.client.getTask(container.id);
+        if (!task || task.status === M.ProcessStatus.STOPPED)
+            throw Object.assign(new Error(`The container ${ContainerStore.Name(container)} is not running`), { code: 'NOT_RUNNING' });
+    }
+    /**
+     * The processes of a running container (their identifiers on the host)
+     * @param container
+     */
+    async processes(container) {
+        return await this.client.listPids(container.id).catch((error) => {
+            throw this.taskError(container, error);
+        });
+    }
+    /**
+     * Wait for the main process of a container to exit: its exit code; the last one at once when it has no task (never
+     * started: 0, stopped: its task deleted, its exit kept by the store)
+     * @param container
+     * @param signal
+     */
+    async wait(container, signal) {
+        try {
+            return (await this.client.wait(container.id, '', signal)).exitStatus;
+        }
+        catch (error) {
+            if (!isGRPCError(error, GRPCStatus.NOT_FOUND))
+                throw error;
+            return (await this.readState(container.id)).exitCode ?? 0;
+        }
+    }
+    /**
+     * The error of an operation on the task of a container: the container is not running when it has no task (or its
+     * process exited)
+     * @param container
+     * @param error
+     * @protected
+     */
+    taskError(container, error) {
+        if (!isGRPCError(error, GRPCStatus.NOT_FOUND))
+            return error;
+        return Object.assign(new Error(`The container ${ContainerStore.Name(container)} is not running (${error.message})`, { cause: error }), { code: 'NOT_RUNNING' });
+    }
+    /**
+     * Remove a container: its task (a running one only when forced), its snapshot, its state directory
+     * @param container
+     * @param force
+     */
+    async remove(container, force = false) {
+        const task = await this.client.getTask(container.id);
+        if (task && task.status !== M.ProcessStatus.STOPPED && task.status !== M.ProcessStatus.CREATED) {
+            if (!force)
+                throw Object.assign(new Error(`The container ${ContainerStore.Name(container)} is running: stop it or remove it by force`), { code: 'RUNNING' });
+            if (container.labels?.[ContainerLabels.RESTART_POLICY])
+                await this.label(container, { [ContainerLabels.RESTART_STATUS]: 'stopped' });
+            if (task.status === M.ProcessStatus.PAUSED)
+                await this.client.resume(container.id).catch(() => undefined);
+            await this.client.kill(container.id, signalNumber('SIGKILL'), { all: true }).catch(() => undefined);
+            await this.waitExit(container.id, 10000);
+        }
+        if (task)
+            await this.client.deleteTask(container.id).catch(() => undefined);
+        //The networks its hook did not delete
+        await this.networks.teardown(this.stateDir(container.id));
+        await this.client.deleteContainer(container.id);
+        await this.client.removeSnapshot(container.snapshotter || this.options.snapshotter, container.snapshotKey || container.id);
+        await rm(this.stateDir(container.id), { recursive: true, force: true });
+    }
+    /**
+     * Execute a command in a running container, its output captured: its standard output and its error apart (FIFOs,
+     * mkfifo of the system), or together in its standard output without mkfifo (the files of the shim)
+     * @param container
+     * @param options
+     */
+    async exec(container, options) {
+        const task = await this.client.getTask(container.id);
+        if (task?.status !== M.ProcessStatus.RUNNING)
+            throw Object.assign(new Error(`The container ${ContainerStore.Name(container)} is not running`), { code: 'NOT_RUNNING' });
+        const spec = ContainerdClient.FromJson(container.spec) ?? {};
+        let user;
+        if (options.user)
+            user = await resolveUser(await this.client.snapshotMounts(container.snapshotter || this.options.snapshotter, container.snapshotKey || container.id), options.user);
+        const process = execProcessSpec(spec, {
+            args: options.cmd,
+            env: Object.entries(options.env ?? {}).map(([key, value]) => `${key}=${value}`),
+            cwd: options.workingDir,
+            uid: user?.uid,
+            gid: user?.gid,
+            additionalGids: user?.additionalGids
+        });
+        const execId = `exec-${randomBytes(8).toString('hex')}`;
+        const directory = path.join(this.stateDir(container.id), 'exec');
+        await mkdir(directory, { recursive: true });
+        const files = [];
+        const fifos = [];
+        const fifo = ProcessFifo.Supported();
+        if (!fifo && options.stdin !== undefined)
+            throw new Error('The standard input of an execution needs mkfifo');
+        try {
+            let stdout, stderr, stdin;
+            let request;
+            if (fifo) {
+                stdout = ProcessFifo.Create(path.join(directory, `${execId}-stdout`), 'read');
+                stderr = ProcessFifo.Create(path.join(directory, `${execId}-stderr`), 'read');
+                fifos.push(stdout, stderr);
+                if (options.stdin !== undefined)
+                    fifos.push(stdin = ProcessFifo.Create(path.join(directory, `${execId}-stdin`), 'write'));
+                request = { stdout: stdout.path, stderr: stderr.path, stdin: stdin?.path ?? '' };
+            }
+            else {
+                const output = path.join(directory, `${execId}.log`);
+                files.push(output);
+                await writeFile(output, '');
+                request = { stdout: `file://${output}`, stderr: `file://${output}` };
+            }
+            files.push(...fifos.map((item) => item.path));
+            await this.client.exec({ containerId: container.id, execId: execId, ...request, spec: ContainerdClient.Json(PROCESS_TYPE_URL, process) });
+            const waiting = this.client.wait(container.id, execId);
+            waiting.catch(() => undefined);
+            await this.client.startTask(container.id, execId);
+            const read = (item) => {
+                if (!item)
+                    return Promise.resolve('');
+                const socket = item.open();
+                const chunks = [];
+                return new Promise((resolve) => {
+                    socket.on('data', (chunk) => chunks.push(chunk));
+                    socket.on('error', () => resolve(Buffer.concat(chunks).toString('utf8')));
+                    socket.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+                });
+            };
+            const outputs = Promise.all([read(stdout), read(stderr)]);
+            if (stdin) {
+                //Written and closed, then the writer of the shim closed (CloseIO): the end of the input once read
+                const writer = stdin.open();
+                await new Promise((resolve) => void writer.end(options.stdin ?? '', () => resolve()));
+                await this.client.closeIO(container.id, execId).catch(() => undefined);
+            }
+            const timer = options.timeout ? setTimeout(() => {
+                this.client.kill(container.id, signalNumber('SIGKILL'), { execId: execId }).catch(() => undefined);
+            }, options.timeout) : undefined;
+            try {
+                const { exitStatus } = await waiting;
+                if (!fifo)
+                    return { exitCode: exitStatus, stdout: await readFile(files[0], 'utf8'), stderr: '' };
+                const [out, err] = await outputs;
+                return { exitCode: exitStatus, stdout: out, stderr: err };
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }
+        finally {
+            fifos.forEach((item) => item.close());
+            await this.client.deleteProcess(container.id, execId).catch(() => undefined);
+            for (const file of files)
+                await rm(file, { force: true });
+        }
+    }
+    /**
+     * The logs of a container: its last lines, followed while it runs (until the signal aborts)
+     * @param container
+     * @param options
+     */
+    logs(container, options = {}) {
+        const file = this.logFile(container.id);
+        const store = this;
+        return Readable.from((async function* () {
+            let content = Buffer.concat([await readFile(`${file}.1`).catch(() => Buffer.alloc(0)), await readFile(file).catch(() => Buffer.alloc(0))]);
+            let offset = (await stat(file).catch(() => undefined))?.size ?? 0;
+            if (options.tail !== undefined && options.tail >= 0) {
+                let end = content.length;
+                if (content[end - 1] === 0x0a)
+                    end--;
+                let start = end;
+                for (let lines = 0; start > 0; start--) {
+                    if (content[start - 1] === 0x0a && ++lines >= options.tail)
+                        break;
+                }
+                content = options.tail === 0 ? Buffer.alloc(0) : content.subarray(start);
+            }
+            if (content.length)
+                yield content;
+            if (!options.follow)
+                return;
+            let idle = 0;
+            while (!options.signal?.aborted) {
+                const size = (await stat(file).catch(() => undefined))?.size ?? 0;
+                //Rotated: read from its start
+                if (size < offset)
+                    offset = 0;
+                if (size > offset) {
+                    const handle = await open(file, 'r');
+                    try {
+                        const buffer = Buffer.alloc(size - offset);
+                        await handle.read(buffer, 0, buffer.length, offset);
+                        offset = size;
+                        yield buffer;
+                    }
+                    finally {
+                        await handle.close();
+                    }
+                    continue;
+                }
+                //Every second, whether it still runs
+                if (++idle % 4 === 0) {
+                    const task = await store.client.getTask(container.id);
+                    //Stopped, and not restarted by its restart monitor
+                    if ((!task || task.status === M.ProcessStatus.STOPPED) && (await store.client.getContainer(container.id))?.labels?.[ContainerLabels.RESTART_STATUS] !== 'running')
+                        break;
+                }
+                await sleep(250, undefined, { signal: options.signal }).catch(() => undefined);
+            }
+        })());
+    }
+    /**
+     * A sample of the resources a running container uses
+     * @param container
+     * @protected
+     */
+    async sample(container) {
+        const task = await this.client.getTask(container.id);
+        if (task?.status !== M.ProcessStatus.RUNNING && task?.status !== M.ProcessStatus.PAUSED)
+            throw Object.assign(new Error(`The container ${ContainerStore.Name(container)} is not running`), { code: 'NOT_RUNNING' });
+        const metric = await this.client.metrics(container.id);
+        const type = String(metric?.data?.typeUrl ?? '');
+        const data = metric?.data?.value ?? new Uint8Array();
+        const unlimited = (value) => !value || value >= 2 ** 62 ? totalmem() : value;
+        const onlineCpus = availableParallelism();
+        const stats = {
+            read: timestampDate(metric?.timestamp) ?? new Date(),
+            pids: { current: 0, limit: 0 },
+            cpu: { usage: 0, user: 0, system: 0, throttledPeriods: 0, throttledTime: 0, onlineCpus: onlineCpus },
+            memory: { usage: 0, workingSet: 0, limit: totalmem(), cache: 0, swap: 0, oomKills: 0 },
+            blkio: { readBytes: 0, writeBytes: 0, readOps: 0, writeOps: 0 },
+            networks: {}
+        };
+        if (type.endsWith('cgroups.v2.Metrics')) {
+            const metrics = metricCounters(decode(M.CgroupV2Metrics, data));
+            stats.pids = { current: metrics.pids?.current ?? 0, limit: metrics.pids?.limit ?? 0 };
+            stats.cpu = {
+                usage: (metrics.cpu?.usageUsec ?? 0) * 1000,
+                user: (metrics.cpu?.userUsec ?? 0) * 1000,
+                system: (metrics.cpu?.systemUsec ?? 0) * 1000,
+                throttledPeriods: metrics.cpu?.nrThrottled ?? 0,
+                throttledTime: (metrics.cpu?.throttledUsec ?? 0) * 1000,
+                onlineCpus: onlineCpus
+            };
+            const usage = metrics.memory?.usage ?? 0;
+            stats.memory = {
+                usage: usage,
+                workingSet: Math.max(0, usage - (metrics.memory?.inactiveFile ?? 0)),
+                limit: unlimited(metrics.memory?.usageLimit ?? 0),
+                cache: metrics.memory?.file ?? 0,
+                swap: metrics.memory?.swapUsage ?? 0,
+                oomKills: metrics.memoryEvents?.oomKill ?? 0
+            };
+            for (const entry of metrics.io?.usage ?? []) {
+                stats.blkio.readBytes += entry.rbytes ?? 0;
+                stats.blkio.writeBytes += entry.wbytes ?? 0;
+                stats.blkio.readOps += entry.rios ?? 0;
+                stats.blkio.writeOps += entry.wios ?? 0;
+            }
+        }
+        else if (type.endsWith('cgroups.v1.Metrics')) {
+            const metrics = metricCounters(decode(M.CgroupV1Metrics, data));
+            stats.pids = { current: metrics.pids?.current ?? 0, limit: metrics.pids?.limit ?? 0 };
+            stats.cpu = {
+                usage: metrics.cpu?.usage?.total ?? 0,
+                user: metrics.cpu?.usage?.user ?? 0,
+                system: metrics.cpu?.usage?.kernel ?? 0,
+                throttledPeriods: metrics.cpu?.throttling?.throttledPeriods ?? 0,
+                throttledTime: metrics.cpu?.throttling?.throttledTime ?? 0,
+                onlineCpus: metrics.cpu?.usage?.perCpu?.length || onlineCpus
+            };
+            const usage = metrics.memory?.usage?.usage ?? 0;
+            stats.memory = {
+                usage: usage,
+                workingSet: Math.max(0, usage - (metrics.memory?.totalInactiveFile ?? 0)),
+                limit: unlimited(metrics.memory?.usage?.limit ?? 0),
+                cache: metrics.memory?.cache ?? 0,
+                swap: Math.max(0, (metrics.memory?.swap?.usage ?? 0) - usage),
+                oomKills: metrics.memoryOomControl?.oomKill ?? 0
+            };
+            for (const entry of metrics.blkio?.ioServiceBytesRecursive ?? []) {
+                if (/^read$/i.test(entry.op))
+                    stats.blkio.readBytes += entry.value ?? 0;
+                if (/^write$/i.test(entry.op))
+                    stats.blkio.writeBytes += entry.value ?? 0;
+            }
+            for (const entry of metrics.blkio?.ioServicedRecursive ?? []) {
+                if (/^read$/i.test(entry.op))
+                    stats.blkio.readOps += entry.value ?? 0;
+                if (/^write$/i.test(entry.op))
+                    stats.blkio.writeOps += entry.value ?? 0;
+            }
+        }
+        //The interfaces of its namespace of network (without its loopback)
+        try {
+            for (const line of (await readFile(`/proc/${task.pid}/net/dev`, 'utf8')).split('\n').slice(2)) {
+                const [name, counters] = line.split(':');
+                if (!counters || name.trim() === 'lo')
+                    continue;
+                const values = counters.trim().split(/\s+/).map(Number);
+                stats.networks[name.trim()] = {
+                    rxBytes: values[0], rxPackets: values[1], rxErrors: values[2], rxDropped: values[3],
+                    txBytes: values[8], txPackets: values[9], txErrors: values[10], txDropped: values[11]
+                };
+            }
+        }
+        catch {
+            //Its process is gone
+        }
+        return stats;
+    }
+    /**
+     * The resources a running container uses: two samples, its use of the CPUs between them
+     * @param container
+     * @param interval milliseconds between the samples
+     */
+    async stats(container, interval = 1000) {
+        const first = await this.sample(container);
+        if (interval <= 0)
+            return { ...first, cpu: { ...first.cpu, percent: 0 } };
+        await sleep(interval);
+        const second = await this.sample(container);
+        const elapsed = (second.read.getTime() - first.read.getTime()) * 1e6 || interval * 1e6;
+        return { ...second, cpu: { ...second.cpu, percent: Math.max(0, (second.cpu.usage - first.cpu.usage) / elapsed * 100) } };
+    }
+    /**
+     * Open a terminal in a running container: a process executed with its pseudo terminal, its input and its output by
+     * FIFOs (mkfifo of the system)
+     * @param container
+     * @param options
+     */
+    async terminal(container, options) {
+        const task = await this.client.getTask(container.id);
+        if (task?.status !== M.ProcessStatus.RUNNING)
+            throw Object.assign(new Error(`The container ${ContainerStore.Name(container)} is not running`), { code: 'NOT_RUNNING' });
+        if (!ProcessFifo.Supported())
+            throw new Error('The terminals of the containers need mkfifo');
+        const spec = ContainerdClient.FromJson(container.spec) ?? {};
+        let user;
+        if (options.user)
+            user = await resolveUser(await this.client.snapshotMounts(container.snapshotter || this.options.snapshotter, container.snapshotKey || container.id), options.user);
+        const env = { TERM: 'xterm', ...options.env ?? {} };
+        const process = execProcessSpec(spec, {
+            args: options.cmd,
+            env: Object.entries(env).map(([key, value]) => `${key}=${value}`),
+            cwd: options.workingDir,
+            uid: user?.uid,
+            gid: user?.gid,
+            additionalGids: user?.additionalGids,
+            terminal: true
+        });
+        if (options.cols && options.rows)
+            process.consoleSize = { height: options.rows, width: options.cols };
+        const execId = `tty-${randomBytes(8).toString('hex')}`;
+        const directory = path.join(this.stateDir(container.id), 'exec');
+        await mkdir(directory, { recursive: true });
+        const stdin = ProcessFifo.Create(path.join(directory, `${execId}-stdin`), 'write');
+        const stdout = ProcessFifo.Create(path.join(directory, `${execId}-stdout`), 'read');
+        let cleaned = false;
+        const cleanup = async () => {
+            if (cleaned)
+                return;
+            cleaned = true;
+            stdin.close();
+            stdout.close();
+            await this.client.deleteProcess(container.id, execId).catch(() => undefined);
+            await rm(stdin.path, { force: true });
+            await rm(stdout.path, { force: true });
+        };
+        try {
+            await this.client.exec({ containerId: container.id, execId: execId, stdin: stdin.path, stdout: stdout.path, terminal: true, spec: ContainerdClient.Json(PROCESS_TYPE_URL, process) });
+            const waiting = this.client.wait(container.id, execId);
+            await this.client.startTask(container.id, execId);
+            const input = stdin.open();
+            const output = stdout.open();
+            input.on('error', () => undefined);
+            const exited = (async () => {
+                try {
+                    return (await waiting).exitStatus;
+                }
+                finally {
+                    //Its output read before the FIFOs are closed
+                    if (!output.readableEnded)
+                        await new Promise((resolve) => {
+                            const timer = setTimeout(resolve, 1000);
+                            output.once('end', () => {
+                                clearTimeout(timer);
+                                resolve();
+                            });
+                        });
+                    await cleanup();
+                }
+            })();
+            exited.catch(() => undefined);
+            return {
+                execId: execId,
+                input: input,
+                output: output,
+                resize: async (cols, rows) => this.client.resizePty(container.id, execId, cols, rows),
+                exited: exited,
+                close: async () => {
+                    if (cleaned)
+                        return;
+                    await this.client.kill(container.id, signalNumber('SIGKILL'), { execId: execId }).catch(() => undefined);
+                    await exited.catch(() => undefined);
+                }
+            };
+        }
+        catch (error) {
+            await cleanup();
+            throw error;
+        }
+    }
+    /**
+     * Update the resources of a container (its specification, the cgroup of its running task) and its restart policy,
+     * without restarting it (docker update)
+     * @param container
+     * @param update
+     */
+    async updateResources(container, update) {
+        const spec = ContainerdClient.FromJson(container.spec) ?? {};
+        const resources = spec.linux.resources ??= {};
+        //The limits removed: absent from the specification, given to the task as the values of no limit (the update of
+        //runc leaves a 0 or an empty value as it is)
+        const removed = {};
+        if (update.memory !== undefined || update.memorySwap !== undefined || update.oomKillDisable !== undefined) {
+            resources.memory ??= {};
+            if (update.memory === 0) {
+                //No swap limit without a memory limit
+                delete resources.memory.limit;
+                delete resources.memory.swap;
+                removed.memory = removed.swap = true;
+            }
+            else if (update.memory !== undefined) {
+                //A swap limit of twice the memory (the default one) follows the memory
+                if (update.memorySwap === undefined && resources.memory.limit > 0 && resources.memory.swap === resources.memory.limit * 2)
+                    resources.memory.swap = update.memory * 2;
+                resources.memory.limit = update.memory;
+            }
+            if (update.memorySwap !== undefined) {
+                if (update.memorySwap > 0 && !(resources.memory.limit > 0))
+                    throw Object.assign(new Error('The swap limit of a container (memorySwap) needs its memory limit (memoryLimit)'), { code: 'INVALID_VALUE' });
+                //0: no swap limit; discarded when the kernel does not limit the swap
+                if (!removed.swap && isSwapLimitSupported())
+                    resources.memory.swap = update.memorySwap || -1;
+            }
+            if (update.oomKillDisable !== undefined)
+                resources.memory.disableOOMKiller = update.oomKillDisable;
+            if (resources.memory.swap > 0 && resources.memory.swap < resources.memory.limit)
+                throw Object.assign(new Error('The swap limit of a container (memorySwap, its memory and its swap) is less than its memory limit (memoryLimit)'), { code: 'INVALID_VALUE' });
+        }
+        if (update.cpus !== undefined || update.cpuShares !== undefined || update.cpuset !== undefined) {
+            resources.cpu ??= {};
+            if (update.cpus !== undefined) {
+                resources.cpu.quota = update.cpus ? Math.round(update.cpus * 100000) : -1;
+                resources.cpu.period = 100000;
+            }
+            if (update.cpuShares === 0) {
+                delete resources.cpu.shares;
+                removed.shares = true;
+            }
+            else if (update.cpuShares !== undefined) {
+                resources.cpu.shares = update.cpuShares;
+            }
+            if (update.cpuset === '') {
+                delete resources.cpu.cpus;
+                removed.cpus = true;
+            }
+            else if (update.cpuset !== undefined) {
+                resources.cpu.cpus = update.cpuset;
+            }
+        }
+        if (update.pidsLimit !== undefined)
+            resources.pids = { limit: update.pidsLimit || -1 };
+        const labels = {};
+        if (update.restart !== undefined) {
+            const restart = update.restart === 'no' ? '' : update.restart;
+            labels[ContainerLabels.RESTART_POLICY] = restart;
+            labels[ContainerLabels.RESTART_LOG_URI] = restart ? container.labels?.[ContainerLabels.LOG_URI] ?? `file://${this.logFile(container.id)}` : '';
+            const task = await this.client.getTask(container.id);
+            labels[ContainerLabels.RESTART_STATUS] = restart && task?.status === M.ProcessStatus.RUNNING ? 'running' : restart ? 'stopped' : '';
+        }
+        //The labels emptied are deleted
+        await this.client.updateContainer({ id: container.id, spec: ContainerdClient.Json(SPEC_TYPE_URL, spec), labels: Object.fromEntries(Object.entries(labels).filter(([, value]) => !!value)) }, ['spec', ...Object.keys(labels).map((key) => `labels.${key}`)]);
+        const task = await this.client.getTask(container.id);
+        if (!task || task.status === M.ProcessStatus.STOPPED)
+            return;
+        const taskResources = structuredClone(resources);
+        if (removed.memory)
+            taskResources.memory.limit = -1;
+        if (removed.swap)
+            taskResources.memory.swap = -1;
+        //The default weight of the kernel
+        if (removed.shares)
+            taskResources.cpu.shares = 1024;
+        if (removed.cpus)
+            taskResources.cpu.cpus = await onlineCpuSet();
+        await this.client.updateTask(container.id, taskResources);
+    }
+    /**
+     * Commit a container: the changes of its file system as a layer on its image, a new image (its container paused
+     * meanwhile when it runs, by default)
+     * @param container
+     * @param options
+     */
+    async commit(container, options) {
+        const task = await this.client.getTask(container.id);
+        const pause = (options.pause ?? true) && task?.status === M.ProcessStatus.RUNNING;
+        if (pause)
+            await this.client.pause(container.id);
+        try {
+            return await this.images.commit({
+                image: container.image,
+                platform: container.labels?.[ContainerLabels.PLATFORM] ? parsePlatform(container.labels[ContainerLabels.PLATFORM]) : hostPlatform(),
+                snapshotKey: container.snapshotKey || container.id,
+                name: options.name,
+                changes: options.changes,
+                comment: options.comment,
+                author: options.author
+            });
+        }
+        finally {
+            if (pause)
+                await this.client.resume(container.id).catch(() => undefined);
+        }
+    }
+    /**
+     * Archive a path of the file system of a container (tar): the one of its running process (its mounts with it), or
+     * the one of its snapshot when it is not running; its symbolic links resolved in it
+     * @param container
+     * @param file an absolute path in the container
+     * @param destination
+     */
+    async archive(container, file, destination) {
+        const task = await this.client.getTask(container.id);
+        const running = !!task && task.status !== M.ProcessStatus.STOPPED && !!task.pid;
+        const tree = running
+            ? new DirectoryTree(`/proc/${task.pid}/root`)
+            : new LayersTree(layerDirectories(await this.client.snapshotMounts(container.snapshotter || this.options.snapshotter, container.snapshotKey || container.id)));
+        const resolved = await tree.resolve(file);
+        const root = await tree.lstat(resolved);
+        if (!root)
+            throw Object.assign(new Error(`The path ${file} does not exist in the container ${ContainerStore.Name(container)}`), { code: 'PATH_NOT_FOUND' });
+        const archive = tarPack();
+        const output = pipeline(archive, destination);
+        const entry = (header) => new Promise((resolve, reject) => void archive.entry(header, (error) => error ? reject(error) : resolve()));
+        const add = async (name, inner, stats, real) => {
+            const header = { name: name, mode: stats.mode & 0o7777, mtime: stats.mtime, uid: stats.uid, gid: stats.gid };
+            if (stats.isDirectory()) {
+                await entry({ ...header, type: 'directory' });
+                for (const child of await tree.readdir(inner)) {
+                    const childInner = path.posix.join(inner, child);
+                    const found = await tree.lstat(childInner);
+                    if (found)
+                        await add(`${name}/${child}`, childInner, found.stats, found.path);
+                }
+            }
+            else if (stats.isSymbolicLink()) {
+                await entry({ ...header, type: 'symlink', linkname: await readlink(real) });
+            }
+            else if (stats.isFile()) {
+                await new Promise((resolve, reject) => {
+                    const writer = archive.entry({ ...header, type: 'file', size: stats.size }, (error) => error ? reject(error) : resolve());
+                    pipeline(createReadStream(real), writer).catch(reject);
+                });
+            }
+            //The devices, the sockets and the pipes are not archived
+        };
+        try {
+            await add(path.posix.basename(resolved) || '.', resolved, root.stats, root.path);
+            archive.finalize();
+        }
+        catch (error) {
+            archive.destroy(error);
+        }
+        await output;
+    }
+    /**
+     * The name of a container
+     * @param container
+     */
+    static Name(container) {
+        return container.labels?.[ContainerLabels.NAME] ?? container.id;
+    }
+}