@lakutata/containerd 0.0.0-stage → 3.0.0-beta.0

package/dist/cjs/client/BuildKit.js

@@ -0,0 +1,536 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BuildKitProgress = exports.BuildKitSession = void 0;
+exports.goFileMode = goFileMode;
+exports.patternExpression = patternExpression;
+exports.walkDirectory = walkDirectory;
+exports.sendDirectory = sendDirectory;
+exports.buildKitBuild = buildKitBuild;
+exports.buildKitVersion = buildKitVersion;
+exports.buildKitPlatforms = buildKitPlatforms;
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
+const node_path_1 = __importDefault(require("node:path"));
+const node_stream_1 = require("node:stream");
+const promises_2 = require("node:stream/promises");
+const grpc_1 = require("@lakutata/grpc");
+const B = __importStar(require("./BuildKitMessages.js"));
+const DockerIgnore_js_1 = require("./DockerIgnore.js");
+const Platforms_js_1 = require("./Platforms.js");
+const Registry_js_1 = require("./Registry.js");
+const ContainerdClient_js_1 = require("./ContainerdClient.js");
+/**
+ * The service of BuildKit
+ */
+const CONTROL = '/moby.buildkit.v1.Control';
+/**
+ * The size of the chunks of the files sent
+ */
+const CHUNK_SIZE = 32 * 1024;
+/**
+ * The mode of a file as Go writes it (os.FileMode), the one of the stats of fsutil
+ * @param stats
+ */
+function goFileMode(stats) {
+    let mode = Number(stats.mode & 511n);
+    if (stats.isDirectory())
+        mode |= 0x80000000;
+    else if (stats.isSymbolicLink())
+        mode |= 0x08000000;
+    else if (stats.isFIFO())
+        mode |= 0x02000000;
+    else if (stats.isSocket())
+        mode |= 0x01000000;
+    else if (stats.isCharacterDevice())
+        mode |= 0x04000000 | 0x00200000;
+    else if (stats.isBlockDevice())
+        mode |= 0x04000000;
+    if (stats.mode & 2048n)
+        mode |= 0x00800000;
+    if (stats.mode & 1024n)
+        mode |= 0x00400000;
+    if (stats.mode & 512n)
+        mode |= 0x00100000;
+    return mode >>> 0;
+}
+/**
+ * The expression of a pattern of the paths of fsutil (filepath.Match, ** for any directories)
+ * @param pattern
+ */
+function patternExpression(pattern) {
+    const normalized = node_path_1.default.posix.normalize(pattern.replace(/\\/g, '/')).replace(/^\/+/, '').replace(/\/+$/, '');
+    let expression = '';
+    for (let index = 0; index < normalized.length; index++) {
+        const character = normalized[index];
+        if (character === '*') {
+            if (normalized[index + 1] === '*') {
+                //**/ any directories, ** anything
+                if (normalized[index + 2] === '/') {
+                    expression += '(?:.*/)?';
+                    index += 2;
+                }
+                else {
+                    expression += '.*';
+                    index++;
+                }
+            }
+            else {
+                expression += '[^/]*';
+            }
+        }
+        else if (character === '?') {
+            expression += '[^/]';
+        }
+        else if (character === '[') {
+            const end = normalized.indexOf(']', index + 1);
+            if (end < 0) {
+                expression += '\\[';
+            }
+            else {
+                expression += `[${normalized.slice(index + 1, end).replace(/^!/, '^').replace(/\\/g, '\\\\')}]`;
+                index = end;
+            }
+        }
+        else {
+            expression += character.replace(/[.+^${}()|\\]/g, '\\$&');
+        }
+    }
+    return new RegExp(`^${expression}$`);
+}
+/**
+ * The entries of a directory sent to BuildKit, in the order fsutil walks them (lexical, the directories before their
+ * content); the parents of the paths included are included
+ * @param root
+ * @param filter
+ */
+async function walkDirectory(root, filter = {}) {
+    const include = [...filter.include ?? []];
+    //The targets of the symbolic links followed (the Dockerfile linked)
+    for (const follow of filter.followPaths ?? []) {
+        try {
+            const absolute = node_path_1.default.join(root, follow);
+            if ((await (0, promises_1.lstat)(absolute)).isSymbolicLink()) {
+                const target = await (0, promises_1.readlink)(absolute);
+                const relative = node_path_1.default.posix.normalize(node_path_1.default.posix.join(node_path_1.default.posix.dirname(follow), target)).replace(/^\/+/, '');
+                if (!relative.startsWith('..'))
+                    include.push(relative);
+            }
+        }
+        catch {
+            //Not there
+        }
+    }
+    const includes = include.map(patternExpression);
+    const literals = include.map((pattern) => node_path_1.default.posix.normalize(pattern).replace(/^\/+/, '').split(/[*?[]/)[0]);
+    const ignore = new DockerIgnore_js_1.DockerIgnore((filter.exclude ?? []).join('\n'));
+    const included = (relative, directory) => {
+        if (!includes.length)
+            return 'yes';
+        const parts = relative.split('/');
+        for (let length = 1; length <= parts.length; length++) {
+            const prefix = parts.slice(0, length).join('/');
+            if (includes.some((expression) => expression.test(prefix)))
+                return 'yes';
+        }
+        //A parent of a path included
+        if (directory && literals.some((literal) => literal.startsWith(`${relative}/`) || (literal === '' && includes.length > 0)))
+            return 'parent';
+        return 'no';
+    };
+    const entries = [];
+    const walk = async (relativeDirectory) => {
+        const names = (await (0, promises_1.readdir)(node_path_1.default.join(root, relativeDirectory))).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
+        for (const name of names) {
+            const relative = relativeDirectory ? `${relativeDirectory}/${name}` : name;
+            const absolute = node_path_1.default.join(root, relative);
+            const stats = await (0, promises_1.lstat)(absolute, { bigint: true });
+            const directory = stats.isDirectory();
+            if (ignore.ignores(relative))
+                continue;
+            const inclusion = included(relative, directory);
+            if (inclusion === 'no')
+                continue;
+            //The sockets are not sent
+            if (stats.isSocket())
+                continue;
+            const stat = {
+                path: relative,
+                mode: goFileMode(stats),
+                uid: Number(stats.uid),
+                gid: Number(stats.gid),
+                modTime: stats.mtimeNs
+            };
+            if (!directory)
+                stat.size = Number(stats.size);
+            if (stats.isSymbolicLink())
+                stat.linkname = await (0, promises_1.readlink)(absolute);
+            if (stats.isCharacterDevice() || stats.isBlockDevice()) {
+                stat.devmajor = Number(((stats.rdev >> 8n) & 0xfffn) | ((stats.rdev >> 32n) & ~0xfffn));
+                stat.devminor = Number((stats.rdev & 0xffn) | ((stats.rdev >> 12n) & ~0xffn));
+            }
+            entries.push({ path: relative, absolute: absolute, stat: stat, file: stats.isFile() });
+            if (directory) {
+                const before = entries.length;
+                await walk(relative);
+                //A parent without any path included under it
+                if (inclusion === 'parent' && entries.length === before)
+                    entries.pop();
+            }
+        }
+    };
+    await walk('');
+    return entries;
+}
+/**
+ * Send a directory as fsutil sends it (FileSync.DiffCopy): its stats, then the files the receiver requests, until it
+ * finishes
+ * @param call
+ * @param root
+ */
+async function sendDirectory(call, root) {
+    const entries = await walkDirectory(root, {
+        include: call.metadata('include-patterns'),
+        exclude: call.metadata('exclude-patterns'),
+        followPaths: call.metadata('followpaths')
+    });
+    let pending = Promise.resolve();
+    const sendFile = async (id) => {
+        const entry = entries[id];
+        if (!entry)
+            throw new grpc_1.GRPCError(grpc_1.GRPCStatus.INVALID_ARGUMENT, `The file ${id} is not sent`, call.path);
+        if (entry.file) {
+            const handle = await (0, promises_1.open)(entry.absolute, 'r');
+            try {
+                const buffer = Buffer.alloc(CHUNK_SIZE);
+                for (;;) {
+                    const { bytesRead } = await handle.read(buffer, 0, CHUNK_SIZE, null);
+                    if (!bytesRead)
+                        break;
+                    await call.send(B.Packet, { type: B.PacketType.DATA, id: id, data: Buffer.from(buffer.subarray(0, bytesRead)) });
+                }
+            }
+            finally {
+                await handle.close();
+            }
+        }
+        await call.send(B.Packet, { type: B.PacketType.DATA, id: id });
+    };
+    const receiving = (async () => {
+        for await (const packet of call.requests(B.Packet)) {
+            switch (packet.type ?? B.PacketType.STAT) {
+                case B.PacketType.REQ: {
+                    const id = packet.id ?? 0;
+                    pending = pending.then(() => sendFile(id));
+                    break;
+                }
+                case B.PacketType.FIN:
+                    await pending;
+                    await call.send(B.Packet, { type: B.PacketType.FIN });
+                    return;
+                case B.PacketType.ERR:
+                    throw new grpc_1.GRPCError(grpc_1.GRPCStatus.ABORTED, `The receiver of ${root} failed: ${Buffer.from(packet.data ?? []).toString()}`, call.path);
+            }
+        }
+    })();
+    receiving.catch(() => undefined);
+    for (const entry of entries)
+        await call.send(B.Packet, { type: B.PacketType.STAT, stat: entry.stat });
+    await call.send(B.Packet, { type: B.PacketType.STAT });
+    await receiving;
+    await pending;
+}
+/**
+ * A session of BuildKit: a connection of HTTP/2 in the stream of Control.Session, where the daemon calls its methods
+ * (the files of the directories of the build, the results exported, the credentials of the registries, its health)
+ */
+class BuildKitSession {
+    #server;
+    #stream;
+    constructor(options) {
+        this.options = options;
+        this.id = (0, node_crypto_1.randomBytes)(16).toString('hex');
+        this.#server = new grpc_1.GRPCServer();
+        this.#server.method('/grpc.health.v1.Health/Check', async (call) => {
+            await call.request(B.HealthCheckRequest);
+            await call.send(B.HealthCheckResponse, { status: 1 });
+        });
+        this.#server.method('/moby.filesync.v1.FileSync/DiffCopy', async (call) => {
+            const name = call.metadata('dir-name')[0] ?? '';
+            const directory = this.options.directories[name];
+            if (!directory)
+                throw new grpc_1.GRPCError(grpc_1.GRPCStatus.NOT_FOUND, `The session has no directory ${name}`, call.path);
+            await sendDirectory(call, directory);
+        });
+        if (this.options.output) {
+            this.#server.method('/moby.filesync.v1.FileSend/DiffCopy', async (call) => {
+                const output = this.options.output();
+                try {
+                    for await (const message of call.requests(B.BytesMessage)) {
+                        if (!message.data?.length)
+                            continue;
+                        if (!output.write(Buffer.from(message.data)))
+                            await new Promise((resolve) => void output.once('drain', resolve));
+                    }
+                    output.end();
+                    await (0, promises_2.finished)(output);
+                }
+                catch (error) {
+                    output.destroy(error);
+                    throw error;
+                }
+            });
+        }
+        this.#server.method('/moby.filesync.v1.Auth/Credentials', async (call) => {
+            const host = (await call.request(B.CredentialsRequest)).host ?? '';
+            const credentials = this.credentials(host);
+            await call.send(B.CredentialsResponse, credentials?.identityToken ? { secret: credentials.identityToken } : { username: credentials?.username ?? '', secret: credentials?.password ?? '' });
+        });
+        this.#server.method('/moby.filesync.v1.Auth/FetchToken', async (call) => {
+            const request = await call.request(B.FetchTokenRequest);
+            const token = await (0, Registry_js_1.fetchRegistryToken)({ realm: request.realm, service: request.service, scopes: request.scopes ?? [] }, this.credentials(request.host ?? ''));
+            if (!token)
+                throw new grpc_1.GRPCError(grpc_1.GRPCStatus.UNAUTHENTICATED, `The registry ${request.host} gives no token`, call.path);
+            await call.send(B.FetchTokenResponse, { token: token.token, expiresIn: token.expiresIn ?? 0, issuedAt: token.issuedAt ?? 0 });
+        });
+        for (const method of ['GetTokenAuthority', 'VerifyTokenAuthority']) {
+            this.#server.method(`/moby.filesync.v1.Auth/${method}`, async (call) => {
+                throw new grpc_1.GRPCError(grpc_1.GRPCStatus.UNIMPLEMENTED, `${method} is not implemented`, call.path);
+            });
+        }
+    }
+    /**
+     * The credentials of a host of a registry (Docker Hub by its names)
+     * @param host
+     * @protected
+     */
+    credentials(host) {
+        const registry = ['registry-1.docker.io', 'index.docker.io', 'registry.hub.docker.com'].includes(host) ? 'docker.io' : host;
+        return this.options.credentials?.(registry) ?? (registry === host ? undefined : this.options.credentials?.(host));
+    }
+    /**
+     * Open the session: the stream of Control.Session, the methods of the session exposed by its headers
+     * @param connection
+     * @param signal
+     */
+    start(connection, signal) {
+        const options = {
+            metadata: {
+                'x-docker-expose-session-uuid': this.id,
+                'x-docker-expose-session-name': this.options.name,
+                'x-docker-expose-session-sharedkey': this.options.sharedKey,
+                'x-docker-expose-session-grpc-method': this.#server.methods
+            },
+            signal: signal
+        };
+        const stream = connection.bidi(`${CONTROL}/Session`, B.BytesMessage, B.BytesMessage, options);
+        this.#stream = stream;
+        const socket = new node_stream_1.Duplex({
+            write: (chunk, _encoding, callback) => {
+                stream.send({ data: chunk }).then(() => callback(), (error) => callback(error));
+            },
+            final: (callback) => {
+                stream.end();
+                callback();
+            },
+            read: () => undefined
+        });
+        (async () => {
+            for await (const message of stream.responses)
+                if (message.data?.length)
+                    socket.push(Buffer.from(message.data));
+            socket.push(null);
+        })().catch(() => {
+            socket.push(null);
+        });
+        this.#server.serve(socket);
+    }
+    /**
+     * Close the session
+     */
+    close() {
+        this.#stream?.end();
+        this.#stream?.cancel();
+    }
+}
+exports.BuildKitSession = BuildKitSession;
+/**
+ * The lines of the progress of a build, as "docker build --progress=plain" shows them: the steps are numbered in their
+ * order ("#3 [2/2] RUN npm ci"), with their output, and whether they are cached, done ("#3 DONE 1.2s") or failed
+ */
+class BuildKitProgress {
+    #steps = new Map();
+    #pending = new Map();
+    step(digest) {
+        let step = this.#steps.get(digest);
+        if (!step)
+            this.#steps.set(digest, step = { number: this.#steps.size + 1, started: false, ended: false });
+        return step;
+    }
+    /**
+     * The lines of a status of the build
+     * @param status
+     */
+    lines(status) {
+        const lines = [];
+        for (const vertex of status.vertexes ?? []) {
+            const step = this.step(vertex.digest);
+            const started = (0, ContainerdClient_js_1.timestampDate)(vertex.started);
+            const completed = (0, ContainerdClient_js_1.timestampDate)(vertex.completed);
+            if (!step.started && (started || vertex.cached)) {
+                step.started = true;
+                lines.push(`#${step.number} ${vertex.name}`);
+            }
+            if (step.ended)
+                continue;
+            if (vertex.cached) {
+                step.ended = true;
+                lines.push(`#${step.number} CACHED`);
+            }
+            else if (vertex.error) {
+                step.ended = true;
+                lines.push(...this.flush(vertex.digest), `#${step.number} ERROR: ${vertex.error}`);
+            }
+            else if (completed) {
+                step.ended = true;
+                const seconds = started ? ` ${((completed.getTime() - started.getTime()) / 1000).toFixed(1)}s` : '';
+                lines.push(...this.flush(vertex.digest), `#${step.number} DONE${seconds}`);
+            }
+        }
+        for (const log of status.logs ?? []) {
+            const output = (this.#pending.get(log.vertex) ?? '') + Buffer.from(log.msg ?? []).toString();
+            const parts = output.split(/\r?\n/);
+            this.#pending.set(log.vertex, parts.pop());
+            const number = this.step(log.vertex).number;
+            parts.forEach((part) => lines.push(`#${number} ${part}`));
+        }
+        for (const warning of status.warnings ?? [])
+            lines.push(`#${this.step(warning.vertex).number} WARN: ${Buffer.from(warning.short ?? []).toString()}`);
+        return lines;
+    }
+    flush(digest) {
+        const pending = this.#pending.get(digest);
+        this.#pending.delete(digest);
+        return pending ? [`#${this.step(digest).number} ${pending}`] : [];
+    }
+}
+exports.BuildKitProgress = BuildKitProgress;
+/**
+ * Build an image with BuildKit (its Dockerfile frontend): its context and its Dockerfile sent by the session, the image
+ * exported to the client as an archive (the oci exporter); the daemon is local or remote, of any worker
+ * @param connection
+ * @param options
+ */
+async function buildKitBuild(connection, options) {
+    const context = node_path_1.default.resolve(options.context);
+    const dockerfile = node_path_1.default.resolve(context, options.dockerfile ?? 'Dockerfile');
+    const session = new BuildKitSession({
+        name: options.name,
+        sharedKey: context,
+        directories: { context: context, dockerfile: node_path_1.default.dirname(dockerfile) },
+        credentials: options.credentials,
+        output: () => (0, node_fs_1.createWriteStream)(options.output)
+    });
+    const abort = new AbortController();
+    const signal = options.signal ? AbortSignal.any([options.signal, abort.signal]) : abort.signal;
+    session.start(connection, signal);
+    const ref = (0, node_crypto_1.randomBytes)(16).toString('hex');
+    const progress = new BuildKitProgress();
+    const status = (async () => {
+        if (!options.progress)
+            return;
+        for await (const response of connection.serverStream(`${CONTROL}/Status`, B.StatusRequest, { ref: ref }, B.StatusResponse, { signal: signal })) {
+            for (const line of progress.lines(response))
+                options.progress(line);
+        }
+    })().catch(() => undefined);
+    const frontendAttrs = { filename: node_path_1.default.basename(dockerfile) };
+    if (options.target)
+        frontendAttrs.target = options.target;
+    for (const [key, value] of Object.entries(options.buildArgs ?? {}))
+        frontendAttrs[`build-arg:${key}`] = value;
+    for (const [key, value] of Object.entries(options.labels ?? {}))
+        frontendAttrs[`label:${key}`] = value;
+    if (options.platforms?.length)
+        frontendAttrs.platform = options.platforms.map((platform) => (0, Platforms_js_1.formatPlatform)((0, Platforms_js_1.normalizePlatform)(platform))).join(',');
+    if (options.noCache)
+        frontendAttrs['no-cache'] = '';
+    if (options.pull)
+        frontendAttrs['image-resolve-mode'] = 'pull';
+    try {
+        const response = await connection.unary(`${CONTROL}/Solve`, B.SolveRequest, {
+            ref: ref,
+            session: session.id,
+            frontend: 'dockerfile.v0',
+            frontendAttrs: frontendAttrs,
+            exporter: 'oci',
+            exporterAttrs: { name: options.name }
+        }, B.SolveResponse, { signal: signal });
+        await status;
+        return response.exporterResponse ?? {};
+    }
+    finally {
+        abort.abort();
+        session.close();
+    }
+}
+/**
+ * The version of BuildKit
+ * @param connection
+ */
+async function buildKitVersion(connection) {
+    const response = await connection.unary(`${CONTROL}/Info`, {}, {}, B.InfoResponse);
+    return { package: response.buildkitVersion?.package ?? '', version: response.buildkitVersion?.version ?? '', revision: response.buildkitVersion?.revision ?? '' };
+}
+/**
+ * The platforms the workers of BuildKit build
+ * @param connection
+ */
+async function buildKitPlatforms(connection) {
+    const response = await connection.unary(`${CONTROL}/ListWorkers`, B.ListWorkersRequest, {}, B.ListWorkersResponse);
+    const platforms = [];
+    for (const record of response.record ?? []) {
+        for (const platform of record.platforms ?? []) {
+            const normalized = (0, Platforms_js_1.normalizePlatform)({ os: platform.os, architecture: platform.architecture, variant: platform.variant || undefined });
+            if (!platforms.some((known) => (0, Platforms_js_1.formatPlatform)(known) === (0, Platforms_js_1.formatPlatform)(normalized)))
+                platforms.push(normalized);
+        }
+    }
+    return platforms;
+}

package/dist/cjs/client/BuildKitMessages.d.ts

@@ -0,0 +1,35 @@
+import { type Schema } from '@lakutata/grpc';
+export declare const BytesMessage: Schema;
+export declare const ExporterMessage: Schema;
+export declare const SolveRequest: Schema;
+export declare const SolveResponse: Schema;
+export declare const StatusRequest: Schema;
+export declare const Vertex: Schema;
+export declare const VertexStatus: Schema;
+export declare const VertexLog: Schema;
+export declare const VertexWarning: Schema;
+export declare const StatusResponse: Schema;
+export declare const BuildkitVersion: Schema;
+export declare const InfoResponse: Schema;
+export declare const BuildKitPlatform: Schema;
+export declare const WorkerRecord: Schema;
+export declare const ListWorkersRequest: Schema;
+export declare const ListWorkersResponse: Schema;
+/**
+ * The types of the packets of fsutil
+ */
+export declare enum PacketType {
+    STAT = 0,
+    REQ = 1,
+    DATA = 2,
+    FIN = 3,
+    ERR = 4
+}
+export declare const Stat: Schema;
+export declare const Packet: Schema;
+export declare const CredentialsRequest: Schema;
+export declare const CredentialsResponse: Schema;
+export declare const FetchTokenRequest: Schema;
+export declare const FetchTokenResponse: Schema;
+export declare const HealthCheckRequest: Schema;
+export declare const HealthCheckResponse: Schema;