@shardflux/sdk 0.6.1 → 0.7.0

package/dist/template-file.js

@@ -0,0 +1,326 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { tarEnd, tarHeader, tarPadding } from "./tar.js";
+let loaded;
+/** The Node modules this file needs, loaded once (throws outside Node). */
+export function nodeModules() {
+    if (typeof process === 'undefined' || typeof process.versions?.node !== 'string') {
+        return Promise.reject(new TemplateFileError('Building from local files needs Node.js (it reads the disk); send a recipe with uploads instead.'));
+    }
+    const modules = (loaded ??= Promise.all([
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:fs'),
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:fs/promises'),
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:path'),
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:os'),
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto'),
+        import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:stream'),
+    ]).then(([fs, fsp, path, os, crypto, stream]) => ({ fs, fsp, path, os, crypto, stream: { Readable: stream.Readable } })));
+    return modules;
+}
+/** A template file that cannot be read, parsed or packed (before any request is made). */
+export class TemplateFileError extends Error {
+    /** The template file or local path concerned, when there is one. */
+    path;
+    constructor(message, path) {
+        super(message);
+        this.name = 'TemplateFileError';
+        this.path = path;
+    }
+}
+/** Largest single upload (contracts §24.2: 5 GiB, one presigned PUT). */
+export const UPLOAD_BYTES_MAX = 5_368_709_120;
+/** Most entries a folder tar may have (the host's extraction limit, §24.2). */
+export const TAR_ENTRIES_MAX = 200_000;
+export const RECIPE_V2_SCHEMA = 'shardflux.template-recipe.v2';
+async function defaultYamlParser() {
+    const spec = 'yaml';
+    try {
+        const mod = (await import(__rewriteRelativeImportExtension(/* webpackIgnore: true */ /* @vite-ignore */ spec)));
+        const parse = mod.parse ?? mod.default?.parse;
+        return typeof parse === 'function' ? parse : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * The document of a template file: `.json` is JSON, anything else YAML (JSON is valid YAML too). YAML needs the
+ * optional `yaml` package (`npm install yaml`) or `parseYaml`; without either, a YAML file that is not valid JSON fails.
+ */
+export async function parseTemplateText(text, fileName, parseYaml) {
+    let doc;
+    if (/\.json$/i.test(fileName)) {
+        try {
+            doc = JSON.parse(text);
+        }
+        catch (err) {
+            throw new TemplateFileError(`${fileName}: not valid JSON (${err instanceof Error ? err.message : String(err)})`, fileName);
+        }
+    }
+    else {
+        const parse = parseYaml ?? (await defaultYamlParser());
+        if (parse) {
+            try {
+                doc = parse(text);
+            }
+            catch (err) {
+                throw new TemplateFileError(`${fileName}: not valid YAML (${err instanceof Error ? (err.message.split('\n')[0] ?? err.message) : String(err)})`, fileName);
+            }
+        }
+        else {
+            try {
+                doc = JSON.parse(text);
+            }
+            catch {
+                throw new TemplateFileError(`${fileName}: reading YAML needs the "yaml" package (npm install yaml) or the parseYaml option`, fileName);
+            }
+        }
+    }
+    return normalizeTemplateDocument(doc, fileName);
+}
+/** The recipe v2 document: a mapping whose `schema` is recipe v2 (filled in when absent). */
+export function normalizeTemplateDocument(doc, where = 'the template') {
+    if (typeof doc !== 'object' || doc === null || Array.isArray(doc))
+        throw new TemplateFileError(`${where}: expected a mapping (the recipe v2 document), got ${doc === null ? 'null' : Array.isArray(doc) ? 'a list' : typeof doc}`, where);
+    const out = { ...doc };
+    if (out.schema === undefined)
+        out.schema = RECIPE_V2_SCHEMA;
+    else if (out.schema !== RECIPE_V2_SCHEMA)
+        throw new TemplateFileError(`${where}: schema must be ${RECIPE_V2_SCHEMA} (got ${JSON.stringify(out.schema)}); Dockerfile recipes (v1) are built with templates.builds.create`, where);
+    return out;
+}
+/** Reads and parses a template file (template.yaml or .json). */
+export async function readTemplateFile(file, opts = {}) {
+    const { fsp } = await nodeModules();
+    let text;
+    try {
+        text = await fsp.readFile(file, 'utf8');
+    }
+    catch (err) {
+        throw new TemplateFileError(`cannot read ${file} (${err.code ?? String(err)})`, file);
+    }
+    return parseTemplateText(text, file, opts.parseYaml);
+}
+/** SHA-256 (lower-case hex) and size of a local file, streamed. */
+export async function hashFile(file) {
+    const { fs, crypto } = await nodeModules();
+    const hash = crypto.createHash('sha256');
+    let size = 0;
+    for await (const chunk of fs.createReadStream(file)) {
+        const b = chunk;
+        size += b.length;
+        hash.update(b);
+    }
+    return { sha256: hash.digest('hex'), size };
+}
+const fatalUtf8 = new TextDecoder('utf-8', { fatal: true });
+/** Every directory, file and symlink below `root` (not following symlinks), sorted by UTF-8 path bytes. */
+async function walk(root) {
+    const { fsp, path } = await nodeModules();
+    const out = [];
+    const visit = async (dirAbs, prefix) => {
+        const names = await fsp.readdir(dirAbs, { encoding: 'buffer' });
+        for (const raw of names) {
+            let name;
+            try {
+                name = fatalUtf8.decode(raw);
+            }
+            catch {
+                throw new TemplateFileError(`${path.join(root, prefix)}: a name that is not UTF-8 cannot be uploaded (${raw.toString('hex')})`, dirAbs);
+            }
+            const rel = prefix ? `${prefix}/${name}` : name;
+            const abs = path.join(dirAbs, name);
+            const st = await fsp.lstat(abs);
+            let entry;
+            if (st.isDirectory())
+                entry = { path: rel, type: 'dir', mode: st.mode & 0o777, size: 0 };
+            else if (st.isFile())
+                entry = { path: rel, type: 'file', mode: st.mode & 0o777, size: st.size };
+            else if (st.isSymbolicLink()) {
+                const target = await fsp.readlink(abs);
+                checkSymlink(rel, target, abs);
+                entry = { path: rel, type: 'symlink', mode: 0o777, size: 0, linkname: target };
+            }
+            else {
+                const kind = st.isFIFO() ? 'a FIFO' : st.isSocket() ? 'a socket' : st.isCharacterDevice() || st.isBlockDevice() ? 'a device' : 'an unsupported file type';
+                throw new TemplateFileError(`${abs}: ${kind} cannot be uploaded (folders may hold files, directories and symlinks)`, abs);
+            }
+            out.push({ rel, key: Buffer.from(rel, 'utf8'), abs, entry });
+            if (out.length > TAR_ENTRIES_MAX)
+                throw new TemplateFileError(`${root}: more than ${TAR_ENTRIES_MAX} entries (the build host's limit for one folder)`, root);
+            if (entry.type === 'dir')
+                await visit(abs, rel);
+        }
+    };
+    await visit(root, '');
+    out.sort((a, b) => Buffer.compare(a.key, b.key));
+    return out;
+}
+/** The host refuses absolute symlink targets and targets that leave the extracted tree (§24.2); refuse them here. */
+function checkSymlink(rel, target, abs) {
+    if (target.startsWith('/'))
+        throw new TemplateFileError(`${abs}: symlink to an absolute path (${target}); the build host refuses it (use a relative target inside the folder)`, abs);
+    const parts = rel.split('/').slice(0, -1);
+    for (const p of target.split('/')) {
+        if (p === '' || p === '.')
+            continue;
+        if (p === '..') {
+            if (parts.length === 0)
+                throw new TemplateFileError(`${abs}: symlink leaves the folder (${target}); the build host refuses it`, abs);
+            parts.pop();
+        }
+        else
+            parts.push(p);
+    }
+}
+/**
+ * Packs the folder `root` into `dest` as the reproducible tar (tar.ts; byte-identical to the Python SDK's): members in
+ * UTF-8 path order, mtime 0, uid/gid 0 without names, permission bits kept (symlinks 0777). Returns its sha256, size and
+ * member count.
+ */
+export async function packDirectory(root, dest) {
+    const { fs, fsp, crypto } = await nodeModules();
+    const entries = await walk(root);
+    const hash = crypto.createHash('sha256');
+    const handle = await fsp.open(dest, 'w', 0o600);
+    let offset = 0;
+    const write = async (b) => {
+        if (b.length === 0)
+            return;
+        hash.update(b);
+        let done = 0;
+        while (done < b.length) {
+            const { bytesWritten } = await handle.write(b, done, b.length - done);
+            done += bytesWritten;
+        }
+        offset += b.length;
+    };
+    try {
+        for (const w of entries) {
+            await write(tarHeader(w.entry));
+            if (w.entry.type !== 'file')
+                continue;
+            let read = 0;
+            for await (const chunk of fs.createReadStream(w.abs)) {
+                const b = chunk;
+                read += b.length;
+                if (read > w.entry.size)
+                    break;
+                await write(b);
+            }
+            if (read !== w.entry.size)
+                throw new TemplateFileError(`${w.abs} changed while it was packed (${w.entry.size} bytes expected, ${read} read)`, w.abs);
+            await write(tarPadding(w.entry.size));
+        }
+        await write(tarEnd(offset));
+    }
+    finally {
+        await handle.close();
+    }
+    return { sha256: hash.digest('hex'), size: offset, entries: entries.length };
+}
+const COMPRESSED = [
+    { name: 'gzip', magic: [0x1f, 0x8b] },
+    { name: 'xz', magic: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00] },
+    { name: 'zstd', magic: [0x28, 0xb5, 0x2f, 0xfd] },
+];
+/** A prepared archive must be an uncompressed tar: the build host refuses gzip, xz and zstd (§24.2). */
+async function checkUncompressed(file) {
+    const { fsp } = await nodeModules();
+    const handle = await fsp.open(file, 'r');
+    try {
+        const head = Buffer.alloc(6);
+        const { bytesRead } = await handle.read(head, 0, 6, 0);
+        for (const c of COMPRESSED) {
+            if (bytesRead >= c.magic.length && c.magic.every((b, i) => head[i] === b))
+                throw new TemplateFileError(`${file}: a ${c.name}-compressed archive; the build host takes uncompressed tar only (or point "from" at the folder)`, file);
+        }
+    }
+    finally {
+        await handle.close();
+    }
+}
+/** Refuses `target` when it resolves, after symlinks, outside `root` (an agent-facing server confines local reads). */
+export async function assertInside(root, target, what) {
+    const { fsp, path } = await nodeModules();
+    let real;
+    let base;
+    try {
+        [real, base] = await Promise.all([fsp.realpath(target), fsp.realpath(root)]);
+    }
+    catch (err) {
+        throw new TemplateFileError(`${what}: cannot read ${target} (${err.code ?? String(err)})`, target);
+    }
+    const rel = path.relative(base, real);
+    if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel))
+        throw new TemplateFileError(`${what} resolves outside ${base}; only paths inside it may be read`, target);
+}
+/**
+ * Resolves `from` (relative to `baseDir`), checks it against `root` when given (after symlinks), and packs or hashes
+ * it. `kind` omitted: a folder is `tar`, a file is `file`. A folder with kind `file` is refused; a file with kind `tar`
+ * is a prepared (uncompressed) archive.
+ */
+export async function resolveLocalSource(from, kind, opts) {
+    const { fsp, path } = await nodeModules();
+    const resolved = path.resolve(opts.baseDir, from);
+    let st;
+    try {
+        st = await fsp.stat(resolved);
+    }
+    catch (err) {
+        throw new TemplateFileError(`from: ${from}: cannot read ${resolved} (${err.code ?? String(err)})`, resolved);
+    }
+    if (opts.root !== undefined)
+        await assertInside(opts.root, resolved, `from: ${from}`);
+    const effective = kind ?? (st.isDirectory() ? 'tar' : 'file');
+    if (st.isDirectory()) {
+        if (effective !== 'tar')
+            throw new TemplateFileError(`from: ${from} is a folder; a folder uploads as kind tar`, resolved);
+        const dest = path.join(await opts.tmpDir(), `${path.basename(resolved) || 'folder'}-${Math.random().toString(16).slice(2, 10)}.tar`);
+        const packed = await packDirectory(resolved, dest);
+        if (packed.size > UPLOAD_BYTES_MAX)
+            throw new TemplateFileError(`from: ${from} packs to ${packed.size} bytes; one upload is at most ${UPLOAD_BYTES_MAX} bytes (5 GiB)`, resolved);
+        return { from, path: resolved, kind: 'tar', uploadPath: dest, ...packed };
+    }
+    if (!st.isFile())
+        throw new TemplateFileError(`from: ${from} is neither a file nor a folder`, resolved);
+    if (st.size > UPLOAD_BYTES_MAX)
+        throw new TemplateFileError(`from: ${from} is ${st.size} bytes; one upload is at most ${UPLOAD_BYTES_MAX} bytes (5 GiB)`, resolved);
+    if (effective === 'tar')
+        await checkUncompressed(resolved);
+    const { sha256, size } = await hashFile(resolved);
+    return { from, path: resolved, kind: effective, uploadPath: resolved, sha256, size, entries: null };
+}
+/** A fetch body that streams a local file (replayable: each call opens it again). */
+export async function fileBody(file) {
+    const { fs, stream } = await nodeModules();
+    return stream.Readable.toWeb(fs.createReadStream(file));
+}
+/** A private temporary directory, created on first use; `cleanup()` removes it. */
+export function tempDir() {
+    let dir;
+    return {
+        get: async () => {
+            if (!dir) {
+                const { fsp, os, path } = await nodeModules();
+                dir = fsp.mkdtemp(path.join(os.tmpdir(), 'shardflux-template-'));
+            }
+            return dir;
+        },
+        cleanup: async () => {
+            if (!dir)
+                return;
+            const d = await dir.catch(() => null);
+            if (d === null)
+                return;
+            const { fsp } = await nodeModules();
+            await fsp.rm(d, { recursive: true, force: true });
+        },
+    };
+}