@shardflux/sdk 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +109 -1
- package/README.md +202 -9
- package/dist/capture-adapters.d.ts +178 -0
- package/dist/capture-adapters.js +432 -0
- package/dist/capture-serialize.d.ts +97 -0
- package/dist/capture-serialize.js +481 -0
- package/dist/capture-text.d.ts +14 -0
- package/dist/capture-text.js +13 -0
- package/dist/capture.d.ts +288 -0
- package/dist/capture.js +1326 -0
- package/dist/cell.d.ts +7 -0
- package/dist/cell.js +13 -0
- package/dist/client.d.ts +16 -0
- package/dist/client.js +21 -4
- package/dist/egress.d.ts +18 -0
- package/dist/errors.d.ts +8 -1
- package/dist/generated/app-api.d.ts +215 -0
- package/dist/http.d.ts +1 -1
- package/dist/http.js +1 -1
- package/dist/index.d.ts +11 -4
- package/dist/index.js +4 -1
- package/dist/lifecycle.d.ts +3 -1
- package/dist/lifecycle.js +5 -1
- package/dist/progress.d.ts +3 -1
- package/dist/tar.d.ts +40 -0
- package/dist/tar.js +150 -0
- package/dist/template-file.d.ts +92 -0
- package/dist/template-file.js +326 -0
- package/dist/templates.d.ts +318 -9
- package/dist/templates.js +432 -13
- package/dist/tools.d.ts +49 -33
- package/dist/tools.js +25 -4
- package/dist/workspace.d.ts +25 -2
- package/dist/workspace.js +49 -2
- package/package.json +25 -2
package/dist/tar.js
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small tar writer for build uploads of folders (contracts §24.2: uncompressed ustar/pax, extracted by the host's
|
|
3
|
+
* static tool). Pure: no Node imports, so the browser bundle can carry it.
|
|
4
|
+
*
|
|
5
|
+
* The bytes equal CPython's `tarfile.open(mode="w", format=tarfile.PAX_FORMAT)` with every member added by hand (the
|
|
6
|
+
* Python SDK's writer), so both SDKs upload the same bytes for the same folder and a template built from either has the
|
|
7
|
+
* same recipe_sha256:
|
|
8
|
+
* - a pax `x` header (`././@PaxHeader`) precedes a member whose name or link target is non-ASCII or longer than 100
|
|
9
|
+
* characters (`path`, `linkpath` records), or whose size does not fit the 11 octal digits (`size`);
|
|
10
|
+
* - the ustar header then holds the name encoded as ASCII with `?` for each non-ASCII code point, truncated to 100 bytes;
|
|
11
|
+
* - the archive ends with two zero blocks and is padded to a multiple of 10240 bytes (tarfile's RECORDSIZE).
|
|
12
|
+
*/
|
|
13
|
+
export const TAR_BLOCK = 512;
|
|
14
|
+
export const TAR_RECORD_SIZE = 20 * TAR_BLOCK;
|
|
15
|
+
const TYPEFLAG = { file: 0x30, dir: 0x35, symlink: 0x32 };
|
|
16
|
+
const utf8 = new TextEncoder();
|
|
17
|
+
function isAscii(s) {
|
|
18
|
+
for (let i = 0; i < s.length; i += 1)
|
|
19
|
+
if (s.charCodeAt(i) > 0x7f)
|
|
20
|
+
return false;
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
/** Python's `s.encode("ascii", "replace")`: one `?` per non-ASCII code point (not per UTF-16 unit). */
|
|
24
|
+
function asciiReplace(s) {
|
|
25
|
+
const out = [];
|
|
26
|
+
for (const ch of s)
|
|
27
|
+
out.push(ch.codePointAt(0) > 0x7f ? 0x3f : ch.charCodeAt(0));
|
|
28
|
+
return Uint8Array.from(out);
|
|
29
|
+
}
|
|
30
|
+
/** tarfile's stn(): the bytes truncated to `length` and NUL-padded. */
|
|
31
|
+
function putString(buf, offset, length, bytes) {
|
|
32
|
+
buf.set(bytes.subarray(0, length), offset);
|
|
33
|
+
}
|
|
34
|
+
/** tarfile's itn() for the POSIX range: `%0*o` with digits-1 digits and a NUL. */
|
|
35
|
+
function putOctal(buf, offset, digits, value) {
|
|
36
|
+
const text = `${value.toString(8).padStart(digits - 1, '0')}\0`;
|
|
37
|
+
buf.set(utf8.encode(text), offset);
|
|
38
|
+
}
|
|
39
|
+
/** Python's len(str): code points. */
|
|
40
|
+
function codePoints(s) {
|
|
41
|
+
return Array.from(s).length;
|
|
42
|
+
}
|
|
43
|
+
/** tarfile's _create_header(info, USTAR_FORMAT, "ascii", "replace"). */
|
|
44
|
+
function ustarHeader(info) {
|
|
45
|
+
const h = new Uint8Array(TAR_BLOCK);
|
|
46
|
+
putString(h, 0, 100, asciiReplace(info.name));
|
|
47
|
+
putOctal(h, 100, 8, info.mode & 0o7777);
|
|
48
|
+
putOctal(h, 108, 8, info.uid);
|
|
49
|
+
putOctal(h, 116, 8, info.gid);
|
|
50
|
+
putOctal(h, 124, 12, info.size);
|
|
51
|
+
putOctal(h, 136, 12, info.mtime);
|
|
52
|
+
h.fill(0x20, 148, 156);
|
|
53
|
+
h[156] = info.type;
|
|
54
|
+
putString(h, 157, 100, asciiReplace(info.linkname));
|
|
55
|
+
h.set(utf8.encode('ustar\x0000'), 257);
|
|
56
|
+
putString(h, 265, 32, asciiReplace(info.uname));
|
|
57
|
+
putString(h, 297, 32, asciiReplace(info.gname));
|
|
58
|
+
// devmajor, devminor and prefix stay NUL (tarfile writes empty strings for non-device members).
|
|
59
|
+
let sum = 0;
|
|
60
|
+
for (const b of h)
|
|
61
|
+
sum += b;
|
|
62
|
+
h.set(utf8.encode(`${sum.toString(8).padStart(6, '0')}\0`), 148);
|
|
63
|
+
return h;
|
|
64
|
+
}
|
|
65
|
+
/** Zero padding after `size` bytes of member data. */
|
|
66
|
+
export function tarPadding(size) {
|
|
67
|
+
const rest = size % TAR_BLOCK;
|
|
68
|
+
return new Uint8Array(rest === 0 ? 0 : TAR_BLOCK - rest);
|
|
69
|
+
}
|
|
70
|
+
/** tarfile's pax record: "%d %s=%s\n" where the length counts itself. */
|
|
71
|
+
function paxRecord(keyword, value) {
|
|
72
|
+
const kv = utf8.encode(`${keyword}=${value}\n`);
|
|
73
|
+
const l = kv.length + 1; // + ' '
|
|
74
|
+
let p = 0;
|
|
75
|
+
for (;;) {
|
|
76
|
+
const n = l + String(p).length;
|
|
77
|
+
if (n === p)
|
|
78
|
+
break;
|
|
79
|
+
p = n;
|
|
80
|
+
}
|
|
81
|
+
const head = utf8.encode(`${p} `);
|
|
82
|
+
const out = new Uint8Array(head.length + kv.length);
|
|
83
|
+
out.set(head, 0);
|
|
84
|
+
out.set(kv, head.length);
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The header block(s) of one member: an optional pax extended header with its records, then the ustar header.
|
|
89
|
+
* Member data (a file's bytes) follows, then tarPadding(size).
|
|
90
|
+
*/
|
|
91
|
+
export function tarHeader(entry) {
|
|
92
|
+
const name = entry.type === 'dir' && !entry.path.endsWith('/') ? `${entry.path}/` : entry.path;
|
|
93
|
+
const info = {
|
|
94
|
+
name,
|
|
95
|
+
mode: entry.mode,
|
|
96
|
+
uid: entry.uid ?? 0,
|
|
97
|
+
gid: entry.gid ?? 0,
|
|
98
|
+
size: entry.type === 'file' ? entry.size : 0,
|
|
99
|
+
mtime: entry.mtime ?? 0,
|
|
100
|
+
type: TYPEFLAG[entry.type],
|
|
101
|
+
linkname: entry.linkname ?? '',
|
|
102
|
+
uname: entry.uname ?? '',
|
|
103
|
+
gname: entry.gname ?? '',
|
|
104
|
+
};
|
|
105
|
+
const records = [];
|
|
106
|
+
for (const [field, keyword, length] of [
|
|
107
|
+
['name', 'path', 100],
|
|
108
|
+
['linkname', 'linkpath', 100],
|
|
109
|
+
['uname', 'uname', 32],
|
|
110
|
+
['gname', 'gname', 32],
|
|
111
|
+
]) {
|
|
112
|
+
const value = info[field];
|
|
113
|
+
if (!isAscii(value) || codePoints(value) > length)
|
|
114
|
+
records.push(paxRecord(keyword, value));
|
|
115
|
+
}
|
|
116
|
+
for (const [field, digits] of [
|
|
117
|
+
['uid', 8],
|
|
118
|
+
['gid', 8],
|
|
119
|
+
['size', 12],
|
|
120
|
+
['mtime', 12],
|
|
121
|
+
]) {
|
|
122
|
+
const value = info[field];
|
|
123
|
+
if (!(value >= 0 && value < 8 ** (digits - 1))) {
|
|
124
|
+
records.push(paxRecord(field, String(value)));
|
|
125
|
+
info[field] = 0;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const header = ustarHeader(info);
|
|
129
|
+
if (records.length === 0)
|
|
130
|
+
return header;
|
|
131
|
+
const total = records.reduce((n, r) => n + r.length, 0);
|
|
132
|
+
const payload = new Uint8Array(total + tarPadding(total).length);
|
|
133
|
+
let at = 0;
|
|
134
|
+
for (const r of records) {
|
|
135
|
+
payload.set(r, at);
|
|
136
|
+
at += r.length;
|
|
137
|
+
}
|
|
138
|
+
const pax = ustarHeader({ name: '././@PaxHeader', mode: 0, uid: 0, gid: 0, size: total, mtime: 0, type: 0x78, linkname: '', uname: '', gname: '' });
|
|
139
|
+
const out = new Uint8Array(pax.length + payload.length + header.length);
|
|
140
|
+
out.set(pax, 0);
|
|
141
|
+
out.set(payload, pax.length);
|
|
142
|
+
out.set(header, pax.length + payload.length);
|
|
143
|
+
return out;
|
|
144
|
+
}
|
|
145
|
+
/** The end of an archive written so far (`offset` bytes): two zero blocks, then zeros up to a multiple of 10240. */
|
|
146
|
+
export function tarEnd(offset) {
|
|
147
|
+
const end = offset + 2 * TAR_BLOCK;
|
|
148
|
+
const rest = end % TAR_RECORD_SIZE;
|
|
149
|
+
return new Uint8Array(2 * TAR_BLOCK + (rest === 0 ? 0 : TAR_RECORD_SIZE - rest));
|
|
150
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
type Fs = typeof import('node:fs');
|
|
2
|
+
type FsPromises = typeof import('node:fs/promises');
|
|
3
|
+
type Path = typeof import('node:path');
|
|
4
|
+
type Os = typeof import('node:os');
|
|
5
|
+
type Crypto = typeof import('node:crypto');
|
|
6
|
+
type Stream = {
|
|
7
|
+
Readable: typeof import('node:stream').Readable;
|
|
8
|
+
};
|
|
9
|
+
interface NodeModules {
|
|
10
|
+
fs: Fs;
|
|
11
|
+
fsp: FsPromises;
|
|
12
|
+
path: Path;
|
|
13
|
+
os: Os;
|
|
14
|
+
crypto: Crypto;
|
|
15
|
+
stream: Stream;
|
|
16
|
+
}
|
|
17
|
+
/** The Node modules this file needs, loaded once (throws outside Node). */
|
|
18
|
+
export declare function nodeModules(): Promise<NodeModules>;
|
|
19
|
+
/** A template file that cannot be read, parsed or packed (before any request is made). */
|
|
20
|
+
export declare class TemplateFileError extends Error {
|
|
21
|
+
/** The template file or local path concerned, when there is one. */
|
|
22
|
+
readonly path: string | undefined;
|
|
23
|
+
constructor(message: string, path?: string);
|
|
24
|
+
}
|
|
25
|
+
/** Largest single upload (contracts §24.2: 5 GiB, one presigned PUT). */
|
|
26
|
+
export declare const UPLOAD_BYTES_MAX = 5368709120;
|
|
27
|
+
/** Most entries a folder tar may have (the host's extraction limit, §24.2). */
|
|
28
|
+
export declare const TAR_ENTRIES_MAX = 200000;
|
|
29
|
+
export declare const RECIPE_V2_SCHEMA = "shardflux.template-recipe.v2";
|
|
30
|
+
/** Parses YAML text: the optional `yaml` package's parse() unless the caller passes its own. */
|
|
31
|
+
export type YamlParser = (text: string) => unknown;
|
|
32
|
+
/**
|
|
33
|
+
* The document of a template file: `.json` is JSON, anything else YAML (JSON is valid YAML too). YAML needs the
|
|
34
|
+
* optional `yaml` package (`npm install yaml`) or `parseYaml`; without either, a YAML file that is not valid JSON fails.
|
|
35
|
+
*/
|
|
36
|
+
export declare function parseTemplateText(text: string, fileName: string, parseYaml?: YamlParser): Promise<Record<string, unknown>>;
|
|
37
|
+
/** The recipe v2 document: a mapping whose `schema` is recipe v2 (filled in when absent). */
|
|
38
|
+
export declare function normalizeTemplateDocument(doc: unknown, where?: string): Record<string, unknown>;
|
|
39
|
+
/** Reads and parses a template file (template.yaml or .json). */
|
|
40
|
+
export declare function readTemplateFile(file: string, opts?: {
|
|
41
|
+
parseYaml?: YamlParser;
|
|
42
|
+
}): Promise<Record<string, unknown>>;
|
|
43
|
+
export interface PackedFile {
|
|
44
|
+
sha256: string;
|
|
45
|
+
size: number;
|
|
46
|
+
/** Tar members (folders), null for a file uploaded as it is. */
|
|
47
|
+
entries: number | null;
|
|
48
|
+
}
|
|
49
|
+
/** SHA-256 (lower-case hex) and size of a local file, streamed. */
|
|
50
|
+
export declare function hashFile(file: string): Promise<{
|
|
51
|
+
sha256: string;
|
|
52
|
+
size: number;
|
|
53
|
+
}>;
|
|
54
|
+
/**
|
|
55
|
+
* Packs the folder `root` into `dest` as the reproducible tar (tar.ts; byte-identical to the Python SDK's): members in
|
|
56
|
+
* UTF-8 path order, mtime 0, uid/gid 0 without names, permission bits kept (symlinks 0777). Returns its sha256, size and
|
|
57
|
+
* member count.
|
|
58
|
+
*/
|
|
59
|
+
export declare function packDirectory(root: string, dest: string): Promise<PackedFile>;
|
|
60
|
+
/** Refuses `target` when it resolves, after symlinks, outside `root` (an agent-facing server confines local reads). */
|
|
61
|
+
export declare function assertInside(root: string, target: string, what: string): Promise<void>;
|
|
62
|
+
/** One `from` entry resolved on the disk: what gets uploaded and how. */
|
|
63
|
+
export interface LocalSource {
|
|
64
|
+
/** `from` as written. */
|
|
65
|
+
from: string;
|
|
66
|
+
/** The resolved local path. */
|
|
67
|
+
path: string;
|
|
68
|
+
kind: 'file' | 'tar';
|
|
69
|
+
/** The bytes to upload: the file itself, or the packed tar in a temporary directory. */
|
|
70
|
+
uploadPath: string;
|
|
71
|
+
sha256: string;
|
|
72
|
+
size: number;
|
|
73
|
+
entries: number | null;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Resolves `from` (relative to `baseDir`), checks it against `root` when given (after symlinks), and packs or hashes
|
|
77
|
+
* it. `kind` omitted: a folder is `tar`, a file is `file`. A folder with kind `file` is refused; a file with kind `tar`
|
|
78
|
+
* is a prepared (uncompressed) archive.
|
|
79
|
+
*/
|
|
80
|
+
export declare function resolveLocalSource(from: string, kind: 'file' | 'tar' | undefined, opts: {
|
|
81
|
+
baseDir: string;
|
|
82
|
+
root?: string | undefined;
|
|
83
|
+
tmpDir: () => Promise<string>;
|
|
84
|
+
}): Promise<LocalSource>;
|
|
85
|
+
/** A fetch body that streams a local file (replayable: each call opens it again). */
|
|
86
|
+
export declare function fileBody(file: string): Promise<ReadableStream<Uint8Array>>;
|
|
87
|
+
/** A private temporary directory, created on first use; `cleanup()` removes it. */
|
|
88
|
+
export declare function tempDir(): {
|
|
89
|
+
get: () => Promise<string>;
|
|
90
|
+
cleanup: () => Promise<void>;
|
|
91
|
+
};
|
|
92
|
+
export {};
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
2
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
3
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
4
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
5
|
+
});
|
|
6
|
+
}
|
|
7
|
+
return path;
|
|
8
|
+
};
|
|
9
|
+
import { tarEnd, tarHeader, tarPadding } from "./tar.js";
|
|
10
|
+
let loaded;
|
|
11
|
+
/** The Node modules this file needs, loaded once (throws outside Node). */
|
|
12
|
+
export function nodeModules() {
|
|
13
|
+
if (typeof process === 'undefined' || typeof process.versions?.node !== 'string') {
|
|
14
|
+
return Promise.reject(new TemplateFileError('Building from local files needs Node.js (it reads the disk); send a recipe with uploads instead.'));
|
|
15
|
+
}
|
|
16
|
+
const modules = (loaded ??= Promise.all([
|
|
17
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:fs'),
|
|
18
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:fs/promises'),
|
|
19
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:path'),
|
|
20
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:os'),
|
|
21
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:crypto'),
|
|
22
|
+
import(/* webpackIgnore: true */ /* @vite-ignore */ 'node:stream'),
|
|
23
|
+
]).then(([fs, fsp, path, os, crypto, stream]) => ({ fs, fsp, path, os, crypto, stream: { Readable: stream.Readable } })));
|
|
24
|
+
return modules;
|
|
25
|
+
}
|
|
26
|
+
/** A template file that cannot be read, parsed or packed (before any request is made). */
|
|
27
|
+
export class TemplateFileError extends Error {
|
|
28
|
+
/** The template file or local path concerned, when there is one. */
|
|
29
|
+
path;
|
|
30
|
+
constructor(message, path) {
|
|
31
|
+
super(message);
|
|
32
|
+
this.name = 'TemplateFileError';
|
|
33
|
+
this.path = path;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** Largest single upload (contracts §24.2: 5 GiB, one presigned PUT). */
|
|
37
|
+
export const UPLOAD_BYTES_MAX = 5_368_709_120;
|
|
38
|
+
/** Most entries a folder tar may have (the host's extraction limit, §24.2). */
|
|
39
|
+
export const TAR_ENTRIES_MAX = 200_000;
|
|
40
|
+
export const RECIPE_V2_SCHEMA = 'shardflux.template-recipe.v2';
|
|
41
|
+
async function defaultYamlParser() {
|
|
42
|
+
const spec = 'yaml';
|
|
43
|
+
try {
|
|
44
|
+
const mod = (await import(__rewriteRelativeImportExtension(/* webpackIgnore: true */ /* @vite-ignore */ spec)));
|
|
45
|
+
const parse = mod.parse ?? mod.default?.parse;
|
|
46
|
+
return typeof parse === 'function' ? parse : null;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The document of a template file: `.json` is JSON, anything else YAML (JSON is valid YAML too). YAML needs the
|
|
54
|
+
* optional `yaml` package (`npm install yaml`) or `parseYaml`; without either, a YAML file that is not valid JSON fails.
|
|
55
|
+
*/
|
|
56
|
+
export async function parseTemplateText(text, fileName, parseYaml) {
|
|
57
|
+
let doc;
|
|
58
|
+
if (/\.json$/i.test(fileName)) {
|
|
59
|
+
try {
|
|
60
|
+
doc = JSON.parse(text);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
throw new TemplateFileError(`${fileName}: not valid JSON (${err instanceof Error ? err.message : String(err)})`, fileName);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
const parse = parseYaml ?? (await defaultYamlParser());
|
|
68
|
+
if (parse) {
|
|
69
|
+
try {
|
|
70
|
+
doc = parse(text);
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
throw new TemplateFileError(`${fileName}: not valid YAML (${err instanceof Error ? (err.message.split('\n')[0] ?? err.message) : String(err)})`, fileName);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
try {
|
|
78
|
+
doc = JSON.parse(text);
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
throw new TemplateFileError(`${fileName}: reading YAML needs the "yaml" package (npm install yaml) or the parseYaml option`, fileName);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return normalizeTemplateDocument(doc, fileName);
|
|
86
|
+
}
|
|
87
|
+
/** The recipe v2 document: a mapping whose `schema` is recipe v2 (filled in when absent). */
|
|
88
|
+
export function normalizeTemplateDocument(doc, where = 'the template') {
|
|
89
|
+
if (typeof doc !== 'object' || doc === null || Array.isArray(doc))
|
|
90
|
+
throw new TemplateFileError(`${where}: expected a mapping (the recipe v2 document), got ${doc === null ? 'null' : Array.isArray(doc) ? 'a list' : typeof doc}`, where);
|
|
91
|
+
const out = { ...doc };
|
|
92
|
+
if (out.schema === undefined)
|
|
93
|
+
out.schema = RECIPE_V2_SCHEMA;
|
|
94
|
+
else if (out.schema !== RECIPE_V2_SCHEMA)
|
|
95
|
+
throw new TemplateFileError(`${where}: schema must be ${RECIPE_V2_SCHEMA} (got ${JSON.stringify(out.schema)}); Dockerfile recipes (v1) are built with templates.builds.create`, where);
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
/** Reads and parses a template file (template.yaml or .json). */
|
|
99
|
+
export async function readTemplateFile(file, opts = {}) {
|
|
100
|
+
const { fsp } = await nodeModules();
|
|
101
|
+
let text;
|
|
102
|
+
try {
|
|
103
|
+
text = await fsp.readFile(file, 'utf8');
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
throw new TemplateFileError(`cannot read ${file} (${err.code ?? String(err)})`, file);
|
|
107
|
+
}
|
|
108
|
+
return parseTemplateText(text, file, opts.parseYaml);
|
|
109
|
+
}
|
|
110
|
+
/** SHA-256 (lower-case hex) and size of a local file, streamed. */
|
|
111
|
+
export async function hashFile(file) {
|
|
112
|
+
const { fs, crypto } = await nodeModules();
|
|
113
|
+
const hash = crypto.createHash('sha256');
|
|
114
|
+
let size = 0;
|
|
115
|
+
for await (const chunk of fs.createReadStream(file)) {
|
|
116
|
+
const b = chunk;
|
|
117
|
+
size += b.length;
|
|
118
|
+
hash.update(b);
|
|
119
|
+
}
|
|
120
|
+
return { sha256: hash.digest('hex'), size };
|
|
121
|
+
}
|
|
122
|
+
const fatalUtf8 = new TextDecoder('utf-8', { fatal: true });
|
|
123
|
+
/** Every directory, file and symlink below `root` (not following symlinks), sorted by UTF-8 path bytes. */
|
|
124
|
+
async function walk(root) {
|
|
125
|
+
const { fsp, path } = await nodeModules();
|
|
126
|
+
const out = [];
|
|
127
|
+
const visit = async (dirAbs, prefix) => {
|
|
128
|
+
const names = await fsp.readdir(dirAbs, { encoding: 'buffer' });
|
|
129
|
+
for (const raw of names) {
|
|
130
|
+
let name;
|
|
131
|
+
try {
|
|
132
|
+
name = fatalUtf8.decode(raw);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
throw new TemplateFileError(`${path.join(root, prefix)}: a name that is not UTF-8 cannot be uploaded (${raw.toString('hex')})`, dirAbs);
|
|
136
|
+
}
|
|
137
|
+
const rel = prefix ? `${prefix}/${name}` : name;
|
|
138
|
+
const abs = path.join(dirAbs, name);
|
|
139
|
+
const st = await fsp.lstat(abs);
|
|
140
|
+
let entry;
|
|
141
|
+
if (st.isDirectory())
|
|
142
|
+
entry = { path: rel, type: 'dir', mode: st.mode & 0o777, size: 0 };
|
|
143
|
+
else if (st.isFile())
|
|
144
|
+
entry = { path: rel, type: 'file', mode: st.mode & 0o777, size: st.size };
|
|
145
|
+
else if (st.isSymbolicLink()) {
|
|
146
|
+
const target = await fsp.readlink(abs);
|
|
147
|
+
checkSymlink(rel, target, abs);
|
|
148
|
+
entry = { path: rel, type: 'symlink', mode: 0o777, size: 0, linkname: target };
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
const kind = st.isFIFO() ? 'a FIFO' : st.isSocket() ? 'a socket' : st.isCharacterDevice() || st.isBlockDevice() ? 'a device' : 'an unsupported file type';
|
|
152
|
+
throw new TemplateFileError(`${abs}: ${kind} cannot be uploaded (folders may hold files, directories and symlinks)`, abs);
|
|
153
|
+
}
|
|
154
|
+
out.push({ rel, key: Buffer.from(rel, 'utf8'), abs, entry });
|
|
155
|
+
if (out.length > TAR_ENTRIES_MAX)
|
|
156
|
+
throw new TemplateFileError(`${root}: more than ${TAR_ENTRIES_MAX} entries (the build host's limit for one folder)`, root);
|
|
157
|
+
if (entry.type === 'dir')
|
|
158
|
+
await visit(abs, rel);
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
await visit(root, '');
|
|
162
|
+
out.sort((a, b) => Buffer.compare(a.key, b.key));
|
|
163
|
+
return out;
|
|
164
|
+
}
|
|
165
|
+
/** The host refuses absolute symlink targets and targets that leave the extracted tree (§24.2); refuse them here. */
|
|
166
|
+
function checkSymlink(rel, target, abs) {
|
|
167
|
+
if (target.startsWith('/'))
|
|
168
|
+
throw new TemplateFileError(`${abs}: symlink to an absolute path (${target}); the build host refuses it (use a relative target inside the folder)`, abs);
|
|
169
|
+
const parts = rel.split('/').slice(0, -1);
|
|
170
|
+
for (const p of target.split('/')) {
|
|
171
|
+
if (p === '' || p === '.')
|
|
172
|
+
continue;
|
|
173
|
+
if (p === '..') {
|
|
174
|
+
if (parts.length === 0)
|
|
175
|
+
throw new TemplateFileError(`${abs}: symlink leaves the folder (${target}); the build host refuses it`, abs);
|
|
176
|
+
parts.pop();
|
|
177
|
+
}
|
|
178
|
+
else
|
|
179
|
+
parts.push(p);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Packs the folder `root` into `dest` as the reproducible tar (tar.ts; byte-identical to the Python SDK's): members in
|
|
184
|
+
* UTF-8 path order, mtime 0, uid/gid 0 without names, permission bits kept (symlinks 0777). Returns its sha256, size and
|
|
185
|
+
* member count.
|
|
186
|
+
*/
|
|
187
|
+
export async function packDirectory(root, dest) {
|
|
188
|
+
const { fs, fsp, crypto } = await nodeModules();
|
|
189
|
+
const entries = await walk(root);
|
|
190
|
+
const hash = crypto.createHash('sha256');
|
|
191
|
+
const handle = await fsp.open(dest, 'w', 0o600);
|
|
192
|
+
let offset = 0;
|
|
193
|
+
const write = async (b) => {
|
|
194
|
+
if (b.length === 0)
|
|
195
|
+
return;
|
|
196
|
+
hash.update(b);
|
|
197
|
+
let done = 0;
|
|
198
|
+
while (done < b.length) {
|
|
199
|
+
const { bytesWritten } = await handle.write(b, done, b.length - done);
|
|
200
|
+
done += bytesWritten;
|
|
201
|
+
}
|
|
202
|
+
offset += b.length;
|
|
203
|
+
};
|
|
204
|
+
try {
|
|
205
|
+
for (const w of entries) {
|
|
206
|
+
await write(tarHeader(w.entry));
|
|
207
|
+
if (w.entry.type !== 'file')
|
|
208
|
+
continue;
|
|
209
|
+
let read = 0;
|
|
210
|
+
for await (const chunk of fs.createReadStream(w.abs)) {
|
|
211
|
+
const b = chunk;
|
|
212
|
+
read += b.length;
|
|
213
|
+
if (read > w.entry.size)
|
|
214
|
+
break;
|
|
215
|
+
await write(b);
|
|
216
|
+
}
|
|
217
|
+
if (read !== w.entry.size)
|
|
218
|
+
throw new TemplateFileError(`${w.abs} changed while it was packed (${w.entry.size} bytes expected, ${read} read)`, w.abs);
|
|
219
|
+
await write(tarPadding(w.entry.size));
|
|
220
|
+
}
|
|
221
|
+
await write(tarEnd(offset));
|
|
222
|
+
}
|
|
223
|
+
finally {
|
|
224
|
+
await handle.close();
|
|
225
|
+
}
|
|
226
|
+
return { sha256: hash.digest('hex'), size: offset, entries: entries.length };
|
|
227
|
+
}
|
|
228
|
+
const COMPRESSED = [
|
|
229
|
+
{ name: 'gzip', magic: [0x1f, 0x8b] },
|
|
230
|
+
{ name: 'xz', magic: [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00] },
|
|
231
|
+
{ name: 'zstd', magic: [0x28, 0xb5, 0x2f, 0xfd] },
|
|
232
|
+
];
|
|
233
|
+
/** A prepared archive must be an uncompressed tar: the build host refuses gzip, xz and zstd (§24.2). */
|
|
234
|
+
async function checkUncompressed(file) {
|
|
235
|
+
const { fsp } = await nodeModules();
|
|
236
|
+
const handle = await fsp.open(file, 'r');
|
|
237
|
+
try {
|
|
238
|
+
const head = Buffer.alloc(6);
|
|
239
|
+
const { bytesRead } = await handle.read(head, 0, 6, 0);
|
|
240
|
+
for (const c of COMPRESSED) {
|
|
241
|
+
if (bytesRead >= c.magic.length && c.magic.every((b, i) => head[i] === b))
|
|
242
|
+
throw new TemplateFileError(`${file}: a ${c.name}-compressed archive; the build host takes uncompressed tar only (or point "from" at the folder)`, file);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
finally {
|
|
246
|
+
await handle.close();
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
/** Refuses `target` when it resolves, after symlinks, outside `root` (an agent-facing server confines local reads). */
|
|
250
|
+
export async function assertInside(root, target, what) {
|
|
251
|
+
const { fsp, path } = await nodeModules();
|
|
252
|
+
let real;
|
|
253
|
+
let base;
|
|
254
|
+
try {
|
|
255
|
+
[real, base] = await Promise.all([fsp.realpath(target), fsp.realpath(root)]);
|
|
256
|
+
}
|
|
257
|
+
catch (err) {
|
|
258
|
+
throw new TemplateFileError(`${what}: cannot read ${target} (${err.code ?? String(err)})`, target);
|
|
259
|
+
}
|
|
260
|
+
const rel = path.relative(base, real);
|
|
261
|
+
if (rel === '..' || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel))
|
|
262
|
+
throw new TemplateFileError(`${what} resolves outside ${base}; only paths inside it may be read`, target);
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Resolves `from` (relative to `baseDir`), checks it against `root` when given (after symlinks), and packs or hashes
|
|
266
|
+
* it. `kind` omitted: a folder is `tar`, a file is `file`. A folder with kind `file` is refused; a file with kind `tar`
|
|
267
|
+
* is a prepared (uncompressed) archive.
|
|
268
|
+
*/
|
|
269
|
+
export async function resolveLocalSource(from, kind, opts) {
|
|
270
|
+
const { fsp, path } = await nodeModules();
|
|
271
|
+
const resolved = path.resolve(opts.baseDir, from);
|
|
272
|
+
let st;
|
|
273
|
+
try {
|
|
274
|
+
st = await fsp.stat(resolved);
|
|
275
|
+
}
|
|
276
|
+
catch (err) {
|
|
277
|
+
throw new TemplateFileError(`from: ${from}: cannot read ${resolved} (${err.code ?? String(err)})`, resolved);
|
|
278
|
+
}
|
|
279
|
+
if (opts.root !== undefined)
|
|
280
|
+
await assertInside(opts.root, resolved, `from: ${from}`);
|
|
281
|
+
const effective = kind ?? (st.isDirectory() ? 'tar' : 'file');
|
|
282
|
+
if (st.isDirectory()) {
|
|
283
|
+
if (effective !== 'tar')
|
|
284
|
+
throw new TemplateFileError(`from: ${from} is a folder; a folder uploads as kind tar`, resolved);
|
|
285
|
+
const dest = path.join(await opts.tmpDir(), `${path.basename(resolved) || 'folder'}-${Math.random().toString(16).slice(2, 10)}.tar`);
|
|
286
|
+
const packed = await packDirectory(resolved, dest);
|
|
287
|
+
if (packed.size > UPLOAD_BYTES_MAX)
|
|
288
|
+
throw new TemplateFileError(`from: ${from} packs to ${packed.size} bytes; one upload is at most ${UPLOAD_BYTES_MAX} bytes (5 GiB)`, resolved);
|
|
289
|
+
return { from, path: resolved, kind: 'tar', uploadPath: dest, ...packed };
|
|
290
|
+
}
|
|
291
|
+
if (!st.isFile())
|
|
292
|
+
throw new TemplateFileError(`from: ${from} is neither a file nor a folder`, resolved);
|
|
293
|
+
if (st.size > UPLOAD_BYTES_MAX)
|
|
294
|
+
throw new TemplateFileError(`from: ${from} is ${st.size} bytes; one upload is at most ${UPLOAD_BYTES_MAX} bytes (5 GiB)`, resolved);
|
|
295
|
+
if (effective === 'tar')
|
|
296
|
+
await checkUncompressed(resolved);
|
|
297
|
+
const { sha256, size } = await hashFile(resolved);
|
|
298
|
+
return { from, path: resolved, kind: effective, uploadPath: resolved, sha256, size, entries: null };
|
|
299
|
+
}
|
|
300
|
+
/** A fetch body that streams a local file (replayable: each call opens it again). */
|
|
301
|
+
export async function fileBody(file) {
|
|
302
|
+
const { fs, stream } = await nodeModules();
|
|
303
|
+
return stream.Readable.toWeb(fs.createReadStream(file));
|
|
304
|
+
}
|
|
305
|
+
/** A private temporary directory, created on first use; `cleanup()` removes it. */
|
|
306
|
+
export function tempDir() {
|
|
307
|
+
let dir;
|
|
308
|
+
return {
|
|
309
|
+
get: async () => {
|
|
310
|
+
if (!dir) {
|
|
311
|
+
const { fsp, os, path } = await nodeModules();
|
|
312
|
+
dir = fsp.mkdtemp(path.join(os.tmpdir(), 'shardflux-template-'));
|
|
313
|
+
}
|
|
314
|
+
return dir;
|
|
315
|
+
},
|
|
316
|
+
cleanup: async () => {
|
|
317
|
+
if (!dir)
|
|
318
|
+
return;
|
|
319
|
+
const d = await dir.catch(() => null);
|
|
320
|
+
if (d === null)
|
|
321
|
+
return;
|
|
322
|
+
const { fsp } = await nodeModules();
|
|
323
|
+
await fsp.rm(d, { recursive: true, force: true });
|
|
324
|
+
},
|
|
325
|
+
};
|
|
326
|
+
}
|