@intellectual-frontiers/formats 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +8 -2
  3. package/dist/archive.d.ts +147 -0
  4. package/dist/archive.js +765 -0
  5. package/dist/base64.d.ts +24 -0
  6. package/dist/base64.js +52 -0
  7. package/dist/cbor.d.ts +39 -0
  8. package/dist/cbor.js +244 -0
  9. package/dist/csv.d.ts +38 -0
  10. package/dist/csv.js +130 -0
  11. package/dist/der.d.ts +67 -0
  12. package/dist/der.js +221 -0
  13. package/dist/glob.d.ts +64 -0
  14. package/dist/glob.js +218 -0
  15. package/dist/hex.d.ts +17 -0
  16. package/dist/hex.js +30 -0
  17. package/dist/http.d.ts +46 -0
  18. package/dist/http.js +102 -0
  19. package/dist/json5.d.ts +46 -0
  20. package/dist/json5.js +291 -0
  21. package/dist/media-types.d.ts +28 -0
  22. package/dist/media-types.js +121 -0
  23. package/dist/mod.d.ts +39 -0
  24. package/dist/mod.js +31 -0
  25. package/dist/png.d.ts +124 -0
  26. package/dist/png.js +607 -0
  27. package/dist/semver.d.ts +94 -0
  28. package/dist/semver.js +331 -0
  29. package/dist/toml.d.ts +51 -0
  30. package/dist/toml.js +583 -0
  31. package/dist/ts-literal.d.ts +62 -0
  32. package/dist/ts-literal.js +173 -0
  33. package/dist/xml.d.ts +73 -0
  34. package/dist/xml.js +282 -0
  35. package/dist/yaml.d.ts +44 -0
  36. package/dist/yaml.js +706 -0
  37. package/package.json +118 -4
  38. package/src/archive.ts +831 -0
  39. package/src/base64.ts +51 -0
  40. package/src/cbor.ts +235 -0
  41. package/src/csv.ts +132 -0
  42. package/src/der.ts +224 -0
  43. package/src/glob.ts +210 -0
  44. package/src/hex.ts +28 -0
  45. package/src/http.ts +104 -0
  46. package/src/json5.ts +267 -0
  47. package/src/media-types.ts +127 -0
  48. package/src/mod.ts +102 -0
  49. package/src/png.ts +625 -0
  50. package/src/semver.ts +373 -0
  51. package/src/toml.ts +525 -0
  52. package/src/ts-literal.ts +183 -0
  53. package/src/xml.ts +291 -0
  54. package/src/yaml.ts +616 -0
package/src/archive.ts ADDED
@@ -0,0 +1,831 @@
1
+ // Spec: 0001-formats FR-010
2
+ /**
3
+ * Archives in memory, with no outside package (commons 0001-formats FR-010). Writing: a POSIX ustar archive, a gzip stream of
4
+ * stored deflate blocks and a zip of stored entries (`zip`), each entry with a fixed time and mode and in the byte order of its path, so the
5
+ * bytes depend on the files alone (taken from erga's `lib/ts/archive`); and, where size matters more than the same bytes on every runtime,
6
+ * a zip of deflated entries (`zipDeflated`, through the web standard `CompressionStream`), the same bytes for the same files on one runtime
7
+ * and the same files unzipped on every runtime. Reading: a tar (ustar, pax and GNU long names), a gzip and a zip whose
8
+ * entries are stored or deflated (through the web standard `DecompressionStream`), each checked (header checksums, CRC-32) and every path
9
+ * refused that would leave the folder it is unpacked into. This is what lets Stolos drop `fflate` and unpack a verified copy of its engine.
10
+ * A tar, gzipped or not, is also read as a stream (`untarStream`, `readTarStream`), an entry at a time with each entry's bytes a stream of
11
+ * their own, with the same checks, so that an archive larger than memory can be read (FR-059).
12
+ *
13
+ * @example
14
+ * ```ts
15
+ * import { tar, untar, zip, unzip } from "./archive.ts";
16
+ * const files = new Map([["a/b.txt", new TextEncoder().encode("hello")]]);
17
+ * const z = zip(files);
18
+ * const back = await unzip(z); // { kind: "ok", value: Map { "a/b.txt" => Uint8Array [104, 101, 108, 108, 111] } }
19
+ * ```
20
+ *
21
+ * @module
22
+ */
23
+ import { absurd, err, ok, type Result } from "@intellectual-frontiers/core";
24
+
25
+ /** The files of an archive, by their path inside it. */
26
+ export type Files = ReadonlyMap<string, Uint8Array>;
27
+
28
+ /** Why an archive could not be read or written. */
29
+ export type ArchiveError =
30
+ | { readonly kind: "damaged"; readonly message: string }
31
+ | { readonly kind: "unsupported"; readonly message: string }
32
+ | { readonly kind: "unsafe-path"; readonly path: string }
33
+ | { readonly kind: "too-long"; readonly path: string };
34
+
35
+ /** An archive error as one sentence. */
36
+ export function formatArchiveError(e: ArchiveError): string {
37
+ switch (e.kind) {
38
+ case "damaged":
39
+ return `the archive is damaged: ${e.message}`;
40
+ case "unsupported":
41
+ return `the archive uses what is not supported: ${e.message}`;
42
+ case "unsafe-path":
43
+ return `the archive holds a path that would leave its folder: ${JSON.stringify(e.path)}`;
44
+ case "too-long":
45
+ return `the path is too long for a ustar header: ${JSON.stringify(e.path)}`;
46
+ default:
47
+ return absurd(e);
48
+ }
49
+ }
50
+
51
+ const damaged = (message: string): Result<never, ArchiveError> => err({ kind: "damaged", message });
52
+ const unsupported = (message: string): Result<never, ArchiveError> => err({ kind: "unsupported", message });
53
+
54
+ const CRC_TABLE: readonly number[] = Array.from({ length: 256 }, (_, i) => {
55
+ let c = i;
56
+ for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
57
+ return c >>> 0;
58
+ });
59
+
60
+ /** The CRC-32 (ISO 3309) that gzip and zip both carry. */
61
+ export function crc32(bytes: Uint8Array): number {
62
+ let c = 0xffffffff;
63
+ for (const b of bytes) c = ((CRC_TABLE[(c ^ b) & 0xff] ?? 0) ^ (c >>> 8)) >>> 0;
64
+ return (c ^ 0xffffffff) >>> 0;
65
+ }
66
+
67
+ /**
68
+ * Whether a path inside an archive stays inside the folder it is unpacked into: relative, with `/` between its parts, no part `..`, no
69
+ * empty part but a trailing one, no backslash, no drive letter and no NUL.
70
+ */
71
+ export function isSafePath(path: string): boolean {
72
+ if (path === "" || path.startsWith("/") || path.includes("\\") || path.includes("\0") || /^[A-Za-z]:/.test(path)) return false;
73
+ const parts = path.replace(/\/$/, "").split("/");
74
+ return parts.every((p) => p !== "" && p !== ".." && p !== ".");
75
+ }
76
+
77
+ /** The paths of the files in the byte order of their UTF-8, the order every writer here uses. */
78
+ export function sortedPaths(paths: Iterable<string>): readonly string[] {
79
+ const enc = new TextEncoder();
80
+ return [...paths].sort((a, b) => {
81
+ const x = enc.encode(a);
82
+ const y = enc.encode(b);
83
+ for (let i = 0; i < Math.min(x.length, y.length); i++) {
84
+ const d = (x[i] ?? 0) - (y[i] ?? 0);
85
+ if (d !== 0) return d;
86
+ }
87
+ return x.length - y.length;
88
+ });
89
+ }
90
+
91
+ class Bytes {
92
+ #parts: Uint8Array[] = [];
93
+ length = 0;
94
+ push(b: Uint8Array): void {
95
+ this.#parts.push(b);
96
+ this.length += b.length;
97
+ }
98
+ u16(n: number): void {
99
+ this.push(new Uint8Array([n & 0xff, (n >>> 8) & 0xff]));
100
+ }
101
+ u32(n: number): void {
102
+ this.push(new Uint8Array([n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, (n >>> 24) & 0xff]));
103
+ }
104
+ done(): Uint8Array {
105
+ return concat(this.#parts);
106
+ }
107
+ }
108
+
109
+ /** Byte arrays joined end to end. */
110
+ export function concat(parts: readonly Uint8Array[]): Uint8Array {
111
+ const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
112
+ let at = 0;
113
+ for (const p of parts) {
114
+ out.set(p, at);
115
+ at += p.length;
116
+ }
117
+ return out;
118
+ }
119
+
120
+ function octal(h: Uint8Array, at: number, width: number, value: number): void {
121
+ const digits = width - 1;
122
+ const text = value.toString(8).padStart(digits, "0").slice(-digits);
123
+ h.set(new TextEncoder().encode(text), at);
124
+ h[at + digits] = 0;
125
+ }
126
+
127
+ /** A path too long for the 100 bytes of a ustar name, split into its prefix (up to 155 bytes) and name; undefined where it cannot be. */
128
+ export function splitName(path: string): readonly [string, string] | undefined {
129
+ const bytes = (s: string): number => new TextEncoder().encode(s).length;
130
+ if (bytes(path) <= 100) return ["", path];
131
+ let cut: number | undefined;
132
+ for (let i = 0; i < path.length; i++) if (path[i] === "/" && bytes(path.slice(0, i)) <= 155 && bytes(path.slice(i + 1)) <= 100) cut = i;
133
+ return cut === undefined ? undefined : [path.slice(0, cut), path.slice(cut + 1)];
134
+ }
135
+
136
+ /** One ustar header block of a file (mode 0644) or a folder (0755), owned by user 0, dated 0; undefined where the path does not fit. */
137
+ export function tarHeader(path: string, size: number, dir: boolean): Uint8Array | undefined {
138
+ const split = splitName(path);
139
+ if (split === undefined) return undefined;
140
+ const enc = new TextEncoder();
141
+ const h = new Uint8Array(512);
142
+ h.set(enc.encode(split[1]), 0);
143
+ octal(h, 100, 8, dir ? 0o755 : 0o644);
144
+ octal(h, 108, 8, 0);
145
+ octal(h, 116, 8, 0);
146
+ octal(h, 124, 12, size);
147
+ octal(h, 136, 12, 0);
148
+ h.set(enc.encode(" "), 148);
149
+ h[156] = dir ? 0x35 : 0x30;
150
+ h.set(enc.encode("ustar\0"), 257);
151
+ h.set(enc.encode("00"), 263);
152
+ h.set(enc.encode(split[0]), 345);
153
+ const sum = h.reduce((s, b) => s + b, 0);
154
+ h.set(enc.encode(`${sum.toString(8).padStart(6, "0")}\0 `), 148);
155
+ return h;
156
+ }
157
+
158
+ /** A ustar archive of the files, each folder they lie in first given its own entry; or the path that does not fit or is not safe. */
159
+ export function tar(files: Files): Result<Uint8Array, ArchiveError> {
160
+ const dirs = new Set<string>();
161
+ for (const path of files.keys()) {
162
+ if (!isSafePath(path)) return err({ kind: "unsafe-path", path });
163
+ for (let i = path.indexOf("/"); i >= 0; i = path.indexOf("/", i + 1)) dirs.add(`${path.slice(0, i)}/`);
164
+ }
165
+ const out = new Bytes();
166
+ for (const d of sortedPaths(dirs)) {
167
+ const h = tarHeader(d, 0, true);
168
+ if (h === undefined) return err({ kind: "too-long", path: d });
169
+ out.push(h);
170
+ }
171
+ for (const path of sortedPaths(files.keys())) {
172
+ const data = files.get(path) ?? new Uint8Array();
173
+ const h = tarHeader(path, data.length, false);
174
+ if (h === undefined) return err({ kind: "too-long", path });
175
+ out.push(h);
176
+ out.push(data);
177
+ out.push(new Uint8Array((512 - (data.length % 512)) % 512));
178
+ }
179
+ out.push(new Uint8Array(1024));
180
+ return ok(out.done());
181
+ }
182
+
183
+ /** A gzip stream (RFC 1952) of the bytes in stored deflate blocks (RFC 1951 §3.2.4), with no name and no time: the same bytes every time. */
184
+ export function gzip(data: Uint8Array): Uint8Array {
185
+ const out = new Bytes();
186
+ out.push(new Uint8Array([0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0xff]));
187
+ if (data.length === 0) out.push(new Uint8Array([1, 0, 0, 0xff, 0xff]));
188
+ for (let at = 0; at < data.length; at += 0xffff) {
189
+ const chunk = data.subarray(at, Math.min(at + 0xffff, data.length));
190
+ out.push(new Uint8Array([at + 0xffff >= data.length ? 1 : 0]));
191
+ out.u16(chunk.length);
192
+ out.u16(~chunk.length & 0xffff);
193
+ out.push(chunk);
194
+ }
195
+ out.u32(crc32(data));
196
+ out.u32(data.length >>> 0);
197
+ return out.done();
198
+ }
199
+
200
+ /** A date as a zip writes it (MS-DOS): `(year - 1980) << 9 | month << 5 | day`. */
201
+ export function dosDate(year: number, month: number, day: number): number {
202
+ return ((year - 1980) << 9) | (month << 5) | day;
203
+ }
204
+
205
+ /** The zip's fixed date: 1980-01-01, the first a zip can hold. */
206
+ export const ZIP_EPOCH: number = dosDate(1980, 1, 1);
207
+
208
+ /** One entry as a zip writer lays it out: its path, CRC-32 and size, and its bytes as written (stored, or deflated). */
209
+ type ZipItem = {
210
+ readonly path: string;
211
+ readonly crc: number;
212
+ readonly size: number;
213
+ readonly method: 0 | 8;
214
+ readonly body: Uint8Array;
215
+ };
216
+
217
+ /** A zip of the entries, in their order, each dated `date` at midnight and readable by everyone. */
218
+ function zipOf(items: readonly ZipItem[], date: number): Uint8Array {
219
+ const out = new Bytes();
220
+ const central = new Bytes();
221
+ const enc = new TextEncoder();
222
+ for (const item of items) {
223
+ const offset = out.length;
224
+ const name = enc.encode(item.path);
225
+ const common = (v: Bytes): void => {
226
+ v.u16(20);
227
+ v.u16(0x0800);
228
+ v.u16(item.method);
229
+ v.u16(0);
230
+ v.u16(date);
231
+ v.u32(item.crc);
232
+ v.u32(item.body.length);
233
+ v.u32(item.size);
234
+ v.u16(name.length);
235
+ v.u16(0);
236
+ };
237
+ out.u32(0x04034b50);
238
+ common(out);
239
+ out.push(name);
240
+ out.push(item.body);
241
+ central.u32(0x02014b50);
242
+ central.u16((3 << 8) | 20);
243
+ common(central);
244
+ central.u16(0);
245
+ central.u16(0);
246
+ central.u16(0);
247
+ central.u32((0o100644 << 16) >>> 0);
248
+ central.u32(offset);
249
+ central.push(name);
250
+ }
251
+ const start = out.length;
252
+ const size = central.length;
253
+ out.push(central.done());
254
+ out.u32(0x06054b50);
255
+ out.u16(0);
256
+ out.u16(0);
257
+ out.u16(items.length);
258
+ out.u16(items.length);
259
+ out.u32(size);
260
+ out.u32(start);
261
+ out.u16(0);
262
+ return out.done();
263
+ }
264
+
265
+ /** A zip (APPNOTE 6.3) of the files, each stored, dated `date` (1980-01-01 unless another is given) at midnight and readable by everyone. */
266
+ export function zip(files: Files, date: number = ZIP_EPOCH): Uint8Array {
267
+ return zipOf(
268
+ sortedPaths(files.keys()).map((path) => {
269
+ const data = files.get(path) ?? new Uint8Array();
270
+ return { path, crc: crc32(data), size: data.length, method: 0, body: data };
271
+ }),
272
+ date,
273
+ );
274
+ }
275
+
276
+ /**
277
+ * A zip (APPNOTE 6.3) of the files as `zip` lays them out, but each entry deflated (method 8) through the web standard
278
+ * `CompressionStream("deflate-raw")`, or stored where deflating would not make it smaller (an empty or incompressible file). The bytes
279
+ * depend on the files alone on one runtime; another runtime's deflater may choose other (equally valid) bytes, so only what they unzip to
280
+ * is the same everywhere.
281
+ */
282
+ export async function zipDeflated(files: Files, date: number = ZIP_EPOCH): Promise<Uint8Array> {
283
+ const items: ZipItem[] = [];
284
+ for (const path of sortedPaths(files.keys())) {
285
+ const data = files.get(path) ?? new Uint8Array();
286
+ const packed = await deflateRaw(data);
287
+ items.push(
288
+ packed.length < data.length
289
+ ? { path, crc: crc32(data), size: data.length, method: 8, body: packed }
290
+ : { path, crc: crc32(data), size: data.length, method: 0, body: data },
291
+ );
292
+ }
293
+ return zipOf(items, date);
294
+ }
295
+
296
+ /** The raw deflate stream (RFC 1951) a web `CompressionStream` gives for the bytes. */
297
+ export async function deflateRaw(data: Uint8Array): Promise<Uint8Array> {
298
+ return drain(new Blob([new Uint8Array(data)]).stream().pipeThrough(new CompressionStream("deflate-raw")));
299
+ }
300
+
301
+ /** Every chunk of a stream of bytes, joined. */
302
+ async function drain(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
303
+ const reader = stream.getReader();
304
+ const parts: Uint8Array[] = [];
305
+ for (;;) {
306
+ const { done, value } = await reader.read();
307
+ if (done) break;
308
+ parts.push(value);
309
+ }
310
+ return concat(parts);
311
+ }
312
+
313
+ /** The bytes a web `DecompressionStream` gives for the input, or the error it raised. */
314
+ async function decompress(format: "gzip" | "deflate-raw", data: Uint8Array): Promise<Result<Uint8Array, ArchiveError>> {
315
+ try {
316
+ return ok(await drain(new Blob([new Uint8Array(data)]).stream().pipeThrough(new DecompressionStream(format))));
317
+ } catch (e) {
318
+ return damaged(`${format}: ${e instanceof Error ? e.message : String(e)}`);
319
+ }
320
+ }
321
+
322
+ /** The bytes of a gzip stream, stored or deflated, its CRC checked by the decompressor. */
323
+ export function gunzip(data: Uint8Array): Promise<Result<Uint8Array, ArchiveError>> {
324
+ if (data.length < 18 || data[0] !== 0x1f || data[1] !== 0x8b) return Promise.resolve(damaged("it does not begin as a gzip stream"));
325
+ return decompress("gzip", data);
326
+ }
327
+
328
+ /** One entry of a tar archive. A folder's path has no trailing `/`; a link's target is in `linkName`. */
329
+ export type TarEntry = {
330
+ readonly path: string;
331
+ readonly kind: "file" | "directory" | "symlink" | "hardlink";
332
+ readonly mode: number;
333
+ readonly data: Uint8Array;
334
+ readonly linkName: string;
335
+ };
336
+
337
+ const cstring = (b: Uint8Array): string => {
338
+ const end = b.indexOf(0);
339
+ return new TextDecoder().decode(end < 0 ? b : b.subarray(0, end));
340
+ };
341
+
342
+ function readNumber(field: Uint8Array): number | undefined {
343
+ if (((field[0] ?? 0) & 0x80) !== 0) {
344
+ let n = 0;
345
+ for (let i = 1; i < field.length; i++) n = n * 256 + (field[i] ?? 0);
346
+ return Number.isSafeInteger(n) ? n : undefined;
347
+ }
348
+ const text = cstring(field).trim();
349
+ if (text === "") return 0;
350
+ return /^[0-7]+$/.test(text) ? Number.parseInt(text, 8) : undefined;
351
+ }
352
+
353
+ function paxRecords(data: Uint8Array): ReadonlyMap<string, string> {
354
+ const out = new Map<string, string>();
355
+ const text = new TextDecoder().decode(data);
356
+ let at = 0;
357
+ while (at < text.length) {
358
+ const space = text.indexOf(" ", at);
359
+ const length = Number.parseInt(text.slice(at, space), 10);
360
+ if (space < 0 || !Number.isInteger(length) || length <= 0) break;
361
+ const record = text.slice(space + 1, at + length - 1);
362
+ const eq = record.indexOf("=");
363
+ if (eq > 0) out.set(record.slice(0, eq), record.slice(eq + 1));
364
+ at += length;
365
+ }
366
+ return out;
367
+ }
368
+
369
+ /** The fields of one tar header block, its checksum checked: what an entry's header says before pax and GNU long names apply. */
370
+ type RawHeader = {
371
+ readonly block: Uint8Array;
372
+ readonly size: number;
373
+ readonly mode: number;
374
+ readonly mtime: number;
375
+ readonly type: string;
376
+ };
377
+
378
+ /** Whether a block of 512 bytes is all zeros, the end of a tar archive. */
379
+ const isZeroBlock = (h: Uint8Array): boolean => h.every((b) => b === 0);
380
+
381
+ /** The 512 bytes of a body of `n` bytes, padded to a whole number of blocks. */
382
+ const padded = (n: number): number => Math.ceil(n / 512) * 512;
383
+
384
+ /** The fields of the header block at byte `at`, or why it is damaged (a wrong checksum, a field that is not a number). */
385
+ function readHeader(h: Uint8Array, at: number): Result<RawHeader, ArchiveError> {
386
+ const stored = readNumber(h.subarray(148, 156));
387
+ let sum = 0;
388
+ for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 0x20 : (h[i] ?? 0);
389
+ if (stored !== sum) return damaged(`the header at byte ${at} has a wrong checksum`);
390
+ const size = readNumber(h.subarray(124, 136));
391
+ const mode = readNumber(h.subarray(100, 108));
392
+ const mtime = readNumber(h.subarray(136, 148));
393
+ if (size === undefined || mode === undefined || mtime === undefined)
394
+ return damaged(`the header at byte ${at} has a field that is not a number`);
395
+ return ok({ block: h, size, mode, mtime, type: String.fromCharCode(h[156] ?? 0) });
396
+ }
397
+
398
+ /** What pax extended headers and GNU long names and links say of the entry after them. */
399
+ type Pending = { readonly pax: ReadonlyMap<string, string>; readonly longName: string | undefined; readonly longLink: string | undefined };
400
+
401
+ const NO_PENDING: Pending = { pax: new Map(), longName: undefined, longLink: undefined };
402
+
403
+ /** The length of the body after a header, a pax `size` before it taking the place of the header's. */
404
+ function bodyLength(raw: RawHeader, pending: Pending, at: number): Result<number, ArchiveError> {
405
+ const paxSize = pending.pax.get("size");
406
+ if (paxSize === undefined) return ok(raw.size);
407
+ const n = /^[0-9]+$/.test(paxSize) ? Number.parseInt(paxSize, 10) : Number.NaN;
408
+ return Number.isSafeInteger(n) ? ok(n) : damaged(`the pax size before byte ${at} is not a number`);
409
+ }
410
+
411
+ /** Whether a header's type is one that only says something of the entry after it (pax `x` and `g`, GNU `L` and `K`). */
412
+ const isMetaType = (type: string): boolean => type === "x" || type === "g" || type === "L" || type === "K";
413
+
414
+ /** What is pending after a header of a meta type (see `isMetaType`) with its body. */
415
+ function pendingAfter(type: string, body: Uint8Array, pending: Pending): Pending {
416
+ switch (type) {
417
+ case "x":
418
+ return { ...pending, pax: paxRecords(body) };
419
+ case "L":
420
+ return { ...pending, longName: cstring(body) };
421
+ case "K":
422
+ return { ...pending, longLink: cstring(body) };
423
+ default:
424
+ return pending;
425
+ }
426
+ }
427
+
428
+ /** The header of one entry of a tar archive: its path (a folder's with no trailing `/`), kind, mode, size of body, time and link target. */
429
+ export type TarHeader = {
430
+ readonly path: string;
431
+ readonly kind: TarEntry["kind"];
432
+ readonly mode: number;
433
+ readonly size: number;
434
+ /** Seconds since 1970, as the header (or a pax `mtime`) gives them. */
435
+ readonly mtime: number;
436
+ readonly linkName: string;
437
+ };
438
+
439
+ /** The entry a header names, after what pending headers said; undefined for the archive's own folder (`.`); or why it is refused. */
440
+ function entryHeader(raw: RawHeader, pending: Pending, size: number): Result<TarHeader | undefined, ArchiveError> {
441
+ const h = raw.block;
442
+ const isUstar = cstring(h.subarray(257, 263)).startsWith("ustar");
443
+ const prefix = isUstar ? cstring(h.subarray(345, 500)) : "";
444
+ const name =
445
+ pending.pax.get("path") ??
446
+ pending.longName ??
447
+ (prefix === "" ? cstring(h.subarray(0, 100)) : `${prefix}/${cstring(h.subarray(0, 100))}`);
448
+ const linkName = pending.pax.get("linkpath") ?? pending.longLink ?? cstring(h.subarray(157, 257));
449
+ const paxTime = Number.parseFloat(pending.pax.get("mtime") ?? "");
450
+ const mtime = Number.isFinite(paxTime) ? paxTime : raw.mtime;
451
+ const path = name.replace(/^\.\//, "").replace(/\/$/, "");
452
+ if (path === "" || path === ".") return ok(undefined);
453
+ if (!isSafePath(path)) return err({ kind: "unsafe-path", path: name });
454
+ let kind: TarEntry["kind"];
455
+ switch (raw.type) {
456
+ case "0":
457
+ case "\0":
458
+ case "7":
459
+ kind = name.endsWith("/") ? "directory" : "file";
460
+ break;
461
+ case "5":
462
+ kind = "directory";
463
+ break;
464
+ case "2":
465
+ kind = "symlink";
466
+ break;
467
+ case "1":
468
+ kind = "hardlink";
469
+ break;
470
+ default:
471
+ return unsupported(`the entry ${JSON.stringify(path)} has type ${JSON.stringify(raw.type)}`);
472
+ }
473
+ return ok({ path, kind, mode: raw.mode, size, mtime, linkName });
474
+ }
475
+
476
+ /** The entries of a tar archive (ustar, with pax extended headers and GNU long names), each header's checksum checked and each path checked safe. */
477
+ export function untar(data: Uint8Array): Result<readonly TarEntry[], ArchiveError> {
478
+ const entries: TarEntry[] = [];
479
+ let at = 0;
480
+ let pending = NO_PENDING;
481
+ while (at + 512 <= data.length) {
482
+ const h = data.subarray(at, at + 512);
483
+ if (isZeroBlock(h)) return ok(entries);
484
+ const raw = readHeader(h, at);
485
+ if (raw.kind === "err") return raw;
486
+ const length = bodyLength(raw.value, pending, at);
487
+ if (length.kind === "err") return length;
488
+ const bodyAt = at + 512;
489
+ if (bodyAt + length.value > data.length) return damaged(`the entry at byte ${at} runs past the end`);
490
+ const body = data.subarray(bodyAt, bodyAt + length.value);
491
+ at = bodyAt + padded(length.value);
492
+ if (isMetaType(raw.value.type)) {
493
+ pending = pendingAfter(raw.value.type, body, pending);
494
+ continue;
495
+ }
496
+ const e = entryHeader(raw.value, pending, length.value);
497
+ pending = NO_PENDING;
498
+ if (e.kind === "err") return e;
499
+ if (e.value === undefined) continue;
500
+ const { path, kind, mode, linkName } = e.value;
501
+ entries.push({ path, kind, mode, data: kind === "file" ? body : new Uint8Array(), linkName });
502
+ }
503
+ return damaged("it ends without the two zero blocks of a tar archive");
504
+ }
505
+
506
+ /** An archive error raised as an `Error`, which is how a stream reports one: the error itself is in `error`. */
507
+ export class ArchiveFailure extends Error {
508
+ /** Why the archive could not be read. */
509
+ readonly error: ArchiveError;
510
+ /** An `Error` for the archive error, its message the error's sentence. */
511
+ constructor(error: ArchiveError) {
512
+ super(formatArchiveError(error));
513
+ this.name = "ArchiveFailure";
514
+ this.error = error;
515
+ }
516
+ }
517
+
518
+ /**
519
+ * One entry of a tar archive read as a stream: its header and its body as a stream (empty but for a file). The body MUST be read to its
520
+ * end or cancelled before the next entry is given, as with `@std/tar`'s `UntarStream`.
521
+ */
522
+ export type TarStreamEntry = TarHeader & { readonly readable: ReadableStream<Uint8Array> };
523
+
524
+ /** The largest body of a pax or GNU long-name header read: such a header names one entry, and a larger one is refused as damaged. */
525
+ const MAX_META = 1 << 20;
526
+
527
+ /** Bytes read from a stream as they are asked for, a chunk at a time, holding no more than one chunk. */
528
+ class ByteSource {
529
+ readonly #reader: ReadableStreamDefaultReader<Uint8Array>;
530
+ #chunk: Uint8Array;
531
+ constructor(reader: ReadableStreamDefaultReader<Uint8Array>, first: Uint8Array = new Uint8Array()) {
532
+ this.#reader = reader;
533
+ this.#chunk = first;
534
+ }
535
+ /** At least one byte and at most `max` (more than 0), or undefined at the end. */
536
+ async some(max: number): Promise<Uint8Array | undefined> {
537
+ while (this.#chunk.length === 0) {
538
+ const r = await this.#reader.read();
539
+ if (r.done) return undefined;
540
+ this.#chunk = r.value;
541
+ }
542
+ const out = this.#chunk.subarray(0, max);
543
+ this.#chunk = this.#chunk.subarray(out.length);
544
+ return out;
545
+ }
546
+ /** `n` bytes, or fewer where the stream ends first. */
547
+ async exactly(n: number): Promise<Uint8Array> {
548
+ const parts: Uint8Array[] = [];
549
+ let got = 0;
550
+ while (got < n) {
551
+ const part = await this.some(n - got);
552
+ if (part === undefined) break;
553
+ parts.push(part);
554
+ got += part.length;
555
+ }
556
+ return parts.length === 1 && parts[0] !== undefined ? parts[0] : concat(parts);
557
+ }
558
+ /** Passes over `n` bytes; false where the stream ends first. */
559
+ async skip(n: number): Promise<boolean> {
560
+ for (let left = n; left > 0;) {
561
+ const part = await this.some(left);
562
+ if (part === undefined) return false;
563
+ left -= part.length;
564
+ }
565
+ return true;
566
+ }
567
+ /** Reads to the end, so that a gzip stream's CRC is checked. */
568
+ async drain(): Promise<void> {
569
+ while ((await this.some(Number.MAX_SAFE_INTEGER)) !== undefined);
570
+ }
571
+ cancel(reason: unknown): Promise<void> {
572
+ return this.#reader.cancel(reason);
573
+ }
574
+ /** The rest of the bytes as a stream of their own, this source no longer read. */
575
+ rest(): ReadableStream<Uint8Array> {
576
+ const reader = this.#reader;
577
+ const first = this.#chunk;
578
+ return new ReadableStream<Uint8Array>(
579
+ {
580
+ start(controller) {
581
+ if (first.length > 0) controller.enqueue(first);
582
+ },
583
+ async pull(controller) {
584
+ const r = await reader.read();
585
+ if (r.done) controller.close();
586
+ else controller.enqueue(r.value);
587
+ },
588
+ cancel(reason) {
589
+ return reader.cancel(reason);
590
+ },
591
+ },
592
+ { highWaterMark: 0 },
593
+ );
594
+ }
595
+ }
596
+
597
+ /** The tar bytes of a stream: the stream itself, or what it decompresses to where it begins with gzip's magic bytes `1f 8b`. */
598
+ async function tarBytes(stream: ReadableStream<Uint8Array>): Promise<ByteSource> {
599
+ const peek = new ByteSource(stream.getReader());
600
+ const head = await peek.exactly(2);
601
+ const start = new ByteSource(peek.rest().getReader(), head);
602
+ if (head[0] !== 0x1f || head[1] !== 0x8b) return start;
603
+ const raw = start.rest() as ReadableStream<Uint8Array<ArrayBuffer>>; // what DecompressionStream's writable takes; the bytes are the same
604
+ return new ByteSource(raw.pipeThrough(new DecompressionStream("gzip")).getReader());
605
+ }
606
+
607
+ const failure = (e: unknown): ArchiveFailure =>
608
+ e instanceof ArchiveFailure ? e : new ArchiveFailure({ kind: "damaged", message: e instanceof Error ? e.message : String(e) });
609
+
610
+ const emptyStream = (): ReadableStream<Uint8Array> =>
611
+ new ReadableStream<Uint8Array>({
612
+ start(controller) {
613
+ controller.close();
614
+ },
615
+ });
616
+
617
+ /**
618
+ * The entries of a tar archive given as a stream of bytes, gzipped or not (told by its first two bytes), one at a time as they are asked
619
+ * for, so that an archive larger than memory is read with one chunk held (commons 0001-formats FR-011). Every header is checked
620
+ * as `untar` checks it (checksum, pax and GNU long names, a safe path); a failure errors the stream with an `ArchiveFailure`. Each entry's
621
+ * `readable` MUST be read to its end or cancelled before the next entry is given; cancelling the stream of entries cancels the bytes.
622
+ */
623
+ export function readTarStream(stream: ReadableStream<Uint8Array>): ReadableStream<TarStreamEntry> {
624
+ let source: ByteSource | undefined;
625
+ let pending = NO_PENDING;
626
+ let at = 0;
627
+ let skipNext = 0;
628
+ let bodyDone: Promise<void> = Promise.resolve();
629
+ let broken: ArchiveFailure | undefined;
630
+ const next = async (src: ByteSource): Promise<TarStreamEntry | undefined> => {
631
+ await bodyDone;
632
+ if (broken !== undefined) throw broken;
633
+ if (!(await src.skip(skipNext)))
634
+ throw new ArchiveFailure({ kind: "damaged", message: `the entry before byte ${at} runs past the end` });
635
+ skipNext = 0;
636
+ for (;;) {
637
+ const h = await src.exactly(512);
638
+ if (h.length < 512) throw new ArchiveFailure({ kind: "damaged", message: "it ends without the two zero blocks of a tar archive" });
639
+ if (isZeroBlock(h)) {
640
+ await src.drain();
641
+ return undefined;
642
+ }
643
+ const raw = readHeader(h, at);
644
+ if (raw.kind === "err") throw new ArchiveFailure(raw.error);
645
+ const length = bodyLength(raw.value, pending, at);
646
+ if (length.kind === "err") throw new ArchiveFailure(length.error);
647
+ const here = at;
648
+ const size = length.value;
649
+ const pad = padded(size) - size;
650
+ at += 512 + padded(size);
651
+ const pastEnd = new ArchiveFailure({ kind: "damaged", message: `the entry at byte ${here} runs past the end` });
652
+ if (isMetaType(raw.value.type)) {
653
+ if (size > MAX_META)
654
+ throw new ArchiveFailure({ kind: "damaged", message: `the extended header at byte ${here} is larger than 1 MiB` });
655
+ const body = await src.exactly(size);
656
+ if (body.length < size || !(await src.skip(pad))) throw pastEnd;
657
+ pending = pendingAfter(raw.value.type, body, pending);
658
+ continue;
659
+ }
660
+ const e = entryHeader(raw.value, pending, size);
661
+ pending = NO_PENDING;
662
+ if (e.kind === "err") throw new ArchiveFailure(e.error);
663
+ if (e.value === undefined || e.value.kind !== "file" || size === 0) {
664
+ if (!(await src.skip(size + pad))) throw pastEnd;
665
+ if (e.value === undefined) continue;
666
+ return { ...e.value, readable: emptyStream() };
667
+ }
668
+ let remaining = size;
669
+ skipNext = pad;
670
+ let finish = (): void => {};
671
+ bodyDone = new Promise<void>((resolve) => {
672
+ finish = resolve;
673
+ });
674
+ const readable = new ReadableStream<Uint8Array>(
675
+ {
676
+ async pull(controller) {
677
+ let part: Uint8Array | undefined;
678
+ try {
679
+ part = await src.some(remaining);
680
+ } catch (thrown) {
681
+ part = undefined;
682
+ broken = failure(thrown);
683
+ }
684
+ if (part === undefined) {
685
+ broken ??= pastEnd;
686
+ controller.error(broken);
687
+ finish();
688
+ return;
689
+ }
690
+ remaining -= part.length;
691
+ controller.enqueue(part);
692
+ if (remaining === 0) {
693
+ controller.close();
694
+ finish();
695
+ }
696
+ },
697
+ cancel() {
698
+ skipNext += remaining;
699
+ remaining = 0;
700
+ finish();
701
+ },
702
+ },
703
+ { highWaterMark: 0 },
704
+ );
705
+ return { ...e.value, readable };
706
+ }
707
+ };
708
+ return new ReadableStream<TarStreamEntry>(
709
+ {
710
+ async pull(controller) {
711
+ try {
712
+ source ??= await tarBytes(stream);
713
+ const e = await next(source);
714
+ if (e === undefined) controller.close();
715
+ else controller.enqueue(e);
716
+ } catch (thrown) {
717
+ const f = failure(thrown);
718
+ await (source === undefined ? stream.cancel(f) : source.cancel(f)).catch(() => undefined);
719
+ throw f;
720
+ }
721
+ },
722
+ async cancel(reason) {
723
+ await (source === undefined ? stream.cancel(reason) : source.cancel(reason));
724
+ },
725
+ },
726
+ { highWaterMark: 0 },
727
+ );
728
+ }
729
+
730
+ /**
731
+ * A stream from a tar archive's bytes, gzipped or not, to its entries, to put in a `pipeThrough` as `@std/tar`'s `UntarStream` is: see
732
+ * `readTarStream` for what it checks and how each entry's body is read (commons 0001-formats FR-011).
733
+ *
734
+ * @example
735
+ * ```ts
736
+ * for await (const entry of file.readable.pipeThrough(untarStream())) {
737
+ * if (entry.path === "bin/tool") await save(entry.readable);
738
+ * else await entry.readable.cancel();
739
+ * }
740
+ * ```
741
+ */
742
+ export function untarStream(): ReadableWritablePair<TarStreamEntry, Uint8Array> {
743
+ const input = new TransformStream<Uint8Array, Uint8Array>();
744
+ return { writable: input.writable, readable: readTarStream(input.readable) };
745
+ }
746
+
747
+ /** The files of a list of tar entries, by path: folders and links left out. */
748
+ export function filesOf(entries: readonly TarEntry[]): Files {
749
+ return new Map(entries.filter((e) => e.kind === "file").map((e) => [e.path, e.data]));
750
+ }
751
+
752
+ /** One entry of a zip's central directory. */
753
+ export type ZipEntry = {
754
+ readonly name: string;
755
+ readonly method: number;
756
+ readonly crc: number;
757
+ readonly compressed: number;
758
+ readonly size: number;
759
+ readonly offset: number;
760
+ };
761
+
762
+ /** The entries of a zip, from its central directory; or why they cannot be read (a ZIP64 archive among the reasons). */
763
+ export function zipEntries(data: Uint8Array): Result<readonly ZipEntry[], ArchiveError> {
764
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
765
+ let end = -1;
766
+ for (let i = data.length - 22; i >= Math.max(0, data.length - 65_557); i--) {
767
+ if (view.getUint32(i, true) === 0x06054b50) {
768
+ end = i;
769
+ break;
770
+ }
771
+ }
772
+ if (end < 0) return damaged("it has no zip directory at its end");
773
+ const count = view.getUint16(end + 10, true);
774
+ const length = view.getUint32(end + 12, true);
775
+ const offset = view.getUint32(end + 16, true);
776
+ if (offset === 0xffffffff || count === 0xffff) return unsupported("it is a ZIP64 archive");
777
+ if (offset + length > data.length) return damaged("its directory runs past the end");
778
+ const entries: ZipEntry[] = [];
779
+ let at = offset;
780
+ for (let i = 0; i < count; i++) {
781
+ if (at + 46 > data.length || view.getUint32(at, true) !== 0x02014b50) return damaged("its directory is damaged");
782
+ const nameLength = view.getUint16(at + 28, true);
783
+ const extraLength = view.getUint16(at + 30, true);
784
+ const commentLength = view.getUint16(at + 32, true);
785
+ entries.push({
786
+ name: new TextDecoder().decode(data.subarray(at + 46, at + 46 + nameLength)),
787
+ method: view.getUint16(at + 10, true),
788
+ crc: view.getUint32(at + 16, true),
789
+ compressed: view.getUint32(at + 20, true),
790
+ size: view.getUint32(at + 24, true),
791
+ offset: view.getUint32(at + 42, true),
792
+ });
793
+ at += 46 + nameLength + extraLength + commentLength;
794
+ }
795
+ return ok(entries);
796
+ }
797
+
798
+ /** The bytes of one entry of a zip, stored (method 0) or deflated (method 8), its size and CRC-32 checked. */
799
+ export async function zipRead(data: Uint8Array, e: ZipEntry): Promise<Result<Uint8Array, ArchiveError>> {
800
+ const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
801
+ if (e.offset + 30 > data.length || view.getUint32(e.offset, true) !== 0x04034b50)
802
+ return damaged(`the entry ${JSON.stringify(e.name)} has no local header`);
803
+ const start = e.offset + 30 + view.getUint16(e.offset + 26, true) + view.getUint16(e.offset + 28, true);
804
+ if (start + e.compressed > data.length) return damaged(`the entry ${JSON.stringify(e.name)} runs past the end`);
805
+ const raw = data.subarray(start, start + e.compressed);
806
+ let bytes: Uint8Array;
807
+ if (e.method === 0) bytes = raw;
808
+ else if (e.method === 8) {
809
+ const r = await decompress("deflate-raw", raw);
810
+ if (r.kind === "err") return r;
811
+ bytes = r.value;
812
+ } else return unsupported(`the entry ${JSON.stringify(e.name)} uses compression method ${e.method}`);
813
+ if (bytes.length !== e.size || crc32(bytes) !== e.crc)
814
+ return damaged(`the entry ${JSON.stringify(e.name)} does not match its size and CRC-32`);
815
+ return ok(bytes);
816
+ }
817
+
818
+ /** Every file of a zip by its path, folders left out, each read and checked, and every path checked safe. */
819
+ export async function unzip(data: Uint8Array): Promise<Result<Files, ArchiveError>> {
820
+ const entries = zipEntries(data);
821
+ if (entries.kind === "err") return entries;
822
+ const out = new Map<string, Uint8Array>();
823
+ for (const e of entries.value) {
824
+ if (!isSafePath(e.name)) return err({ kind: "unsafe-path", path: e.name });
825
+ if (e.name.endsWith("/")) continue;
826
+ const r = await zipRead(data, e);
827
+ if (r.kind === "err") return r;
828
+ out.set(e.name, r.value);
829
+ }
830
+ return ok(out);
831
+ }