@drupflare/untarl 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +180 -0
- package/package.json +68 -0
- package/src/index.ts +1 -0
- package/src/untar.ts +347 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gregory Mitchell
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# 📦 untarl
|
|
2
|
+
|
|
3
|
+
> Tar and tar.gz extraction with no dependencies and no Node APIs
|
|
4
|
+
|
|
5
|
+
[](https://github.com/drupflare/untarl/actions/workflows/build.yml)
|
|
6
|
+
[](https://github.com/drupflare/untarl/actions/workflows/prettier.yml)
|
|
7
|
+
[](https://codecov.io/gh/drupflare/untarl)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
**Reads a `.tar` or `.tar.gz` on workerd, in the browser, in Bun and in Node, with the same
|
|
11
|
+
code and zero dependencies.** Gzip is handled by the platform's own
|
|
12
|
+
`DecompressionStream('gzip')` rather than a bundled inflater, which is what keeps the
|
|
13
|
+
dependency count at zero and the bundle cost at nothing.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## 📋 Table of Contents
|
|
18
|
+
|
|
19
|
+
- [Why](#-why)
|
|
20
|
+
- [Install](#-install)
|
|
21
|
+
- [Usage](#-usage)
|
|
22
|
+
- [API](#-api)
|
|
23
|
+
- [Untrusted Input Is the Whole Design](#-untrusted-input-is-the-whole-design)
|
|
24
|
+
- [What It Deliberately Does Not Do](#-what-it-deliberately-does-not-do)
|
|
25
|
+
- [Testing](#-testing)
|
|
26
|
+
- [Related Repositories](#-related-repositories)
|
|
27
|
+
- [License](#-license)
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## 🎯 Why
|
|
32
|
+
|
|
33
|
+
The original problem was installing a Drupal contrib module at runtime, which means reading a
|
|
34
|
+
drupal.org `.tar.gz` from inside a Cloudflare Worker. Two obvious answers were both closed:
|
|
35
|
+
|
|
36
|
+
1. **Do it in PHP.** The wasm build has neither `ext-zip` nor `ext-phar`, and adding either is
|
|
37
|
+
a multi-hour Docker rebuild of the interpreter.
|
|
38
|
+
2. **Use a tar library.** Every popular one reaches for `node:zlib`, `node:fs` or `Buffer`.
|
|
39
|
+
workerd has none of those.
|
|
40
|
+
|
|
41
|
+
What workerd does have is `DecompressionStream('gzip')`, natively. And tar is nothing but
|
|
42
|
+
512-byte blocks with an octal header. So the whole job fits in ~320 lines of platform-only
|
|
43
|
+
JavaScript, out where `await` is legal and no build flag has to change.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## 📥 Install
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
bun add @drupflare/untarl
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**One entry point, deliberately.** The `exports` map declares `.` and `./package.json` and nothing
|
|
54
|
+
else, so every symbol arrives from `@drupflare/untarl` and `@drupflare/untarl/src/untar.ts` is refused
|
|
55
|
+
by the resolver. A second name for the only module would be two ways to import the same thing with no
|
|
56
|
+
way to say which is canonical; the sibling `cartridge` splits by subpath because its modules have
|
|
57
|
+
different dependency costs, and here there are none to differ.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## 🚀 Usage
|
|
62
|
+
|
|
63
|
+
The common case is two calls: inflate and parse, then flatten to the files you want to write.
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import { tarEntryTree, untarGzip } from '@drupflare/untarl';
|
|
67
|
+
|
|
68
|
+
const response = await fetch('https://ftp.drupal.org/files/projects/token-8.x-1.15.tar.gz');
|
|
69
|
+
const entries = await untarGzip(response.body!);
|
|
70
|
+
|
|
71
|
+
// strip = 1 drops the `token/` wrapper every drupal.org tarball has
|
|
72
|
+
for (const [path, bytes] of tarEntryTree(entries, 1)) {
|
|
73
|
+
// write `bytes` to `path` here
|
|
74
|
+
}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
If you already hold the bytes, or the archive is not gzipped, skip straight to the parser:
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
import { parseTar } from '@drupflare/untarl';
|
|
81
|
+
|
|
82
|
+
const entries = parseTar(new Uint8Array(await file.arrayBuffer()));
|
|
83
|
+
const names = entries.filter((e) => e.type === 'file').map((e) => e.name);
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## 🔧 API
|
|
89
|
+
|
|
90
|
+
| Export | Signature | What it does |
|
|
91
|
+
| --------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------ |
|
|
92
|
+
| `untarGzip(stream)` | `(ReadableStream<Uint8Array>) => Promise<TarEntry[]>` | gunzips through `DecompressionStream`, then parses |
|
|
93
|
+
| `parseTar(bytes)` | `(Uint8Array) => TarEntry[]` | parses a POSIX/ustar archive faithfully; refuses nothing on path grounds |
|
|
94
|
+
| `tarEntryTree(entries, strip= 1)` | `(TarEntry[], number) => Map<string, Uint8Array>` | flattens to the files to write, refusing paths that escape |
|
|
95
|
+
| `TarEntry` | `{ name, size, type, mode, bytes }` | one member; `bytes` is an owned copy, so the archive can be released |
|
|
96
|
+
| `TarEntryType` | `'file' \| 'directory'` | the several typeflags that mean the same thing, folded together |
|
|
97
|
+
| `TarParseError` | `Error & { offset?: number }` | malformed, truncated or out-of-range archive |
|
|
98
|
+
| `TarPathError` | `Error & { path: string }` | a member path that would escape the target directory |
|
|
99
|
+
|
|
100
|
+
**The split between the two error types is the design.** `parseTar()` is faithful to the bytes
|
|
101
|
+
and will happily hand back an entry whose name climbs out of the target directory with `..`
|
|
102
|
+
segments. `tarEntryTree()` is the layer that
|
|
103
|
+
decides what is safe to write, and it is the only one that throws `TarPathError`. Keeping them
|
|
104
|
+
apart means you can inspect a hostile archive without a parser that lies about its contents.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 🛡 Untrusted Input Is the Whole Design
|
|
109
|
+
|
|
110
|
+
A tarball off the network is untrusted, so every read is bounds-checked and every failure is a
|
|
111
|
+
**named** error rather than a read past the end of the buffer, an infinite loop, or a plausible
|
|
112
|
+
empty result.
|
|
113
|
+
|
|
114
|
+
| Refused | Error |
|
|
115
|
+
| ------------------------------------------------- | ----------------------------------------------- |
|
|
116
|
+
| a header cut short by a truncated download | `TarParseError`, with the byte offset |
|
|
117
|
+
| a `size` field that overruns the buffer | `TarParseError` |
|
|
118
|
+
| a non-octal `size` or `mode` | `TarParseError` |
|
|
119
|
+
| a nameless entry | `TarParseError` |
|
|
120
|
+
| an absolute member path | `TarPathError`, from `tarEntryTree()` |
|
|
121
|
+
| a member path containing `..` | `TarPathError`, from `tarEntryTree()` |
|
|
122
|
+
| a hostile **directory** name, even though dropped | `TarPathError` — checked before the type filter |
|
|
123
|
+
| a negative or non-integer `strip` | `RangeError` |
|
|
124
|
+
|
|
125
|
+
**The header checksum at offset 148 is deliberately not enforced.** Writers disagree on whether
|
|
126
|
+
that field sums the bytes as signed or unsigned, so enforcing it rejects real archives; the octal
|
|
127
|
+
and bounds checks already stop a malformed header from being read. That is a decision, not an
|
|
128
|
+
omission.
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## 🚫 What It Deliberately Does Not Do
|
|
133
|
+
|
|
134
|
+
- **It does not stream the parse.** The archive is buffered whole, because a tar header carries no
|
|
135
|
+
back-pointer: an entry cannot be found without walking every block before it. For a contrib
|
|
136
|
+
module that is a few megabytes, which is the size this was built for.
|
|
137
|
+
- **It does not follow links.** Symlinks, hard links and device nodes are consumed and dropped.
|
|
138
|
+
Nothing in a module install should follow one.
|
|
139
|
+
- **It does not write files.** `tarEntryTree()` hands you a `Map`; where those bytes land is yours.
|
|
140
|
+
- **It does not create archives.** Extraction only.
|
|
141
|
+
|
|
142
|
+
It does handle the shapes real drupal.org tarballs contain: the ustar `prefix` split, GNU `L` long
|
|
143
|
+
names, pax `x`/`g` headers (skipped, though an `x` header's `path=` record renames the entry it
|
|
144
|
+
describes), and the v7 habit of writing a directory as typeflag `0` with a trailing slash.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## 🧪 Testing
|
|
149
|
+
|
|
150
|
+
```sh
|
|
151
|
+
bun run typecheck
|
|
152
|
+
bun run test # 87 assertions across 2 specs
|
|
153
|
+
bun run test:coverage
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
**87 passing**, at a measured **98.67% statements**, over hand-built archives rather than fixtures, so
|
|
157
|
+
every refusal above has a case that trips it. The fixtures are constructed in the spec because the
|
|
158
|
+
interesting inputs are the malformed ones, and a malformed file in `tests/` is indistinguishable from
|
|
159
|
+
a corrupted checkout.
|
|
160
|
+
|
|
161
|
+
The one uncovered line is `src/untar.ts:231`, the "does not advance" guard, and it is **provably
|
|
162
|
+
unreachable** rather than untested: `next` is `headerAt + 512 + ceil(size / 512) * 512`, so
|
|
163
|
+
`next <= headerAt` cannot hold while a header is a whole block. It stays because the failure it
|
|
164
|
+
guards against is a hang rather than an error.
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## 🔗 Related Repositories
|
|
169
|
+
|
|
170
|
+
| Repository | What it is |
|
|
171
|
+
| --------------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
|
172
|
+
| [`drupflare/worker`](https://github.com/drupflare/worker) | the consumer: Drupal 11 on Cloudflare Workers, which reads module tarballs |
|
|
173
|
+
| [`drupflare/cartridge`](https://github.com/drupflare/cartridge) | the host side of running a blocking interpreter in a Durable Object |
|
|
174
|
+
| [`drupflare/durabledb`](https://github.com/drupflare/durabledb) | the measured limits of Durable Object SQLite, plus the codec that survives them |
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## 📄 License
|
|
179
|
+
|
|
180
|
+
MIT (c) Gregory Mitchell 2026. See [LICENSE](LICENSE).
|
package/package.json
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@drupflare/untarl",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Dependency-free, workerd-safe tar and tar.gz extraction. No Node APIs.",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "./src/index.ts",
|
|
8
|
+
"types": "./src/index.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": "./src/index.ts",
|
|
11
|
+
"./package.json": "./package.json"
|
|
12
|
+
},
|
|
13
|
+
"publishConfig": {
|
|
14
|
+
"registry": "https://registry.npmjs.org",
|
|
15
|
+
"access": "public"
|
|
16
|
+
},
|
|
17
|
+
"sideEffects": false,
|
|
18
|
+
"files": [
|
|
19
|
+
"src",
|
|
20
|
+
"LICENSE",
|
|
21
|
+
"README.md"
|
|
22
|
+
],
|
|
23
|
+
"scripts": {
|
|
24
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
25
|
+
"test": "vitest run",
|
|
26
|
+
"test:watch": "vitest",
|
|
27
|
+
"test:coverage": "vitest run --coverage",
|
|
28
|
+
"docs:build": "typedoc",
|
|
29
|
+
"docs:serve": "typedoc --watch",
|
|
30
|
+
"prettier": "prettier --write .",
|
|
31
|
+
"prettier:check": "prettier --check .",
|
|
32
|
+
"prepare": "husky"
|
|
33
|
+
},
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"@cloudflare/workers-types": "^5.20260811.1",
|
|
36
|
+
"@vitest/coverage-v8": "^4.1.0",
|
|
37
|
+
"husky": "^9.1.7",
|
|
38
|
+
"lint-staged": "^17.3.0",
|
|
39
|
+
"prettier": "^3.9.6",
|
|
40
|
+
"prettier-plugin-organize-imports": "^4.3.0",
|
|
41
|
+
"prettier-plugin-sh": "^0.19.0",
|
|
42
|
+
"typedoc": "^0.28.20",
|
|
43
|
+
"typescript": "^5.9.3",
|
|
44
|
+
"vitest": "^4.1.0"
|
|
45
|
+
},
|
|
46
|
+
"author": "Gregory Mitchell <me@gmitch215.xyz>",
|
|
47
|
+
"homepage": "https://github.com/drupflare/untarl",
|
|
48
|
+
"repository": {
|
|
49
|
+
"type": "git",
|
|
50
|
+
"url": "git+https://github.com/drupflare/untarl.git"
|
|
51
|
+
},
|
|
52
|
+
"bugs": {
|
|
53
|
+
"url": "https://github.com/drupflare/untarl/issues"
|
|
54
|
+
},
|
|
55
|
+
"keywords": [
|
|
56
|
+
"tar",
|
|
57
|
+
"untar",
|
|
58
|
+
"tar-gz",
|
|
59
|
+
"gzip",
|
|
60
|
+
"cloudflare",
|
|
61
|
+
"workers",
|
|
62
|
+
"workerd",
|
|
63
|
+
"edge"
|
|
64
|
+
],
|
|
65
|
+
"lint-staged": {
|
|
66
|
+
"*.{ts,js,mjs,css,md,json,jsonc}": "prettier --write"
|
|
67
|
+
}
|
|
68
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './untar.js';
|
package/src/untar.ts
ADDED
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tar + gzip extraction, host-side in JavaScript.
|
|
3
|
+
*
|
|
4
|
+
* WHY IT LIVES HERE RATHER THAN IN PHP: installing a contrib module at runtime means reading a
|
|
5
|
+
* drupal.org `.tar.gz`, and this wasm build has neither ext-zip nor ext-phar. Adding either is a
|
|
6
|
+
* multi-hour Docker rebuild of the binary. `DecompressionStream('gzip')` is native in workerd and
|
|
7
|
+
* tar is nothing but 512-byte blocks with an octal header, so the whole job fits out here where
|
|
8
|
+
* awaiting is legal and no build flag has to change.
|
|
9
|
+
*
|
|
10
|
+
* A tarball off the network is UNTRUSTED INPUT. Every read is bounds-checked and throws a named
|
|
11
|
+
* error rather than reading past the buffer or looping, and no path becomes a filename until
|
|
12
|
+
* `tarEntryTree()` has refused the ones that escape. `parseTar()` stays faithful to the bytes;
|
|
13
|
+
* `tarEntryTree()` is the layer that decides what is safe to write.
|
|
14
|
+
*
|
|
15
|
+
* The header checksum at offset 148 is deliberately NOT enforced. Writers disagree on whether the
|
|
16
|
+
* field is summed as signed or unsigned bytes, so enforcing it rejects real archives, while the
|
|
17
|
+
* octal and bounds checks below already stop a malformed header from being read.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** What an entry is, after the several typeflags that mean the same thing are folded together. */
|
|
21
|
+
export type TarEntryType = 'file' | 'directory';
|
|
22
|
+
|
|
23
|
+
/** One archive member. `bytes` is an owned copy, so the archive buffer can be released. */
|
|
24
|
+
export interface TarEntry {
|
|
25
|
+
/** full path as the archive names it, ustar prefix and GNU long names already resolved */
|
|
26
|
+
name: string;
|
|
27
|
+
/** size in bytes */
|
|
28
|
+
size: number;
|
|
29
|
+
/** type of the entry */
|
|
30
|
+
type: TarEntryType;
|
|
31
|
+
/** permission bits from the octal `mode` field; 0 when the field is blank */
|
|
32
|
+
mode: number;
|
|
33
|
+
/** the entry's data */
|
|
34
|
+
bytes: Uint8Array;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A malformed, truncated, or out-of-range archive. Carries the byte offset when one is known. */
|
|
38
|
+
export class TarParseError extends Error {
|
|
39
|
+
/** The byte offset where the error occurred, if known. */
|
|
40
|
+
offset: number | undefined;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Creates a new parse error with a message and optional offset. The message is prefixed with
|
|
44
|
+
* "untar: " to distinguish it from other errors.
|
|
45
|
+
*
|
|
46
|
+
* @param message - The error message.
|
|
47
|
+
* @param offset - The byte offset where the error occurred, if known.
|
|
48
|
+
*/
|
|
49
|
+
constructor(message: string, offset?: number) {
|
|
50
|
+
super(`untar: ${message}`);
|
|
51
|
+
this.name = 'TarParseError';
|
|
52
|
+
this.offset = offset;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** A member whose path would escape the target directory. Never thrown by `parseTar()`. */
|
|
57
|
+
export class TarPathError extends Error {
|
|
58
|
+
/** The path that was refused. */
|
|
59
|
+
path: string;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Creates a new path error with a message and the refused path. The message is prefixed with
|
|
63
|
+
* "untar: " to distinguish it from other errors.
|
|
64
|
+
*
|
|
65
|
+
* @param path - The path that was refused.
|
|
66
|
+
* @param reason - The reason why the path was refused.
|
|
67
|
+
*/
|
|
68
|
+
constructor(path: string, reason: string) {
|
|
69
|
+
super(`untar: refused path ${JSON.stringify(path)}: ${reason}`);
|
|
70
|
+
this.name = 'TarPathError';
|
|
71
|
+
this.path = path;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const BLOCK = 512;
|
|
76
|
+
|
|
77
|
+
const NAME_AT = 0;
|
|
78
|
+
const NAME_LEN = 100;
|
|
79
|
+
const MODE_AT = 100;
|
|
80
|
+
const MODE_LEN = 8;
|
|
81
|
+
const SIZE_AT = 124;
|
|
82
|
+
const SIZE_LEN = 12;
|
|
83
|
+
const TYPE_AT = 156;
|
|
84
|
+
const PREFIX_AT = 345;
|
|
85
|
+
const PREFIX_LEN = 155;
|
|
86
|
+
|
|
87
|
+
const NUL = 0x00;
|
|
88
|
+
const SPACE = 0x20;
|
|
89
|
+
|
|
90
|
+
const TYPE_OLD_FILE = 0x00;
|
|
91
|
+
const TYPE_FILE = 0x30; // '0'
|
|
92
|
+
const TYPE_DIR = 0x35; // '5'
|
|
93
|
+
const TYPE_CONTIGUOUS = 0x37; // '7', a regular file everywhere that still emits it
|
|
94
|
+
const TYPE_PAX = 0x78; // 'x'
|
|
95
|
+
const TYPE_PAX_GLOBAL = 0x67; // 'g'
|
|
96
|
+
const TYPE_LONGNAME = 0x4c; // 'L'
|
|
97
|
+
const TYPE_LONGLINK = 0x4b; // 'K'
|
|
98
|
+
|
|
99
|
+
const utf8 = new TextDecoder();
|
|
100
|
+
|
|
101
|
+
/** Every byte read goes through here, so a short buffer throws instead of yielding undefined. */
|
|
102
|
+
function byteAt(bytes: Uint8Array, index: number): number {
|
|
103
|
+
const b = bytes[index];
|
|
104
|
+
if (b === undefined)
|
|
105
|
+
throw new TarParseError(`read past end of archive at byte ${index}`, index);
|
|
106
|
+
return b;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** returns false at the first non-zero byte, which for a real header is byte 0 */
|
|
110
|
+
function isZeroBlock(bytes: Uint8Array, at: number): boolean {
|
|
111
|
+
for (let i = at; i < at + BLOCK; i++) if (byteAt(bytes, i) !== NUL) return false;
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Reads a NUL-terminated header field. Names are UTF-8 in every archive that matters. */
|
|
116
|
+
function readField(bytes: Uint8Array, at: number, len: number): string {
|
|
117
|
+
let end = at;
|
|
118
|
+
while (end < at + len && byteAt(bytes, end) !== NUL) end++;
|
|
119
|
+
return utf8.decode(bytes.subarray(at, end));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Reads a NUL-terminated string out of an entry body (GNU long name). */
|
|
123
|
+
function readCString(body: Uint8Array): string {
|
|
124
|
+
let end = 0;
|
|
125
|
+
while (end < body.length && byteAt(body, end) !== NUL) end++;
|
|
126
|
+
return utf8.decode(body.subarray(0, end));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Parses one octal header field.
|
|
131
|
+
*
|
|
132
|
+
* Padding is NUL or space and may come before or after the digits; anything else is garbage and
|
|
133
|
+
* throws, which is what stops a random buffer from producing a plausible size. A blank field is 0.
|
|
134
|
+
*/
|
|
135
|
+
function parseOctal(bytes: Uint8Array, at: number, len: number, field: string): number {
|
|
136
|
+
// GNU stores a size above 8 GiB as base-256 with the high bit set; nothing that large can be
|
|
137
|
+
// held in a Worker's memory anyway, so refuse it loudly instead of misreading it as octal
|
|
138
|
+
if ((byteAt(bytes, at) & 0x80) !== 0) {
|
|
139
|
+
throw new TarParseError(
|
|
140
|
+
`${field} field uses GNU base-256 encoding, which is unsupported`,
|
|
141
|
+
at
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
let value = 0;
|
|
146
|
+
let digits = 0;
|
|
147
|
+
let terminated = false;
|
|
148
|
+
|
|
149
|
+
for (let i = at; i < at + len; i++) {
|
|
150
|
+
const b = byteAt(bytes, i);
|
|
151
|
+
if (b === NUL || b === SPACE) {
|
|
152
|
+
terminated = digits > 0;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
if (terminated) {
|
|
156
|
+
throw new TarParseError(`${field} field has junk after its terminator`, at);
|
|
157
|
+
}
|
|
158
|
+
if (b < 0x30 || b > 0x37) {
|
|
159
|
+
throw new TarParseError(`${field} field is not octal (byte 0x${b.toString(16)})`, at);
|
|
160
|
+
}
|
|
161
|
+
value = value * 8 + (b - 0x30);
|
|
162
|
+
digits++;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return value;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Pulls the `path` override out of a pax extended header body.
|
|
170
|
+
*
|
|
171
|
+
* Records are `"<len> <key>=<value>\n"` where `len` counts itself, so the length prefix is what
|
|
172
|
+
* makes a value containing a newline unambiguous. Anything unparseable ends the scan; a pax header
|
|
173
|
+
* we cannot read is not a reason to fail an otherwise good archive.
|
|
174
|
+
*/
|
|
175
|
+
function paxPath(body: Uint8Array): string | null {
|
|
176
|
+
let found: string | null = null;
|
|
177
|
+
let i = 0;
|
|
178
|
+
|
|
179
|
+
while (i < body.length) {
|
|
180
|
+
let sp = i;
|
|
181
|
+
while (sp < body.length && byteAt(body, sp) !== SPACE) sp++;
|
|
182
|
+
const len = Number(utf8.decode(body.subarray(i, sp)));
|
|
183
|
+
if (!Number.isInteger(len) || len <= 0 || i + len > body.length || sp + 1 >= i + len) break;
|
|
184
|
+
|
|
185
|
+
// i + len - 1 drops the trailing newline
|
|
186
|
+
const record = utf8.decode(body.subarray(sp + 1, i + len - 1));
|
|
187
|
+
const eq = record.indexOf('=');
|
|
188
|
+
if (eq > 0 && record.slice(0, eq) === 'path') found = record.slice(eq + 1);
|
|
189
|
+
i += len;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return found;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Parses a POSIX/ustar tar archive.
|
|
197
|
+
*
|
|
198
|
+
* Handles the shapes real drupal.org tarballs contain: the ustar `prefix` split, GNU `L` long
|
|
199
|
+
* names, pax `x`/`g` headers (skipped, though an `x` header's `path=` record renames the entry it
|
|
200
|
+
* describes), and the v7 habit of writing a directory as typeflag `0` with a trailing slash.
|
|
201
|
+
* Symlinks, hard links and device nodes are consumed and dropped -- nothing in a module install
|
|
202
|
+
* should follow one.
|
|
203
|
+
*
|
|
204
|
+
* @throws {TarParseError} on a truncated header or body, a size that overruns the buffer, a
|
|
205
|
+
* non-octal size or mode, or a nameless entry.
|
|
206
|
+
*/
|
|
207
|
+
export function parseTar(bytes: Uint8Array): TarEntry[] {
|
|
208
|
+
const entries: TarEntry[] = [];
|
|
209
|
+
let offset = 0;
|
|
210
|
+
let zeroBlocks = 0;
|
|
211
|
+
// set by an 'L' body or a pax `path=` record, consumed by the next real entry
|
|
212
|
+
let pendingName: string | null = null;
|
|
213
|
+
|
|
214
|
+
while (offset < bytes.length) {
|
|
215
|
+
const remaining = bytes.length - offset;
|
|
216
|
+
if (remaining < BLOCK) {
|
|
217
|
+
// tar pads the archive out to whole blocks, so a partial one is a cut-off download; after
|
|
218
|
+
// the end-of-archive marker it is only padding and can be ignored
|
|
219
|
+
if (zeroBlocks > 0) break;
|
|
220
|
+
throw new TarParseError(
|
|
221
|
+
`truncated header: ${remaining} of ${BLOCK} bytes at offset ${offset}`,
|
|
222
|
+
offset
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (isZeroBlock(bytes, offset)) {
|
|
227
|
+
offset += BLOCK;
|
|
228
|
+
zeroBlocks++;
|
|
229
|
+
if (zeroBlocks >= 2) break;
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
zeroBlocks = 0;
|
|
233
|
+
|
|
234
|
+
const headerAt = offset;
|
|
235
|
+
const rawName = readField(bytes, offset + NAME_AT, NAME_LEN);
|
|
236
|
+
const prefix = readField(bytes, offset + PREFIX_AT, PREFIX_LEN);
|
|
237
|
+
const mode = parseOctal(bytes, offset + MODE_AT, MODE_LEN, 'mode');
|
|
238
|
+
const size = parseOctal(bytes, offset + SIZE_AT, SIZE_LEN, 'size');
|
|
239
|
+
const flag = byteAt(bytes, offset + TYPE_AT);
|
|
240
|
+
|
|
241
|
+
const dataAt = offset + BLOCK;
|
|
242
|
+
const available = bytes.length - dataAt;
|
|
243
|
+
if (size > available) {
|
|
244
|
+
throw new TarParseError(
|
|
245
|
+
`entry ${JSON.stringify(pendingName ?? rawName)} claims ${size} bytes but ${available} remain`,
|
|
246
|
+
headerAt
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const next = dataAt + Math.ceil(size / BLOCK) * BLOCK;
|
|
251
|
+
// unreachable while a header is a whole block, asserted because the cost of being wrong is a
|
|
252
|
+
// hang rather than an error
|
|
253
|
+
if (next <= headerAt) {
|
|
254
|
+
throw new TarParseError(`entry ${JSON.stringify(rawName)} does not advance`, headerAt);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const body = bytes.subarray(dataAt, dataAt + size);
|
|
258
|
+
offset = next;
|
|
259
|
+
|
|
260
|
+
if (flag === TYPE_LONGNAME) {
|
|
261
|
+
pendingName = readCString(body);
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (flag === TYPE_LONGLINK) continue;
|
|
265
|
+
if (flag === TYPE_PAX || flag === TYPE_PAX_GLOBAL) {
|
|
266
|
+
// a global header applies to the rest of the archive, which is a claim about metadata we
|
|
267
|
+
// do not carry, so only the per-file form can rename anything
|
|
268
|
+
const path = flag === TYPE_PAX ? paxPath(body) : null;
|
|
269
|
+
if (path !== null) pendingName = path;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const name = pendingName ?? (prefix === '' ? rawName : `${prefix}/${rawName}`);
|
|
274
|
+
pendingName = null;
|
|
275
|
+
|
|
276
|
+
const isFile = flag === TYPE_OLD_FILE || flag === TYPE_FILE || flag === TYPE_CONTIGUOUS;
|
|
277
|
+
if (!isFile && flag !== TYPE_DIR) continue;
|
|
278
|
+
|
|
279
|
+
if (name === '') {
|
|
280
|
+
throw new TarParseError(`entry at offset ${headerAt} has no name`, headerAt);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// v7 tar had no directory typeflag and marked one with a trailing slash instead
|
|
284
|
+
const type: TarEntryType = flag === TYPE_DIR || name.endsWith('/') ? 'directory' : 'file';
|
|
285
|
+
|
|
286
|
+
entries.push({
|
|
287
|
+
name,
|
|
288
|
+
size,
|
|
289
|
+
type,
|
|
290
|
+
mode,
|
|
291
|
+
bytes: type === 'directory' ? new Uint8Array(0) : body.slice()
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
return entries;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Gunzips a stream and parses the tar inside it.
|
|
300
|
+
*
|
|
301
|
+
* The archive is buffered whole because a tar header carries no back-pointer, so an entry cannot
|
|
302
|
+
* be found without walking every block before it. For a contrib module that is a few megabytes.
|
|
303
|
+
*
|
|
304
|
+
* @throws {TarParseError} for the same reasons as `parseTar()`; a corrupt gzip member rejects from
|
|
305
|
+
* `DecompressionStream` itself.
|
|
306
|
+
*/
|
|
307
|
+
export async function untarGzip(stream: ReadableStream<Uint8Array>): Promise<TarEntry[]> {
|
|
308
|
+
const inflated = stream.pipeThrough(new DecompressionStream('gzip'));
|
|
309
|
+
const buffer = await new Response(inflated).arrayBuffer();
|
|
310
|
+
return parseTar(new Uint8Array(buffer));
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Splits a member path, refusing the two forms that escape the target directory. */
|
|
314
|
+
function safeParts(name: string): string[] {
|
|
315
|
+
if (name.startsWith('/')) throw new TarPathError(name, 'absolute paths are refused');
|
|
316
|
+
const parts = name.split('/').filter((p) => p !== '' && p !== '.');
|
|
317
|
+
for (const p of parts) {
|
|
318
|
+
if (p === '..') throw new TarPathError(name, '".." would escape the target directory');
|
|
319
|
+
}
|
|
320
|
+
return parts;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Flattens entries into the file tree to write, with the leading `strip` path components removed.
|
|
325
|
+
*
|
|
326
|
+
* `strip = 1` is the useful default because a drupal.org tarball wraps everything in one
|
|
327
|
+
* `modulename/` directory. Directories are dropped -- a file's own path implies them -- and so is
|
|
328
|
+
* any file left with nothing after the strip, which is what tar's own `--strip-components` does.
|
|
329
|
+
* A later entry with the same resulting path wins, also matching tar.
|
|
330
|
+
*
|
|
331
|
+
* @throws {TarPathError} on a member path that is absolute or contains a `..` component.
|
|
332
|
+
*/
|
|
333
|
+
export function tarEntryTree(entries: TarEntry[], strip = 1): Map<string, Uint8Array> {
|
|
334
|
+
if (!Number.isInteger(strip) || strip < 0) {
|
|
335
|
+
throw new RangeError(`untar: strip must be a non-negative integer, got ${strip}`);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const out = new Map<string, Uint8Array>();
|
|
339
|
+
for (const entry of entries) {
|
|
340
|
+
// checked for every entry, not just files, so a hostile directory name still fails
|
|
341
|
+
const parts = safeParts(entry.name);
|
|
342
|
+
if (entry.type !== 'file') continue;
|
|
343
|
+
if (parts.length <= strip) continue;
|
|
344
|
+
out.set(parts.slice(strip).join('/'), entry.bytes);
|
|
345
|
+
}
|
|
346
|
+
return out;
|
|
347
|
+
}
|