@hoardodile/host 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +73 -0
- package/dist/chunks/worker-entry.mjs +422 -0
- package/dist/container-fKxefO7p.d.ts +58 -0
- package/dist/contract/index.d.ts +49 -0
- package/dist/contract/index.js +187 -0
- package/dist/contract/index.js.map +1 -0
- package/dist/hoard/index.d.ts +754 -0
- package/dist/hoard/index.js +1939 -0
- package/dist/hoard/index.js.map +1 -0
- package/dist/image-Dx2Be3Bj.d.ts +168 -0
- package/dist/index.d.ts +788 -0
- package/dist/index.js +3311 -0
- package/dist/index.js.map +1 -0
- package/dist/media/index.d.ts +96 -0
- package/dist/media/index.js +152 -0
- package/dist/media/index.js.map +1 -0
- package/dist/probe/index.d.ts +113 -0
- package/dist/probe/index.js +454 -0
- package/dist/probe/index.js.map +1 -0
- package/dist/probe-cache-BkRI7HEK.d.ts +25 -0
- package/dist/render/index.d.ts +125 -0
- package/dist/render/index.js +430 -0
- package/dist/render/index.js.map +1 -0
- package/package.json +103 -0
- package/src/__testutils__/zip-fixtures.ts +139 -0
- package/src/activation.ts +116 -0
- package/src/api-types.ts +60 -0
- package/src/api.test.ts +564 -0
- package/src/api.ts +501 -0
- package/src/archive/7z.test.ts +222 -0
- package/src/archive/7z.ts +261 -0
- package/src/archive/archive-ops.test.ts +654 -0
- package/src/archive/extract-archive.test.ts +237 -0
- package/src/archive/extract-archive.ts +534 -0
- package/src/archive/extract.ts +693 -0
- package/src/archive/format.ts +49 -0
- package/src/archive/index.ts +116 -0
- package/src/archive/listing.ts +121 -0
- package/src/archive/materialize.ts +88 -0
- package/src/archive/name-decode.test.ts +34 -0
- package/src/archive/name-decode.ts +287 -0
- package/src/archive/nested-cd-cache.ts +55 -0
- package/src/archive/nested-entry.test.ts +194 -0
- package/src/archive/nested-entry.ts +254 -0
- package/src/archive/pack.ts +40 -0
- package/src/archive/zip-entries.test.ts +56 -0
- package/src/archive/zip-entries.ts +336 -0
- package/src/capability-guard.ts +73 -0
- package/src/container.ts +62 -0
- package/src/contract/container-fixture.ts +74 -0
- package/src/contract/index.ts +10 -0
- package/src/contract/suite.test.ts +101 -0
- package/src/contract/suite.ts +205 -0
- package/src/define-plugin.test.ts +53 -0
- package/src/define-plugin.ts +12 -0
- package/src/dev-runner.test.ts +98 -0
- package/src/dev-runner.ts +73 -0
- package/src/directory-api.test.ts +229 -0
- package/src/directory-api.ts +42 -0
- package/src/directory-container.ts +141 -0
- package/src/discovery.ts +295 -0
- package/src/errors.ts +74 -0
- package/src/fixtures.ts +8 -0
- package/src/hash.test.ts +301 -0
- package/src/hash.ts +167 -0
- package/src/hoard/dir-size.ts +58 -0
- package/src/hoard/index.ts +84 -0
- package/src/hoard/migrate-script.test.ts +539 -0
- package/src/hoard/order-manifest.test.ts +115 -0
- package/src/hoard/order-manifest.ts +131 -0
- package/src/hoard/paths.test.ts +226 -0
- package/src/hoard/paths.ts +569 -0
- package/src/hoard/plugin-vault.test.ts +160 -0
- package/src/hoard/plugin-vault.ts +233 -0
- package/src/hoard/sanitize.test.ts +149 -0
- package/src/hoard/sanitize.ts +230 -0
- package/src/hoard/staging-dir.ts +133 -0
- package/src/hoard/version.test.ts +285 -0
- package/src/hoard/version.ts +216 -0
- package/src/hoard/versioned-folder-ops.ts +152 -0
- package/src/hoard/write-versioned.test.ts +67 -0
- package/src/hoard/write-versioned.ts +39 -0
- package/src/hooks.test.ts +528 -0
- package/src/hooks.ts +428 -0
- package/src/index.ts +135 -0
- package/src/loader.test.ts +442 -0
- package/src/loader.ts +213 -0
- package/src/media/index.ts +16 -0
- package/src/media/render-cache.ts +27 -0
- package/src/media/seekable.ts +35 -0
- package/src/media/thumb-input.test.ts +109 -0
- package/src/media/thumb-input.ts +194 -0
- package/src/nested-view.test.ts +223 -0
- package/src/nested-view.ts +239 -0
- package/src/probe/audio.test.ts +135 -0
- package/src/probe/audio.ts +140 -0
- package/src/probe/av.ts +84 -0
- package/src/probe/ffprobe.ts +197 -0
- package/src/probe/image.test.ts +251 -0
- package/src/probe/image.ts +336 -0
- package/src/probe/index.ts +54 -0
- package/src/probe/sniff.test.ts +72 -0
- package/src/probe/sniff.ts +67 -0
- package/src/probe/video.test.ts +41 -0
- package/src/probe/video.ts +116 -0
- package/src/probe-cache.test.ts +89 -0
- package/src/probe-cache.ts +65 -0
- package/src/render/ffmpeg.ts +78 -0
- package/src/render/index.ts +34 -0
- package/src/render/pipeline.test.ts +187 -0
- package/src/render/pipeline.ts +479 -0
- package/src/sandbox/call-timers.ts +68 -0
- package/src/sandbox/fixtures/api-error-plugin.mjs +11 -0
- package/src/sandbox/fixtures/asset-plugin.mjs +13 -0
- package/src/sandbox/fixtures/chatty-plugin.mjs +14 -0
- package/src/sandbox/fixtures/container-plugin.mjs +8 -0
- package/src/sandbox/fixtures/context-plugin.mjs +19 -0
- package/src/sandbox/fixtures/crash-plugin.mjs +6 -0
- package/src/sandbox/fixtures/echo-plugin.mjs +17 -0
- package/src/sandbox/fixtures/exit-plugin.mjs +6 -0
- package/src/sandbox/fixtures/fetch-plugin.mjs +8 -0
- package/src/sandbox/fixtures/flood-api-plugin.mjs +8 -0
- package/src/sandbox/fixtures/flood-log-plugin.mjs +8 -0
- package/src/sandbox/fixtures/hostile-dynamic-plugin.mjs +9 -0
- package/src/sandbox/fixtures/hostile-plugin.mjs +13 -0
- package/src/sandbox/fixtures/huge-result-plugin.mjs +8 -0
- package/src/sandbox/fixtures/logging-plugin.mjs +9 -0
- package/src/sandbox/fixtures/range-plugin.mjs +9 -0
- package/src/sandbox/fixtures/slow-api-plugin.mjs +11 -0
- package/src/sandbox/fixtures/spin-plugin.mjs +9 -0
- package/src/sandbox/fixtures/stuck-plugin.mjs +12 -0
- package/src/sandbox/fixtures/thrower-plugin.mjs +7 -0
- package/src/sandbox/host.test.ts +619 -0
- package/src/sandbox/host.ts +933 -0
- package/src/sandbox/host.unit.test.ts +330 -0
- package/src/sandbox/protocol.test.ts +85 -0
- package/src/sandbox/protocol.ts +147 -0
- package/src/sandbox/sandboxed-plugin.ts +30 -0
- package/src/sandbox/worker-entry.mjs +422 -0
- package/src/seed.test.ts +102 -0
- package/src/seed.ts +160 -0
- package/src/settings-store.ts +26 -0
- package/src/types.ts +21 -0
|
@@ -0,0 +1,693 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto"
|
|
2
|
+
import { createWriteStream, existsSync } from "node:fs"
|
|
3
|
+
import {
|
|
4
|
+
chmod,
|
|
5
|
+
lstat,
|
|
6
|
+
mkdir,
|
|
7
|
+
readdir,
|
|
8
|
+
rename,
|
|
9
|
+
rm,
|
|
10
|
+
writeFile,
|
|
11
|
+
} from "node:fs/promises"
|
|
12
|
+
import { tmpdir } from "node:os"
|
|
13
|
+
import { dirname, join, normalize, resolve, sep } from "node:path"
|
|
14
|
+
import { Transform } from "node:stream"
|
|
15
|
+
import { pipeline } from "node:stream/promises"
|
|
16
|
+
import yauzl, { type Entry, type ZipFile } from "yauzl"
|
|
17
|
+
import { invalid } from "../errors.ts"
|
|
18
|
+
import {
|
|
19
|
+
extractSevenZipInto,
|
|
20
|
+
listSevenZipEntries,
|
|
21
|
+
resolveSevenZipPath,
|
|
22
|
+
} from "./7z.ts"
|
|
23
|
+
import {
|
|
24
|
+
type ContainerFormat,
|
|
25
|
+
SNIFF_WINDOW_BYTES,
|
|
26
|
+
sniffContainerFormat,
|
|
27
|
+
} from "./format.ts"
|
|
28
|
+
import { listingBudgetExceeded } from "./listing.ts"
|
|
29
|
+
import { decodeLegacyZipName } from "./name-decode.ts"
|
|
30
|
+
import { listZipEntries, normalizeZipError } from "./zip-entries.ts"
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Whole-archive extraction. Every supported format (zip, tar, 7z, rar,
|
|
34
|
+
* xz, gzip) extracts through 7-Zip with listing-validated budgets; when
|
|
35
|
+
* the optional 7-Zip binary is absent, zip falls back to a yauzl
|
|
36
|
+
* streaming path with a per-byte budget counter. Used by plugin
|
|
37
|
+
* installation, folder-import extraction and the plugin extract API —
|
|
38
|
+
* the workflows that need a directory layout.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Extraction progress reporter for {@link extractArchiveInto}. The
|
|
43
|
+
* 7-Zip path reports the two phases only (listing pre-check, done);
|
|
44
|
+
* the yauzl fallback reports per-entry events.
|
|
45
|
+
*/
|
|
46
|
+
export type ZipExtractReporter = (event: ZipExtractReport) => void
|
|
47
|
+
|
|
48
|
+
export type ZipExtractReport =
|
|
49
|
+
| {
|
|
50
|
+
readonly phase: "started"
|
|
51
|
+
readonly totalEntries: number
|
|
52
|
+
readonly totalBytes: number
|
|
53
|
+
}
|
|
54
|
+
| {
|
|
55
|
+
readonly phase: "entry"
|
|
56
|
+
readonly entriesDone: number
|
|
57
|
+
readonly totalEntries: number
|
|
58
|
+
readonly bytesDone: number
|
|
59
|
+
readonly totalBytes: number
|
|
60
|
+
}
|
|
61
|
+
| { readonly phase: "done" }
|
|
62
|
+
|
|
63
|
+
export type ExtractArchiveOptions = {
|
|
64
|
+
/** Cumulative uncompressed byte budget for the whole archive. */
|
|
65
|
+
readonly maxBytes: number
|
|
66
|
+
/** Optional entry-count budget (enforced on the listing). */
|
|
67
|
+
readonly maxEntries?: number
|
|
68
|
+
readonly onProgress?: ZipExtractReporter
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Stream an archive into `destDir`, format-agnostic: every supported
|
|
73
|
+
* format (zip/tar/7z/rar/xz/gzip) extracts through 7-Zip after
|
|
74
|
+
* validating budgets, encryption and paths from its listing, then
|
|
75
|
+
* re-verifies the extracted tree (size + symlink scan). Without a
|
|
76
|
+
* 7-Zip binary, zip extracts through yauzl with a per-byte budget
|
|
77
|
+
* counter; the other formats error with "7-Zip is not installed".
|
|
78
|
+
* Used by plugin installation and folder-import extraction (the
|
|
79
|
+
* workflows that need a directory layout).
|
|
80
|
+
*
|
|
81
|
+
* Refuses any entry whose normalised path escapes `destDir` (zip-slip)
|
|
82
|
+
* or contains an absolute path / drive letter. Defends against zip
|
|
83
|
+
* bombs via `maxBytes` (and `maxEntries` when provided).
|
|
84
|
+
*
|
|
85
|
+
* @throws DomainError VALIDATION when the archive is malformed,
|
|
86
|
+
* contains an unsafe entry, exceeds the budgets, is encrypted, or is
|
|
87
|
+
* not a supported archive at all.
|
|
88
|
+
*/
|
|
89
|
+
export async function extractArchiveInto(
|
|
90
|
+
source: NodeJS.ReadableStream,
|
|
91
|
+
destDir: string,
|
|
92
|
+
opts: ExtractArchiveOptions,
|
|
93
|
+
): Promise<void> {
|
|
94
|
+
const buffer = await readToBuffer(source)
|
|
95
|
+
const format = sniffContainerFormat(buffer.subarray(0, SNIFF_WINDOW_BYTES))
|
|
96
|
+
if (format === undefined) {
|
|
97
|
+
throw invalid(
|
|
98
|
+
"resource.archive_open_failed",
|
|
99
|
+
"not a supported archive (zip/tar/7z/rar/xz/gzip)",
|
|
100
|
+
{},
|
|
101
|
+
)
|
|
102
|
+
}
|
|
103
|
+
if (format === "zip" && resolveSevenZipPath() === undefined) {
|
|
104
|
+
return extractZipBuffer(buffer, destDir, opts)
|
|
105
|
+
}
|
|
106
|
+
return extractViaSevenZip(buffer, destDir, opts, format)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Zip branch when 7-Zip is absent: yauzl streaming with per-byte budget
|
|
111
|
+
* and entry progress. Kept so zip imports work on installs where the
|
|
112
|
+
* optional binary failed to download.
|
|
113
|
+
*/
|
|
114
|
+
async function extractZipBuffer(
|
|
115
|
+
buffer: Buffer,
|
|
116
|
+
destDir: string,
|
|
117
|
+
opts: ExtractArchiveOptions,
|
|
118
|
+
): Promise<void> {
|
|
119
|
+
const { maxBytes, maxEntries, onProgress } = opts
|
|
120
|
+
const zipfile = await openZipFromBuffer(buffer)
|
|
121
|
+
if (maxEntries !== undefined && zipfile.entryCount > maxEntries) {
|
|
122
|
+
zipfile.close()
|
|
123
|
+
throw invalid(
|
|
124
|
+
"resource.archive_too_large",
|
|
125
|
+
`archive has ${zipfile.entryCount} entries, exceeding the limit of ${maxEntries}`,
|
|
126
|
+
{ maxEntries },
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
const root = resolve(destDir)
|
|
130
|
+
const budget = { remaining: maxBytes, max: maxBytes }
|
|
131
|
+
const totalEntries = zipfile.entryCount
|
|
132
|
+
const counters = { entriesDone: 0, bytesDone: 0, totalBytes: 0 }
|
|
133
|
+
if (onProgress !== undefined) {
|
|
134
|
+
onProgress({ phase: "started", totalEntries, totalBytes: 0 })
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
await new Promise<void>((resolveDone, rejectDone) => {
|
|
138
|
+
zipfile.on("error", (err: unknown) => rejectDone(normalizeZipError(err)))
|
|
139
|
+
zipfile.on("end", resolveDone)
|
|
140
|
+
zipfile.on("entry", (entry: Entry) => {
|
|
141
|
+
handleExtractEntry(zipfile, entry, root, budget).then(() => {
|
|
142
|
+
counters.entriesDone += 1
|
|
143
|
+
counters.bytesDone += entry.uncompressedSize
|
|
144
|
+
counters.totalBytes += entry.uncompressedSize
|
|
145
|
+
if (onProgress !== undefined) {
|
|
146
|
+
onProgress({
|
|
147
|
+
phase: "entry",
|
|
148
|
+
entriesDone: counters.entriesDone,
|
|
149
|
+
totalEntries,
|
|
150
|
+
bytesDone: counters.bytesDone,
|
|
151
|
+
totalBytes: counters.totalBytes,
|
|
152
|
+
})
|
|
153
|
+
}
|
|
154
|
+
zipfile.readEntry()
|
|
155
|
+
}, rejectDone)
|
|
156
|
+
})
|
|
157
|
+
zipfile.readEntry()
|
|
158
|
+
})
|
|
159
|
+
} finally {
|
|
160
|
+
zipfile.close()
|
|
161
|
+
}
|
|
162
|
+
if (onProgress !== undefined) {
|
|
163
|
+
onProgress({ phase: "done" })
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* 7-Zip branch for every supported format: write the bytes to a temp
|
|
169
|
+
* file (7-Zip needs a real path), validate budgets, encryption and
|
|
170
|
+
* entry paths from the listing, extract in one pass, then verify the
|
|
171
|
+
* extracted tree (size re-check + symlink refusal).
|
|
172
|
+
*/
|
|
173
|
+
async function extractViaSevenZip(
|
|
174
|
+
buffer: Buffer,
|
|
175
|
+
destDir: string,
|
|
176
|
+
opts: ExtractArchiveOptions,
|
|
177
|
+
format: ContainerFormat,
|
|
178
|
+
): Promise<void> {
|
|
179
|
+
const { maxBytes, maxEntries, onProgress } = opts
|
|
180
|
+
const tempPath = join(
|
|
181
|
+
tmpdir(),
|
|
182
|
+
`hoardodile-7z-${process.pid}-${randomUUID().slice(0, 8)}`,
|
|
183
|
+
)
|
|
184
|
+
await writeFile(tempPath, buffer)
|
|
185
|
+
try {
|
|
186
|
+
const entries = await listSevenZipEntries(tempPath)
|
|
187
|
+
const files = entries.filter((e) => !e.folder)
|
|
188
|
+
const exceeded = listingBudgetExceeded(files, { maxBytes, maxEntries })
|
|
189
|
+
if (exceeded?.kind === "bytes") {
|
|
190
|
+
throw invalid(
|
|
191
|
+
"resource.archive_too_large",
|
|
192
|
+
`archive extracts to more than ${maxBytes} bytes`,
|
|
193
|
+
{ maxExtractedBytes: maxBytes },
|
|
194
|
+
)
|
|
195
|
+
}
|
|
196
|
+
if (exceeded?.kind === "entries") {
|
|
197
|
+
throw invalid(
|
|
198
|
+
"resource.archive_too_large",
|
|
199
|
+
`archive has ${files.length} entries, exceeding the limit of ${maxEntries}`,
|
|
200
|
+
{ maxEntries },
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
if (entries.some((e) => e.encrypted)) {
|
|
204
|
+
throw invalid(
|
|
205
|
+
"resource.archive_invalid_entry",
|
|
206
|
+
"archive is password-protected — encrypted archives are not supported",
|
|
207
|
+
{},
|
|
208
|
+
)
|
|
209
|
+
}
|
|
210
|
+
for (const entry of files) assertSafeEntryPath(entry.name)
|
|
211
|
+
const root = resolve(destDir)
|
|
212
|
+
await mkdir(root, { recursive: true })
|
|
213
|
+
const totalBytes = files.reduce((acc, e) => acc + e.sizeBytes, 0)
|
|
214
|
+
onProgress?.({ phase: "started", totalEntries: files.length, totalBytes })
|
|
215
|
+
await extractSevenZipInto(tempPath, root)
|
|
216
|
+
// 7-Zip writes legacy zip names verbatim on POSIX and restores
|
|
217
|
+
// entries' mode bits, which can strip the app's own access; fix
|
|
218
|
+
// both up before the tree is re-walked or probed below. The
|
|
219
|
+
// decoded names of the archive itself are the ground truth the
|
|
220
|
+
// legacy-rename pass matches against (see `renameLegacyZipNames`)
|
|
221
|
+
// — 7-Zip's text listing loses the invalid bytes on POSIX.
|
|
222
|
+
const decoded =
|
|
223
|
+
format === "zip"
|
|
224
|
+
? await decodeZipNames(
|
|
225
|
+
tempPath,
|
|
226
|
+
files.map((e) => ({ name: e.name, sizeBytes: e.sizeBytes })),
|
|
227
|
+
)
|
|
228
|
+
: undefined
|
|
229
|
+
await normalizeExtractedTree(root, {
|
|
230
|
+
legacyZipNames: format === "zip",
|
|
231
|
+
expectedNames: decoded?.paths,
|
|
232
|
+
})
|
|
233
|
+
// Post-hoc bomb re-check: the listing sizes are advisory; count
|
|
234
|
+
// what actually landed (and refuse symlinks — 7-Zip creates them
|
|
235
|
+
// from zip/tar entries carrying unix link modes).
|
|
236
|
+
await assertExtractedTree(root, maxBytes)
|
|
237
|
+
onProgress?.({ phase: "done" })
|
|
238
|
+
} finally {
|
|
239
|
+
await rm(tempPath, { force: true }).catch(() => {})
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Walk an extracted tree verifying that every entry is a regular file
|
|
245
|
+
* or directory (symlinks and special files are refused outright — a
|
|
246
|
+
* malicious archive could otherwise plant a link out of the extraction
|
|
247
|
+
* root) and that the total size stays within `maxBytes`.
|
|
248
|
+
*/
|
|
249
|
+
export async function assertExtractedTree(
|
|
250
|
+
root: string,
|
|
251
|
+
maxBytes: number,
|
|
252
|
+
): Promise<void> {
|
|
253
|
+
let total = 0
|
|
254
|
+
async function walk(dir: string): Promise<void> {
|
|
255
|
+
const entries = await readdir(dir, { withFileTypes: true })
|
|
256
|
+
for (const entry of entries) {
|
|
257
|
+
const full = join(dir, entry.name)
|
|
258
|
+
// lstat, not dirent: on Windows, junction points (reparse
|
|
259
|
+
// tags) are only reported as links by lstat.
|
|
260
|
+
const info = await lstat(full)
|
|
261
|
+
if (info.isSymbolicLink()) {
|
|
262
|
+
throw invalid(
|
|
263
|
+
"resource.archive_invalid_entry",
|
|
264
|
+
`extracted entry is a symlink: ${entry.name}`,
|
|
265
|
+
{ name: entry.name },
|
|
266
|
+
)
|
|
267
|
+
}
|
|
268
|
+
if (info.isDirectory()) {
|
|
269
|
+
await walk(full)
|
|
270
|
+
continue
|
|
271
|
+
}
|
|
272
|
+
if (!info.isFile()) {
|
|
273
|
+
throw invalid(
|
|
274
|
+
"resource.archive_invalid_entry",
|
|
275
|
+
`extracted entry is not a regular file: ${entry.name}`,
|
|
276
|
+
{ name: entry.name },
|
|
277
|
+
)
|
|
278
|
+
}
|
|
279
|
+
total += info.size
|
|
280
|
+
if (total > maxBytes) {
|
|
281
|
+
throw invalid(
|
|
282
|
+
"resource.archive_too_large",
|
|
283
|
+
`archive extracts to more than ${maxBytes} bytes`,
|
|
284
|
+
{ maxExtractedBytes: maxBytes },
|
|
285
|
+
)
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
await walk(root)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Post-extraction hygiene for a 7-Zip materialized tree, run before the
|
|
294
|
+
* tree is re-walked by {@link assertExtractedTree} or probed for the
|
|
295
|
+
* manifest. For zip archives, entries whose on-disk names are not valid
|
|
296
|
+
* UTF-8 are renamed to their cp437-decoded form (7-Zip writes the raw
|
|
297
|
+
* bytes verbatim on POSIX, and macOS stores those bytes `%XX`-escaped —
|
|
298
|
+
* see {@link renameLegacyZipNames} for both shapes); every entry gets
|
|
299
|
+
* owner permissions OR'd in so the app can always read, serve and clean
|
|
300
|
+
* up what it extracted (zip/tar entries can carry mode bits that strip
|
|
301
|
+
* owner access). Never follows symlinks — refusal stays with
|
|
302
|
+
* {@link assertExtractedTree}. Windows is unaffected: names are already
|
|
303
|
+
* decoded and mode bits are ignored there.
|
|
304
|
+
*/
|
|
305
|
+
export async function normalizeExtractedTree(
|
|
306
|
+
root: string,
|
|
307
|
+
opts: {
|
|
308
|
+
readonly legacyZipNames: boolean
|
|
309
|
+
/**
|
|
310
|
+
* Decoded listing names (forward-slash, relative) of the archive
|
|
311
|
+
* being extracted. The disambiguator for the macOS `%XX`-escaped
|
|
312
|
+
* name shape: an escaped name is valid UTF-8, so validity alone
|
|
313
|
+
* cannot tell `caf%82.jpg` (a legacy cp437 `café.jpg` after
|
|
314
|
+
* macOS escaping) apart from a file literally named
|
|
315
|
+
* `report%82.jpg` — only the listing's decoded name settles it.
|
|
316
|
+
*/
|
|
317
|
+
readonly expectedNames?: readonly string[]
|
|
318
|
+
},
|
|
319
|
+
): Promise<void> {
|
|
320
|
+
if (opts.legacyZipNames) {
|
|
321
|
+
const expected = opts.expectedNames
|
|
322
|
+
? new Set(opts.expectedNames.map((n) => n.replace(/\/+$/, "")))
|
|
323
|
+
: undefined
|
|
324
|
+
await renameLegacyZipNames(Buffer.from(root), root, root, expected)
|
|
325
|
+
}
|
|
326
|
+
await makeReadable(root)
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Decoded names of a zip archive — the ground truth for legacy (cp437)
|
|
331
|
+
* entry names. 7-Zip's text listing drops the bytes that are not valid
|
|
332
|
+
* UTF-8 in the archive: its console conversion removes the escape points
|
|
333
|
+
* on POSIX, so the reported name loses the character entirely. The zip
|
|
334
|
+
* itself is authoritative, so names come from yauzl (`decodeStrings`
|
|
335
|
+
* decodes non-UTF-8 names as cp437). Directory entries are absent from
|
|
336
|
+
* that listing, so every ancestor directory of a file path is added to
|
|
337
|
+
* `paths` for the rename pass. Falls back to the 7-Zip-listing names when
|
|
338
|
+
* the zip cannot be parsed a second time — extraction then proceeds
|
|
339
|
+
* exactly as before.
|
|
340
|
+
*/
|
|
341
|
+
export async function decodeZipNames(
|
|
342
|
+
zipPath: string,
|
|
343
|
+
fallback: readonly { readonly name: string; readonly sizeBytes: number }[],
|
|
344
|
+
): Promise<{
|
|
345
|
+
readonly files: readonly {
|
|
346
|
+
readonly name: string
|
|
347
|
+
readonly sizeBytes: number
|
|
348
|
+
}[]
|
|
349
|
+
readonly paths: readonly string[]
|
|
350
|
+
}> {
|
|
351
|
+
try {
|
|
352
|
+
const records = await listZipEntries(zipPath, { dataOffsets: false })
|
|
353
|
+
const files = records.map((e) => ({
|
|
354
|
+
name: e.name,
|
|
355
|
+
sizeBytes: e.uncompressedSize,
|
|
356
|
+
}))
|
|
357
|
+
const paths = new Set<string>()
|
|
358
|
+
for (const file of files) {
|
|
359
|
+
const segments = file.name.split("/")
|
|
360
|
+
for (let i = 1; i < segments.length; i++) {
|
|
361
|
+
paths.add(segments.slice(0, i).join("/"))
|
|
362
|
+
}
|
|
363
|
+
paths.add(file.name)
|
|
364
|
+
}
|
|
365
|
+
return { files, paths: [...paths] }
|
|
366
|
+
} catch {
|
|
367
|
+
return { files: fallback, paths: fallback.map((e) => e.name) }
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Rename every entry whose name does not match its decoded listing
|
|
373
|
+
* form. Legacy zip names can land on disk in two shapes:
|
|
374
|
+
*
|
|
375
|
+
* - raw bytes, when the archive name is not valid UTF-8 (7-Zip's POSIX
|
|
376
|
+
* and Windows builds pass/write them verbatim);
|
|
377
|
+
* - `%XX`-escaped bytes, when macOS's UTF-8 filesystem layer stores the
|
|
378
|
+
* raw names from the first shape (`XNU`'s `vfs_utfconv.c` converts an
|
|
379
|
+
* illegal byte to `%` + two hex digits while normalizing);
|
|
380
|
+
* - 7-Zip escape-plane characters (U+EF00+byte), the round-trip form
|
|
381
|
+
* for an illegal byte written without the byte-level conversion.
|
|
382
|
+
*
|
|
383
|
+
* The last two are valid UTF-8 on disk and are reconciled against the
|
|
384
|
+
* decoded names of the archive itself (see {@link decodeZipNames}) — the
|
|
385
|
+
* only lossless source, since 7-Zip's text listing drops those bytes on
|
|
386
|
+
* POSIX. A rename never clobbers an existing entry, and either escaped
|
|
387
|
+
* shape only wins when the recovered name matches a decoded listing path
|
|
388
|
+
* at the same position — a file genuinely named `report%82.jpg` (or with
|
|
389
|
+
* a private-use character) therefore stays untouched.
|
|
390
|
+
*/
|
|
391
|
+
async function renameLegacyZipNames(
|
|
392
|
+
rawRoot: Buffer,
|
|
393
|
+
decodedRoot: string,
|
|
394
|
+
root: string,
|
|
395
|
+
expected: ReadonlySet<string> | undefined,
|
|
396
|
+
): Promise<void> {
|
|
397
|
+
const entries = await readdir(rawRoot, {
|
|
398
|
+
withFileTypes: true,
|
|
399
|
+
encoding: "buffer",
|
|
400
|
+
})
|
|
401
|
+
for (const entry of entries) {
|
|
402
|
+
const childRaw = Buffer.concat([rawRoot, Buffer.from(sep), entry.name])
|
|
403
|
+
const decoded = decodeLegacyZipName(entry.name)
|
|
404
|
+
const target = legacyRenameTarget(
|
|
405
|
+
entry.name,
|
|
406
|
+
decoded,
|
|
407
|
+
decodedRoot,
|
|
408
|
+
root,
|
|
409
|
+
expected,
|
|
410
|
+
)
|
|
411
|
+
if (entry.isDirectory()) {
|
|
412
|
+
if (target !== undefined && !existsSync(target)) {
|
|
413
|
+
await rename(childRaw, target)
|
|
414
|
+
await renameLegacyZipNames(Buffer.from(target), target, root, expected)
|
|
415
|
+
continue
|
|
416
|
+
}
|
|
417
|
+
// Either no rename applies, or the decoded target already
|
|
418
|
+
// exists (a real entry owns the decoded name) — the raw
|
|
419
|
+
// directory itself stays, but its descendants must still be
|
|
420
|
+
// renamed under the decoded prefix.
|
|
421
|
+
await renameLegacyZipNames(
|
|
422
|
+
childRaw,
|
|
423
|
+
target ?? `${decodedRoot}/${decoded}`,
|
|
424
|
+
root,
|
|
425
|
+
expected,
|
|
426
|
+
)
|
|
427
|
+
continue
|
|
428
|
+
}
|
|
429
|
+
if (target === undefined || existsSync(target)) continue
|
|
430
|
+
await rename(childRaw, target)
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* The decoded path an extracted entry should be renamed to, or
|
|
436
|
+
* `undefined` when the on-disk name already satisfies the decoded
|
|
437
|
+
* convention (or cannot be decided safely).
|
|
438
|
+
*/
|
|
439
|
+
function legacyRenameTarget(
|
|
440
|
+
rawName: Buffer,
|
|
441
|
+
decoded: string,
|
|
442
|
+
decodedRoot: string,
|
|
443
|
+
root: string,
|
|
444
|
+
expected: ReadonlySet<string> | undefined,
|
|
445
|
+
): string | undefined {
|
|
446
|
+
const utf8Name = rawName.toString("utf8")
|
|
447
|
+
if (decoded !== utf8Name) {
|
|
448
|
+
// Raw bytes are not valid UTF-8: 7-Zip wrote the legacy name
|
|
449
|
+
// verbatim and the fs layer stored it byte-for-byte (Linux,
|
|
450
|
+
// Windows). Decode as cp437, matching the listing.
|
|
451
|
+
return `${decodedRoot}/${decoded}`
|
|
452
|
+
}
|
|
453
|
+
// The name IS valid UTF-8 — but the raw legacy bytes may have been
|
|
454
|
+
// re-encoded on the way to disk (macOS `%XX` escapes, or 7-Zip's
|
|
455
|
+
// escape-plane characters U+EF00+byte). Recover the original bytes and
|
|
456
|
+
// re-decode; only the listing's blessing keeps a literal
|
|
457
|
+
// `report%82.jpg` from being renamed to `reporté.jpg`.
|
|
458
|
+
const raw = recoverRawLegacyName(rawName)
|
|
459
|
+
if (raw === undefined) return undefined
|
|
460
|
+
const recovered = decodeLegacyZipName(raw)
|
|
461
|
+
if (recovered === utf8Name || expected === undefined) return undefined
|
|
462
|
+
const rel =
|
|
463
|
+
decodedRoot === root
|
|
464
|
+
? recovered
|
|
465
|
+
: `${decodedRoot.slice(root.length + 1)}/${recovered}`
|
|
466
|
+
return expected.has(rel) ? `${decodedRoot}/${recovered}` : undefined
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* The original archive bytes of an on-disk name that carried a legacy
|
|
471
|
+
* cp437 byte, and `undefined` when the name has no mangle at all. Two
|
|
472
|
+
* transient shapes exist for a raw byte `b`:
|
|
473
|
+
*
|
|
474
|
+
* - `%XX` hex escapes: macOS's UTF-8 filesystem layer stores illegal
|
|
475
|
+
* filename bytes that way (`XNU`'s `vfs_utfconv.c`);
|
|
476
|
+
* - the escape-plane character U+EF00+b: 7-Zip's internal representation
|
|
477
|
+
* for a byte it cannot show as UTF-8, written verbatim when the name
|
|
478
|
+
* is encoded without the byte-level round-trip.
|
|
479
|
+
*
|
|
480
|
+
* Either shape maps back to exactly one byte per source byte.
|
|
481
|
+
*/
|
|
482
|
+
function recoverRawLegacyName(name: Buffer): Buffer | undefined {
|
|
483
|
+
const escaped = unescapePercentEscapes(name)
|
|
484
|
+
if (escaped !== undefined) return escaped
|
|
485
|
+
const text = name.toString("utf8")
|
|
486
|
+
const out: number[] = []
|
|
487
|
+
for (const ch of text) {
|
|
488
|
+
const code = ch.codePointAt(0) ?? 0
|
|
489
|
+
if (code >= 0xef80 && code <= 0xefff) {
|
|
490
|
+
out.push(code - 0xef00)
|
|
491
|
+
continue
|
|
492
|
+
}
|
|
493
|
+
if (code > 0x7f) return undefined // mixed with real text — not a mangle
|
|
494
|
+
out.push(code)
|
|
495
|
+
}
|
|
496
|
+
return out.length === text.length ? Buffer.from(out) : undefined
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* Expand the `%XX` sequences a macOS UTF-8 filesystem writes for illegal
|
|
501
|
+
* filename bytes back to their original byte values. Returns `undefined`
|
|
502
|
+
* when the name contains no `%XX` sequence (nothing to recover). Only
|
|
503
|
+
* sequences whose bytes are illegal as UTF-8 matter to the caller —
|
|
504
|
+
* legitimate `%XX` substrings simply survive as decoded characters and
|
|
505
|
+
* fail the listing match.
|
|
506
|
+
*/
|
|
507
|
+
function unescapePercentEscapes(name: Buffer): Buffer | undefined {
|
|
508
|
+
const text = name.toString("latin1")
|
|
509
|
+
if (!text.includes("%")) return undefined
|
|
510
|
+
const out: number[] = []
|
|
511
|
+
let changed = false
|
|
512
|
+
for (let i = 0; i < text.length; i++) {
|
|
513
|
+
const hex = i + 2 < text.length ? text.slice(i + 1, i + 3) : undefined
|
|
514
|
+
if (text[i] === "%" && hex !== undefined && /^[0-9a-fA-F]{2}$/.test(hex)) {
|
|
515
|
+
out.push(Number.parseInt(hex, 16))
|
|
516
|
+
changed = true
|
|
517
|
+
i += 2
|
|
518
|
+
continue
|
|
519
|
+
}
|
|
520
|
+
out.push(text.charCodeAt(i))
|
|
521
|
+
}
|
|
522
|
+
return changed ? Buffer.from(out) : undefined
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* OR owner permissions into every entry (directories rwx, files rw) so
|
|
527
|
+
* extracted archives stay accessible to the app whatever mode bits the
|
|
528
|
+
* entries carried. Symlinks and special files are skipped — they are
|
|
529
|
+
* refused by {@link assertExtractedTree} afterwards and must never be
|
|
530
|
+
* followed here.
|
|
531
|
+
*/
|
|
532
|
+
async function makeReadable(root: string): Promise<void> {
|
|
533
|
+
const entries = await readdir(root, { withFileTypes: true })
|
|
534
|
+
for (const entry of entries) {
|
|
535
|
+
const full = join(root, entry.name)
|
|
536
|
+
if (entry.isDirectory()) {
|
|
537
|
+
await chmod(full, (await lstat(full)).mode | 0o700)
|
|
538
|
+
await makeReadable(full)
|
|
539
|
+
continue
|
|
540
|
+
}
|
|
541
|
+
if (entry.isFile()) {
|
|
542
|
+
await chmod(full, (await lstat(full)).mode | 0o600)
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/** Reject absolute paths and traversal segments (zip-slip guard). */
|
|
548
|
+
function assertSafeEntryPath(rawName: string): void {
|
|
549
|
+
if (
|
|
550
|
+
rawName.length === 0 ||
|
|
551
|
+
/^([a-zA-Z]:)?[\\/]/.test(rawName) ||
|
|
552
|
+
rawName.split("/").includes("..")
|
|
553
|
+
) {
|
|
554
|
+
throw invalid(
|
|
555
|
+
"resource.archive_invalid_entry",
|
|
556
|
+
`archive entry has an unsafe path: ${rawName}`,
|
|
557
|
+
{ rawName },
|
|
558
|
+
)
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
async function handleExtractEntry(
|
|
563
|
+
zipfile: ZipFile,
|
|
564
|
+
entry: Entry,
|
|
565
|
+
root: string,
|
|
566
|
+
budget: { remaining: number; readonly max: number },
|
|
567
|
+
): Promise<void> {
|
|
568
|
+
const safe = safeExtractEntryPath(entry.fileName, root)
|
|
569
|
+
if (entry.fileName.endsWith("/")) {
|
|
570
|
+
await mkdir(safe, { recursive: true })
|
|
571
|
+
return
|
|
572
|
+
}
|
|
573
|
+
await mkdir(dirname(safe), { recursive: true })
|
|
574
|
+
const stream = await openZipEntryStream(zipfile, entry)
|
|
575
|
+
await pipeline(stream, makeByteBudgetCounter(budget), createWriteStream(safe))
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
function safeExtractEntryPath(rawName: string, root: string): string {
|
|
579
|
+
if (rawName.length === 0) {
|
|
580
|
+
throw invalid(
|
|
581
|
+
"resource.archive_invalid_entry",
|
|
582
|
+
"archive entry has empty name",
|
|
583
|
+
{ rawName },
|
|
584
|
+
)
|
|
585
|
+
}
|
|
586
|
+
if (/^([a-zA-Z]:)?[\\/]/.test(rawName)) {
|
|
587
|
+
throw invalid(
|
|
588
|
+
"resource.archive_invalid_entry",
|
|
589
|
+
`archive entry has absolute path: ${rawName}`,
|
|
590
|
+
{ rawName },
|
|
591
|
+
)
|
|
592
|
+
}
|
|
593
|
+
const normalised = normalize(rawName).replace(/\\/g, "/")
|
|
594
|
+
if (normalised.startsWith("../") || normalised === "..") {
|
|
595
|
+
throw invalid(
|
|
596
|
+
"resource.archive_invalid_entry",
|
|
597
|
+
`archive entry escapes destination: ${rawName}`,
|
|
598
|
+
{ rawName },
|
|
599
|
+
)
|
|
600
|
+
}
|
|
601
|
+
const candidate = resolve(root, normalised)
|
|
602
|
+
if (candidate !== root && !candidate.startsWith(root + sep)) {
|
|
603
|
+
throw invalid(
|
|
604
|
+
"resource.archive_invalid_entry",
|
|
605
|
+
`archive entry escapes destination: ${rawName}`,
|
|
606
|
+
{ rawName },
|
|
607
|
+
)
|
|
608
|
+
}
|
|
609
|
+
return candidate
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Transform that subtracts each chunk's size from a shared byte budget
|
|
614
|
+
* and destroys itself with a VALIDATION error once the budget would go
|
|
615
|
+
* negative. Used to defend against zip bombs across all entries in a
|
|
616
|
+
* single archive on the yauzl fallback path (the 7-Zip path pre-checks
|
|
617
|
+
* the listing and re-verifies the extracted tree instead).
|
|
618
|
+
*/
|
|
619
|
+
function makeByteBudgetCounter(budget: {
|
|
620
|
+
remaining: number
|
|
621
|
+
readonly max: number
|
|
622
|
+
}): Transform {
|
|
623
|
+
return new Transform({
|
|
624
|
+
transform(chunk, _enc, cb) {
|
|
625
|
+
const len = Buffer.isBuffer(chunk)
|
|
626
|
+
? chunk.length
|
|
627
|
+
: Buffer.byteLength(chunk)
|
|
628
|
+
budget.remaining -= len
|
|
629
|
+
if (budget.remaining < 0) {
|
|
630
|
+
cb(
|
|
631
|
+
invalid(
|
|
632
|
+
"resource.archive_too_large",
|
|
633
|
+
`archive extracts to more than ${budget.max} bytes`,
|
|
634
|
+
{ maxExtractedBytes: budget.max },
|
|
635
|
+
),
|
|
636
|
+
)
|
|
637
|
+
return
|
|
638
|
+
}
|
|
639
|
+
cb(undefined, chunk)
|
|
640
|
+
},
|
|
641
|
+
})
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
async function readToBuffer(source: NodeJS.ReadableStream): Promise<Buffer> {
|
|
645
|
+
const chunks: Buffer[] = []
|
|
646
|
+
for await (const chunk of source) {
|
|
647
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk)
|
|
648
|
+
}
|
|
649
|
+
return Buffer.concat(chunks)
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
function openZipFromBuffer(buffer: Buffer): Promise<ZipFile> {
|
|
653
|
+
return new Promise<ZipFile>((res, rej) => {
|
|
654
|
+
yauzl.fromBuffer(
|
|
655
|
+
buffer,
|
|
656
|
+
{ lazyEntries: true },
|
|
657
|
+
(err: Error | null, zip: ZipFile) => {
|
|
658
|
+
if (err !== null || zip === undefined) {
|
|
659
|
+
rej(
|
|
660
|
+
invalid(
|
|
661
|
+
"resource.archive_open_failed",
|
|
662
|
+
err?.message ?? "could not open archive",
|
|
663
|
+
{},
|
|
664
|
+
),
|
|
665
|
+
)
|
|
666
|
+
return
|
|
667
|
+
}
|
|
668
|
+
res(zip)
|
|
669
|
+
},
|
|
670
|
+
)
|
|
671
|
+
})
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
function openZipEntryStream(
|
|
675
|
+
zipfile: ZipFile,
|
|
676
|
+
entry: Entry,
|
|
677
|
+
): Promise<NodeJS.ReadableStream> {
|
|
678
|
+
return new Promise((res, rej) => {
|
|
679
|
+
zipfile.openReadStream(entry, (err, stream) => {
|
|
680
|
+
if ((err !== null && err !== undefined) || stream === undefined) {
|
|
681
|
+
rej(
|
|
682
|
+
invalid(
|
|
683
|
+
"resource.archive_entry_unreadable",
|
|
684
|
+
err?.message ?? `could not read archive entry: ${entry.fileName}`,
|
|
685
|
+
{ entry: entry.fileName },
|
|
686
|
+
),
|
|
687
|
+
)
|
|
688
|
+
return
|
|
689
|
+
}
|
|
690
|
+
res(stream)
|
|
691
|
+
})
|
|
692
|
+
})
|
|
693
|
+
}
|