7zip-native 0.0.8 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -14
- package/bench/compare.mts +25 -11
- package/bench/folder-concurrency.mts +5 -12
- package/bench/multiprocess-extract.cjs +334 -0
- package/dist/index.cjs +215 -110
- package/dist/index.d.cts +10 -9
- package/dist/index.d.mts +10 -9
- package/dist/index.d.ts +10 -9
- package/dist/index.mjs +217 -112
- package/native/addon.cpp +1399 -71
- package/package.json +4 -2
- package/patches/deflate-input-buffer-size.patch +47 -0
- package/prebuilds/darwin-arm64/7zip_native.node +0 -0
- package/prebuilds/darwin-arm64/lib7z.dylib +0 -0
- package/prebuilds/darwin-x64/7zip_native.node +0 -0
- package/prebuilds/darwin-x64/lib7z.dylib +0 -0
- package/prebuilds/linux-arm/7z.so +0 -0
- package/prebuilds/linux-arm/7zip_native.node +0 -0
- package/prebuilds/linux-arm64/7z.so +0 -0
- package/prebuilds/linux-arm64/7zip_native.node +0 -0
- package/prebuilds/linux-ia32/7z.so +0 -0
- package/prebuilds/linux-ia32/7zip_native.node +0 -0
- package/prebuilds/linux-x64/7z.so +0 -0
- package/prebuilds/linux-x64/7zip_native.node +0 -0
- package/prebuilds/win32-arm64/7z.dll +0 -0
- package/prebuilds/win32-arm64/7zip_native.node +0 -0
- package/prebuilds/win32-x64/7z.dll +0 -0
- package/prebuilds/win32-x64/7zip_native.node +0 -0
- package/scripts/build-native.mjs +22 -1
- package/vendor/7zip/CPP/7zip/Archive/Archive2.def +1 -0
- package/vendor/7zip/CPP/7zip/Bundles/Format7zF/_o/7z.so +0 -0
- package/vendor/7zip/CPP/7zip/Bundles/Format7zF/_o/Alloc.o +0 -0
- package/vendor/7zip/CPP/7zip/Bundles/Format7zF/_o/DeflateDecoder.o +0 -0
- package/vendor/7zip/CPP/7zip/Compress/DeflateDecoder.cpp +15 -1
package/README.md
CHANGED
|
@@ -66,6 +66,11 @@ const archive = await Archive.open('./archive.7z', {password: 'secret'});
|
|
|
66
66
|
`Archive.open()` accepts these options:
|
|
67
67
|
|
|
68
68
|
- `password`: Passed to the upstream handler if it requests a password. It is held for the lifetime of the archive handle, is not logged, and is limited to 1 MiB of UTF-8 data.
|
|
69
|
+
- `deflateBufferSize`: Input buffer size in bytes for Deflate extraction. Defaults to `32768`; accepts integers from `4096` through `2097152`. It is stored per archive and applied to each extraction, so different open archives can use different values.
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
const archive = await Archive.open('./archive.zip', {deflateBufferSize: 32768});
|
|
73
|
+
```
|
|
69
74
|
|
|
70
75
|
### Listing and looking up entries
|
|
71
76
|
|
|
@@ -109,14 +114,24 @@ const header = await archive.read(entry.path, {
|
|
|
109
114
|
offset: 0,
|
|
110
115
|
length: 64 * 1024,
|
|
111
116
|
});
|
|
117
|
+
const headers = await archive.read([first.path, second.id], {
|
|
118
|
+
offset: 0,
|
|
119
|
+
length: 64 * 1024,
|
|
120
|
+
});
|
|
121
|
+
const prefixes = await archive.read([
|
|
122
|
+
{entry: first.path, offset: 0, length: 8192},
|
|
123
|
+
{entry: second.id, offset: 0, length: 64},
|
|
124
|
+
]);
|
|
112
125
|
```
|
|
113
126
|
|
|
127
|
+
`read()` accepts one entry selector or a batch. For a selector array, the same range is used for every entry; use `ReadRequest[]` to specify a different range per entry. Both batch forms return buffers in request order and run as one native extraction operation. The selector-array form preserves repeated selectors; `ReadRequest[]` must contain unique entries.
|
|
128
|
+
|
|
114
129
|
`read()` accepts these options:
|
|
115
130
|
|
|
116
131
|
- `offset`: Byte position at which to start reading. Defaults to `0`; accepts a non-negative safe integer or `bigint`.
|
|
117
132
|
- `length`: Number of bytes to return. Required; accepts a non-negative safe integer or `bigint`. A range read is limited to 64 MiB.
|
|
118
133
|
|
|
119
|
-
A range read returns only the requested bytes. It uses the handler's normal extraction path, however, so a sequential or solid format may still need to decompress preceding data or the whole entry. It is not guaranteed to be constant-time random access.
|
|
134
|
+
A range read returns only the requested bytes. A batch is limited to 64 MiB per entry and 256 MiB total output. It uses the handler's normal extraction path, however, so a sequential or solid format may still need to decompress preceding data or the whole entry. It is not guaranteed to be constant-time random access.
|
|
120
135
|
|
|
121
136
|
### Streams and output destinations
|
|
122
137
|
|
|
@@ -169,13 +184,15 @@ await Archive.create('./archive.zip', [
|
|
|
169
184
|
- `sourcePath`: Filesystem path of the regular file to add.
|
|
170
185
|
- `path`: Desired path inside the archive, using `/` separators.
|
|
171
186
|
|
|
187
|
+
The source file's modification time is stored in the archive.
|
|
188
|
+
|
|
172
189
|
It also accepts these options:
|
|
173
190
|
|
|
174
191
|
- `format`: Archive format. If omitted, inferred as ZIP from `.zip`, 7z from `.7z`, TAR from `.tar`, or WIM from `.wim`. Set to `'zip'`, `'7z'`, `'tar'`, or `'wim'` to use another output extension.
|
|
175
192
|
- `compressionLevel`: Integer from 0 to 9; defaults to 5 and applies only to ZIP and 7z. TAR is uncompressed; WIM uses the upstream handler's default settings.
|
|
176
193
|
- `overwrite`: Defaults to `false`. When `true`, an existing output file is replaced after the new archive has been written.
|
|
177
194
|
|
|
178
|
-
Duplicate,
|
|
195
|
+
Duplicate, absolute, or traversal paths are rejected, but platform-supported filename characters such as trailing spaces are preserved in the archive-internal path. The output is first written to a temporary file beside the destination, then installed atomically, so an existing file is preserved if archive creation fails. Archive creation runs in the bounded native worker pool and does not launch a 7-Zip executable.
|
|
179
196
|
|
|
180
197
|
### Extracting to a folder
|
|
181
198
|
|
|
@@ -184,11 +201,8 @@ await archive.extractToFolder('./unpacked', {
|
|
|
184
201
|
paths: ['chapter/page1.jpg', 'chapter/page2.jpg'], // alternatively, use ids
|
|
185
202
|
concurrency: 2,
|
|
186
203
|
overwrite: true,
|
|
187
|
-
|
|
188
|
-
console.log(
|
|
189
|
-
},
|
|
190
|
-
onExtract: ({entry, outputPath}) => {
|
|
191
|
-
console.log(`Extracted ${entry.path} to ${outputPath}`);
|
|
204
|
+
onExtract: ({entry, outputPath, percent}) => {
|
|
205
|
+
console.log(`Extracted ${entry.path} to ${outputPath} (${percent}%)`);
|
|
192
206
|
},
|
|
193
207
|
maxBytes: 2n * 1024n * 1024n * 1024n,
|
|
194
208
|
signal: abortController.signal,
|
|
@@ -199,14 +213,15 @@ await archive.extractToFolder('./unpacked', {
|
|
|
199
213
|
|
|
200
214
|
- `ids`: Entry IDs to extract. Omit both `ids` and `paths` to extract every entry.
|
|
201
215
|
- `paths`: Exact, original archive paths to extract, using `/` separators. This cannot be combined with `ids`.
|
|
202
|
-
- `concurrency`: Maximum number of
|
|
203
|
-
- `overwrite`: Defaults to `false`, so existing files are not overwritten. When `true`,
|
|
204
|
-
- `
|
|
205
|
-
- `onExtract`: Called once for each regular file after it is fully written and, when overwriting, after the final replacement. Receives `{entry, outputPath}`; `entry` retains the original archive metadata and `outputPath` is the final destination path.
|
|
216
|
+
- `concurrency`: Maximum number of native extraction batches this call runs simultaneously. Must be an integer from 1 to 128; defaults to 1 and is capped at the process-wide native worker-pool concurrency. This does not change `Archive.concurrency()`.
|
|
217
|
+
- `overwrite`: Defaults to `false`, so existing files are not overwritten. When `true`, an existing regular file is removed before the new file is written directly to its final path. A failed extraction can leave the destination missing.
|
|
218
|
+
- `onExtract`: Called after each regular file has been completely written and closed; it does not wait for the rest of the batch. Concurrent extraction can make event order differ from selection order. Receives `{entry, outputPath, percent}`, where `percent` is the completed-file count divided by the total selected regular files. `entry` retains the original archive metadata and `outputPath` is the final destination path.
|
|
206
219
|
- `maxBytes`: Maximum total uncompressed output size. Defaults to 4 GiB; set to `null` to disable the limit.
|
|
207
220
|
- `signal`: Optional `AbortSignal`. Cancellation stops new files from starting, cancels or waits for active extractions, cleans temporary replacement files, and rejects the call.
|
|
208
221
|
|
|
209
|
-
|
|
222
|
+
Each native extraction call receives a batch of entry IDs and output paths. The addon queues disk writes asynchronously while 7-Zip continues extracting later entries; `onExtract` runs after that file's queued writes have completed and the file is closed. File contents are not streamed through Node.js or Electron. The `extractToFolder()` promise resolves after all `onExtract` callbacks have run.
|
|
223
|
+
|
|
224
|
+
On Windows only, unsupported filename characters (including control characters), trailing dots/spaces in each destination component, and reserved device names are normalized for the output path; selectors and entry metadata remain unchanged. On Linux and macOS, platform-supported filename characters are preserved. Unsafe archive paths, path collisions, symlinked destination components, and symlink or non-file targets are rejected. Archive links are written as regular file data; the API does not create symbolic or hard links. If extraction fails partway through, files already written (or partially written) may remain in the destination.
|
|
210
225
|
|
|
211
226
|
### Concurrency
|
|
212
227
|
|
|
@@ -329,7 +344,7 @@ BENCH_ITERATIONS=20 npm run benchmark -- "./archive.zip"
|
|
|
329
344
|
|
|
330
345
|
It measures native open/list, prefix reads, full extraction, and reads across independent archive handles. `BENCH_ITERATIONS` defaults to 15 and must be at least 3.
|
|
331
346
|
|
|
332
|
-
Compare the native API with the `
|
|
347
|
+
Compare the native API with the `7zzc` command-line baseline from `7zip-bin-full`:
|
|
333
348
|
|
|
334
349
|
```sh
|
|
335
350
|
npm run benchmark:compare -- \
|
|
@@ -340,7 +355,7 @@ npm run benchmark:compare -- \
|
|
|
340
355
|
--concurrency 4
|
|
341
356
|
```
|
|
342
357
|
|
|
343
|
-
The comparison validates entry listings and output data before reporting median and p95 latency. CLI timings include process startup.
|
|
358
|
+
The comparison validates entry listings and output data before reporting median and p95 latency. CLI timings include process startup. Prefix reads use `-slb`; non-solid archives abort decoding per item once the byte limit is reached. For a range, the CLI emits through `offset + length` and slices stdout because it has no ranged-read API; this still needs to decode the preceding bytes. Captured output is capped at 256 MiB, and full-archive comparison is skipped if an entry exceeds that limit.
|
|
344
359
|
|
|
345
360
|
Compare extraction concurrency:
|
|
346
361
|
|
|
@@ -354,6 +369,23 @@ npm run benchmark:concurrency -- \
|
|
|
354
369
|
|
|
355
370
|
The report includes median and p95 latency, throughput, speedup, and the number of entries that can overlap. `--max-files` selects evenly spaced entries, which is useful when sampling solid archives. Benchmarks report measurements only; results depend on the archive, machine, and workload and do not guarantee a speedup.
|
|
356
371
|
|
|
372
|
+
Compare parallel extraction of every file in the archive, divided among promises:
|
|
373
|
+
|
|
374
|
+
```sh
|
|
375
|
+
npm run benchmark:multiprocess -- \
|
|
376
|
+
--archive "./archive.zip" \
|
|
377
|
+
--parallel-counts 1,2,4 \
|
|
378
|
+
--iterations 3
|
|
379
|
+
|
|
380
|
+
ELECTRON_RUN_AS_NODE=1 electron bench/multiprocess-extract.cjs \
|
|
381
|
+
--archive "./archive.zip" \
|
|
382
|
+
--parallel-counts 1,2,4 \
|
|
383
|
+
--iterations 3 \
|
|
384
|
+
--deflate-buffer-size 262144
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
For each parallel count, the benchmark uses `Promise.all()`: every `7zip-native` promise opens its own handle to the same archive, and every `7zip-bin-full` promise spawns one `7zzc` command. All non-directory files in the archive are divided as evenly as possible among the concurrent operations. The native extraction pool is sized to the maximum requested parallel count so native promises can run concurrently. Outputs go to separate temporary directories and are verified by file size before cleanup. The report includes `Archive.open()` and CLI startup time. Native operations stay in the benchmark's Node/Electron process; only the CLI baseline spawns child processes.
|
|
388
|
+
|
|
357
389
|
Compare `extractToFolder()` concurrency (1 through 16) across one or more archives:
|
|
358
390
|
|
|
359
391
|
```sh
|
package/bench/compare.mts
CHANGED
|
@@ -35,12 +35,12 @@ if(!Number.isSafeInteger(parallelConcurrency) || parallelConcurrency < 1 || para
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
const require = createRequire(import.meta.url);
|
|
38
|
-
const binaryPackage = require('7zip-bin-full') as {
|
|
39
|
-
if(typeof binaryPackage.
|
|
38
|
+
const binaryPackage = require('7zip-bin-full') as {path7zc?: unknown};
|
|
39
|
+
if(typeof binaryPackage.path7zc !== 'string')
|
|
40
40
|
{
|
|
41
|
-
throw new TypeError('7zip-bin-full does not provide a
|
|
41
|
+
throw new TypeError('7zip-bin-full does not provide a path7zc binary for this platform');
|
|
42
42
|
}
|
|
43
|
-
const binaryPath = binaryPackage.
|
|
43
|
+
const binaryPath = binaryPackage.path7zc;
|
|
44
44
|
|
|
45
45
|
function runCli(args: readonly string[], outputLimit = maxCapturedBytes): Promise<Buffer>
|
|
46
46
|
{
|
|
@@ -142,10 +142,18 @@ function cliList(): Promise<CliEntry[]>
|
|
|
142
142
|
return runCli(['l', '-slt', '-sccUTF-8', archivePath]).then(parseCliEntries);
|
|
143
143
|
}
|
|
144
144
|
|
|
145
|
-
function cliExtract(
|
|
145
|
+
function cliExtract(
|
|
146
|
+
paths: readonly string[],
|
|
147
|
+
outputLimit = maxCapturedBytes,
|
|
148
|
+
byteLimit?: number,
|
|
149
|
+
): Promise<Buffer>
|
|
146
150
|
{
|
|
147
|
-
|
|
148
|
-
|
|
151
|
+
const args = ['x'];
|
|
152
|
+
if(byteLimit !== undefined)
|
|
153
|
+
{
|
|
154
|
+
args.push(`-slb${byteLimit}`);
|
|
155
|
+
}
|
|
156
|
+
args.push(
|
|
149
157
|
archivePath,
|
|
150
158
|
...paths,
|
|
151
159
|
'-so',
|
|
@@ -154,7 +162,8 @@ function cliExtract(paths: readonly string[], outputLimit = maxCapturedBytes): P
|
|
|
154
162
|
'-bb0',
|
|
155
163
|
'-spd',
|
|
156
164
|
'-sccUTF-8',
|
|
157
|
-
|
|
165
|
+
);
|
|
166
|
+
return runCli(args, outputLimit);
|
|
158
167
|
}
|
|
159
168
|
|
|
160
169
|
function isGroupedExtractionResult(value: BenchmarkResult): value is GroupedExtractionResult
|
|
@@ -502,7 +511,7 @@ try
|
|
|
502
511
|
));
|
|
503
512
|
}
|
|
504
513
|
|
|
505
|
-
const rangeNote = '
|
|
514
|
+
const rangeNote = '7zzc uses -slb; non-solid archives abort per item, while ranges emit through offset + length and slice stdout';
|
|
506
515
|
for(const [name, offset, length] of [
|
|
507
516
|
['read_prefix', 0n, prefixLength],
|
|
508
517
|
['read_range_at_one_third', rangeOffset, rangeLength],
|
|
@@ -515,8 +524,13 @@ try
|
|
|
515
524
|
}
|
|
516
525
|
const nativeRead = () => archive.read(file.id, {offset, length});
|
|
517
526
|
const cliRead = async () => {
|
|
518
|
-
|
|
519
|
-
|
|
527
|
+
if(length === 0n)
|
|
528
|
+
{
|
|
529
|
+
return Buffer.alloc(0);
|
|
530
|
+
}
|
|
531
|
+
const limit = Number(offset + length);
|
|
532
|
+
const output = await cliExtract([file.path], limit, limit);
|
|
533
|
+
return output.subarray(Number(offset), limit);
|
|
520
534
|
};
|
|
521
535
|
results.push(await measurePair(name, nativeRead, cliRead));
|
|
522
536
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {mkdtemp, rm, stat} from 'node:fs/promises';
|
|
2
2
|
import {tmpdir} from 'node:os';
|
|
3
3
|
import {join, resolve} from 'node:path';
|
|
4
|
-
import {Archive
|
|
4
|
+
import {Archive} from '../dist/index.mjs';
|
|
5
5
|
|
|
6
6
|
const usage = 'npm run benchmark:folder-concurrency -- --archive <path> [--archive <path> ...] [--iterations <count>] [--concurrency <workers> | --concurrencies <workers,...>]';
|
|
7
7
|
const archivePaths: string[] = [];
|
|
@@ -119,27 +119,20 @@ try
|
|
|
119
119
|
for(let iteration = 0; iteration < iterations; iteration++)
|
|
120
120
|
{
|
|
121
121
|
const temporaryDirectory = await mkdtemp(join(tmpdir(), '7zip-native-folder-bench-'));
|
|
122
|
-
let
|
|
122
|
+
let finalPercent = 0;
|
|
123
123
|
let completedFiles = 0;
|
|
124
124
|
try
|
|
125
125
|
{
|
|
126
126
|
const start = process.hrtime.bigint();
|
|
127
127
|
await archive.extractToFolder(join(temporaryDirectory, 'output'), {
|
|
128
128
|
concurrency,
|
|
129
|
-
|
|
130
|
-
finalProgress = progress;
|
|
131
|
-
},
|
|
132
|
-
onExtract: () => {
|
|
129
|
+
onExtract: ({percent}) => {
|
|
133
130
|
completedFiles++;
|
|
131
|
+
finalPercent = percent;
|
|
134
132
|
},
|
|
135
133
|
});
|
|
136
134
|
durations.push(Number(process.hrtime.bigint() - start) / 1e6);
|
|
137
|
-
if(
|
|
138
|
-
|| finalProgress.filesExtracted !== files.length
|
|
139
|
-
|| finalProgress.totalFiles !== files.length
|
|
140
|
-
|| finalProgress.bytesProcessed !== expectedBytes
|
|
141
|
-
|| finalProgress.totalBytes !== expectedBytes
|
|
142
|
-
|| completedFiles !== files.length)
|
|
135
|
+
if(completedFiles !== files.length || finalPercent !== 100)
|
|
143
136
|
{
|
|
144
137
|
throw new Error(`Extraction verification failed for ${archivePath} at concurrency ${concurrency}`);
|
|
145
138
|
}
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const {spawn} = require('node:child_process');
|
|
4
|
+
const {mkdtemp, rm, stat} = require('node:fs/promises');
|
|
5
|
+
const {tmpdir} = require('node:os');
|
|
6
|
+
const path = require('node:path');
|
|
7
|
+
|
|
8
|
+
function elapsedMs(start)
|
|
9
|
+
{
|
|
10
|
+
return Number(process.hrtime.bigint() - start) / 1e6;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function errorText(chunks)
|
|
14
|
+
{
|
|
15
|
+
return Buffer.concat(chunks).toString('utf8').trim();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async function extractNative(Archive, archivePath, outputPath, files, deflateBufferSize)
|
|
19
|
+
{
|
|
20
|
+
const started = process.hrtime.bigint();
|
|
21
|
+
let archive;
|
|
22
|
+
try
|
|
23
|
+
{
|
|
24
|
+
archive = await Archive.open(archivePath, {deflateBufferSize});
|
|
25
|
+
await archive.extractToFolder(outputPath, {
|
|
26
|
+
paths: files.map(file => file.path),
|
|
27
|
+
concurrency: 1,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
finally
|
|
31
|
+
{
|
|
32
|
+
await archive?.close();
|
|
33
|
+
}
|
|
34
|
+
return elapsedMs(started);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function extractWithCli(binaryPath, archivePath, outputPath, files)
|
|
38
|
+
{
|
|
39
|
+
return new Promise((resolve, reject) => {
|
|
40
|
+
const started = process.hrtime.bigint();
|
|
41
|
+
const args = [
|
|
42
|
+
'x',
|
|
43
|
+
archivePath,
|
|
44
|
+
...files.map(file => file.path),
|
|
45
|
+
`-o${outputPath}`,
|
|
46
|
+
'-y',
|
|
47
|
+
'-bd',
|
|
48
|
+
'-bb0',
|
|
49
|
+
'-mmt=1',
|
|
50
|
+
'-sccUTF-8',
|
|
51
|
+
'-spd',
|
|
52
|
+
];
|
|
53
|
+
const child = spawn(binaryPath, args, {stdio: ['ignore', 'ignore', 'pipe']});
|
|
54
|
+
const stderr = [];
|
|
55
|
+
let stderrLength = 0;
|
|
56
|
+
child.stderr.on('data', chunk => {
|
|
57
|
+
if(stderrLength < 64 * 1024)
|
|
58
|
+
{
|
|
59
|
+
const kept = chunk.subarray(0, 64 * 1024 - stderrLength);
|
|
60
|
+
stderr.push(kept);
|
|
61
|
+
stderrLength += kept.length;
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
child.once('error', reject);
|
|
65
|
+
child.once('close', (code, signal) => {
|
|
66
|
+
if(code !== 0)
|
|
67
|
+
{
|
|
68
|
+
const detail = errorText(stderr);
|
|
69
|
+
reject(new Error(
|
|
70
|
+
`7zip-bin-full exited with ${signal ?? code}${detail ? `: ${detail}` : ''}`,
|
|
71
|
+
));
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
resolve(elapsedMs(started));
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function safeOutputPath(root, entryPath)
|
|
80
|
+
{
|
|
81
|
+
const normalized = entryPath.replace(/\\/g, '/');
|
|
82
|
+
const parts = normalized.split('/').filter(part => part !== '' && part !== '.');
|
|
83
|
+
if(normalized.startsWith('/') || parts.length === 0 || parts.some(part => part === '..'))
|
|
84
|
+
{
|
|
85
|
+
throw new Error(`Unsafe archive path in benchmark input: ${entryPath}`);
|
|
86
|
+
}
|
|
87
|
+
return path.resolve(root, ...parts);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function verifyOutput(outputPath, files)
|
|
91
|
+
{
|
|
92
|
+
let totalBytes = 0n;
|
|
93
|
+
for(const file of files)
|
|
94
|
+
{
|
|
95
|
+
const outputFile = safeOutputPath(outputPath, file.path);
|
|
96
|
+
const outputStat = await stat(outputFile);
|
|
97
|
+
if(!outputStat.isFile() || BigInt(outputStat.size) !== file.size)
|
|
98
|
+
{
|
|
99
|
+
throw new Error(
|
|
100
|
+
`Extraction mismatch for ${file.path}: expected ${file.size} bytes, got ${outputStat.size}`,
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
totalBytes += file.size;
|
|
104
|
+
}
|
|
105
|
+
return totalBytes;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function runWave(
|
|
109
|
+
backend,
|
|
110
|
+
Archive,
|
|
111
|
+
binaryPath,
|
|
112
|
+
archivePath,
|
|
113
|
+
groups,
|
|
114
|
+
expectedFileCount,
|
|
115
|
+
parallelOperations,
|
|
116
|
+
deflateBufferSize,
|
|
117
|
+
)
|
|
118
|
+
{
|
|
119
|
+
if(groups.length !== parallelOperations
|
|
120
|
+
|| groups.some(files => files.length === 0)
|
|
121
|
+
|| groups.reduce((sum, files) => sum + files.length, 0) !== expectedFileCount)
|
|
122
|
+
{
|
|
123
|
+
throw new Error(`The concurrent operations must partition exactly ${expectedFileCount} files`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const temporaryRoot = await mkdtemp(path.join(tmpdir(), '7zip-native-parallel-extract-'));
|
|
127
|
+
const started = process.hrtime.bigint();
|
|
128
|
+
try
|
|
129
|
+
{
|
|
130
|
+
const operations = groups.map((files, index) => {
|
|
131
|
+
const outputPath = path.join(temporaryRoot, `operation-${index + 1}`);
|
|
132
|
+
return backend === 'native'
|
|
133
|
+
? extractNative(Archive, archivePath, outputPath, files, deflateBufferSize)
|
|
134
|
+
: extractWithCli(binaryPath, archivePath, outputPath, files);
|
|
135
|
+
});
|
|
136
|
+
let operationTimes;
|
|
137
|
+
try
|
|
138
|
+
{
|
|
139
|
+
operationTimes = await Promise.all(operations);
|
|
140
|
+
}
|
|
141
|
+
catch(error)
|
|
142
|
+
{
|
|
143
|
+
await Promise.allSettled(operations);
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
const wallMs = elapsedMs(started);
|
|
147
|
+
let totalBytes = 0n;
|
|
148
|
+
for(let index = 0; index < parallelOperations; index++)
|
|
149
|
+
{
|
|
150
|
+
const files = groups[index];
|
|
151
|
+
totalBytes += await verifyOutput(
|
|
152
|
+
path.join(temporaryRoot, `operation-${index + 1}`),
|
|
153
|
+
files,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
return {wallMs, operationTimes, totalBytes};
|
|
157
|
+
}
|
|
158
|
+
finally
|
|
159
|
+
{
|
|
160
|
+
await rm(temporaryRoot, {recursive: true, force: true});
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function summarize(values)
|
|
165
|
+
{
|
|
166
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
167
|
+
const median = sorted.length % 2 === 0
|
|
168
|
+
? (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2
|
|
169
|
+
: sorted[Math.floor(sorted.length / 2)];
|
|
170
|
+
const p95 = sorted[Math.ceil(sorted.length * 0.95) - 1];
|
|
171
|
+
return {median_ms: median, p95_ms: p95, samples_ms: values};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function partitionFiles(files, partitionCount)
|
|
175
|
+
{
|
|
176
|
+
const groups = [];
|
|
177
|
+
const baseSize = Math.floor(files.length / partitionCount);
|
|
178
|
+
const remainder = files.length % partitionCount;
|
|
179
|
+
let offset = 0;
|
|
180
|
+
for(let index = 0; index < partitionCount; index++)
|
|
181
|
+
{
|
|
182
|
+
const groupSize = baseSize + (index < remainder ? 1 : 0);
|
|
183
|
+
groups.push(files.slice(offset, offset + groupSize));
|
|
184
|
+
offset += groupSize;
|
|
185
|
+
}
|
|
186
|
+
return groups;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function parseArguments(args)
|
|
190
|
+
{
|
|
191
|
+
const values = new Map();
|
|
192
|
+
for(let index = 0; index < args.length; index++)
|
|
193
|
+
{
|
|
194
|
+
const argument = args[index];
|
|
195
|
+
if(!argument.startsWith('--'))
|
|
196
|
+
{
|
|
197
|
+
throw new TypeError(`Unexpected argument ${argument}`);
|
|
198
|
+
}
|
|
199
|
+
const equals = argument.indexOf('=');
|
|
200
|
+
const key = argument.slice(2, equals === -1 ? undefined : equals);
|
|
201
|
+
const value = equals === -1 ? args[++index] : argument.slice(equals + 1);
|
|
202
|
+
if(!value || value.startsWith('--') || values.has(key))
|
|
203
|
+
{
|
|
204
|
+
throw new TypeError(`Invalid or repeated option --${key}`);
|
|
205
|
+
}
|
|
206
|
+
if(!['archive', 'parallel-counts', 'iterations', 'deflate-buffer-size'].includes(key))
|
|
207
|
+
{
|
|
208
|
+
throw new TypeError(`Unknown option --${key}`);
|
|
209
|
+
}
|
|
210
|
+
values.set(key, value);
|
|
211
|
+
}
|
|
212
|
+
const archivePath = values.get('archive');
|
|
213
|
+
if(!archivePath)
|
|
214
|
+
{
|
|
215
|
+
throw new TypeError(
|
|
216
|
+
'Usage: node bench/multiprocess-extract.cjs --archive <zip> [--parallel-counts 1,2,4] [--iterations 3] [--deflate-buffer-size 262144]',
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
const parallelCounts = [...new Set((values.get('parallel-counts') ?? '1,2,4')
|
|
220
|
+
.split(',')
|
|
221
|
+
.map(value => Number(value)))];
|
|
222
|
+
if(parallelCounts.length === 0
|
|
223
|
+
|| parallelCounts.some(value => !Number.isSafeInteger(value) || value < 1 || value > 32))
|
|
224
|
+
{
|
|
225
|
+
throw new RangeError('--parallel-counts must contain integers between 1 and 32');
|
|
226
|
+
}
|
|
227
|
+
const iterations = Number(values.get('iterations') ?? 3);
|
|
228
|
+
if(!Number.isSafeInteger(iterations) || iterations < 1 || iterations > 20)
|
|
229
|
+
{
|
|
230
|
+
throw new RangeError('--iterations must be an integer between 1 and 20');
|
|
231
|
+
}
|
|
232
|
+
const deflateBufferSize = Number(values.get('deflate-buffer-size') ?? 256 * 1024);
|
|
233
|
+
if(!Number.isSafeInteger(deflateBufferSize)
|
|
234
|
+
|| deflateBufferSize < 4 * 1024
|
|
235
|
+
|| deflateBufferSize > 2 * 1024 * 1024)
|
|
236
|
+
{
|
|
237
|
+
throw new RangeError('--deflate-buffer-size must be between 4096 and 2097152 bytes');
|
|
238
|
+
}
|
|
239
|
+
return {archivePath: path.resolve(archivePath), parallelCounts, iterations, deflateBufferSize};
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function main()
|
|
243
|
+
{
|
|
244
|
+
const {Archive} = require('7zip-native');
|
|
245
|
+
const {path7zc} = require('7zip-bin-full');
|
|
246
|
+
const options = parseArguments(process.argv.slice(2));
|
|
247
|
+
Archive.concurrency(Math.max(...options.parallelCounts));
|
|
248
|
+
const listingArchive = await Archive.open(options.archivePath);
|
|
249
|
+
let files;
|
|
250
|
+
try
|
|
251
|
+
{
|
|
252
|
+
files = (await listingArchive.entries())
|
|
253
|
+
.filter(entry => !entry.isDirectory)
|
|
254
|
+
.map(entry => ({path: entry.path, size: entry.size}));
|
|
255
|
+
}
|
|
256
|
+
finally
|
|
257
|
+
{
|
|
258
|
+
await listingArchive.close();
|
|
259
|
+
}
|
|
260
|
+
if(files.length === 0)
|
|
261
|
+
{
|
|
262
|
+
throw new RangeError('The archive contains no files to extract');
|
|
263
|
+
}
|
|
264
|
+
if(options.parallelCounts.some(count => count > files.length))
|
|
265
|
+
{
|
|
266
|
+
throw new RangeError('The parallel count cannot exceed the number of files in the archive');
|
|
267
|
+
}
|
|
268
|
+
const runtime = process.versions.electron ? 'electron-as-node' : 'node';
|
|
269
|
+
const totalBytes = files.reduce((sum, file) => sum + file.size, 0n);
|
|
270
|
+
const results = [];
|
|
271
|
+
for(const parallelOperations of options.parallelCounts)
|
|
272
|
+
{
|
|
273
|
+
const groups = partitionFiles(files, parallelOperations);
|
|
274
|
+
const samples = {native: [], cli: []};
|
|
275
|
+
await runWave(
|
|
276
|
+
'native', Archive, path7zc, options.archivePath, groups, files.length,
|
|
277
|
+
parallelOperations, options.deflateBufferSize,
|
|
278
|
+
);
|
|
279
|
+
await runWave(
|
|
280
|
+
'cli', Archive, path7zc, options.archivePath, groups, files.length,
|
|
281
|
+
parallelOperations, options.deflateBufferSize,
|
|
282
|
+
);
|
|
283
|
+
for(let iteration = 0; iteration < options.iterations; iteration++)
|
|
284
|
+
{
|
|
285
|
+
const backends = iteration % 2 === 0 ? ['native', 'cli'] : ['cli', 'native'];
|
|
286
|
+
for(const backend of backends)
|
|
287
|
+
{
|
|
288
|
+
samples[backend].push(await runWave(
|
|
289
|
+
backend, Archive, path7zc, options.archivePath, groups, files.length,
|
|
290
|
+
parallelOperations, options.deflateBufferSize,
|
|
291
|
+
));
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
for(const backend of ['native', 'cli'])
|
|
295
|
+
{
|
|
296
|
+
const waves = samples[backend];
|
|
297
|
+
const wall = summarize(waves.map(wave => wave.wallMs));
|
|
298
|
+
const operationTimes = summarize(waves.flatMap(wave => wave.operationTimes));
|
|
299
|
+
const medianWall = wall.median_ms;
|
|
300
|
+
results.push({
|
|
301
|
+
backend: backend === 'native' ? '7zip-native' : '7zip-bin-full (7zzc)',
|
|
302
|
+
parallel_promises: parallelOperations,
|
|
303
|
+
files_per_promise: groups.map(group => group.length),
|
|
304
|
+
total_file_extractions: files.length,
|
|
305
|
+
unique_files: files.length,
|
|
306
|
+
total_output_bytes: totalBytes.toString(),
|
|
307
|
+
median_wall_ms: medianWall,
|
|
308
|
+
p95_wall_ms: wall.p95_ms,
|
|
309
|
+
median_operation_ms: operationTimes.median_ms,
|
|
310
|
+
throughput_mib_per_second: Number(totalBytes) / (medianWall / 1000) / 1024 / 1024,
|
|
311
|
+
samples_ms: wall.samples_ms,
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
console.log(JSON.stringify({
|
|
316
|
+
runtime,
|
|
317
|
+
archive: options.archivePath,
|
|
318
|
+
archive_file_count: files.length,
|
|
319
|
+
iterations: options.iterations,
|
|
320
|
+
deflate_buffer_size: options.deflateBufferSize,
|
|
321
|
+
total_files_per_run: files.length,
|
|
322
|
+
files_partitioned_across_promises: true,
|
|
323
|
+
includes_archive_open: true,
|
|
324
|
+
includes_cli_process_startup: true,
|
|
325
|
+
native_extraction_runs_in_benchmark_process: true,
|
|
326
|
+
cli_binary: path7zc,
|
|
327
|
+
results,
|
|
328
|
+
}, null, 2));
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
main().catch((error) => {
|
|
332
|
+
console.error(error);
|
|
333
|
+
process.exitCode = 1;
|
|
334
|
+
});
|