neozipkit 0.7.2 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -24
- package/WHATS_NEW.md +114 -0
- package/dist/browser/zstd-browser-shim.d.ts +28 -0
- package/dist/browser/zstd-browser-shim.d.ts.map +1 -0
- package/dist/browser/zstd-browser-shim.js +66 -0
- package/dist/browser/zstd-browser-shim.js.map +1 -0
- package/dist/browser-esm/index.js +1786 -41737
- package/dist/browser-esm/index.js.map +4 -4
- package/dist/browser-umd/index.js +1786 -41737
- package/dist/browser-umd/index.js.map +4 -4
- package/dist/browser-umd/index.min.js +3 -21
- package/dist/core/ZipCompress.d.ts +2 -7
- package/dist/core/ZipCompress.d.ts.map +1 -1
- package/dist/core/ZipCompress.js +45 -22
- package/dist/core/ZipCompress.js.map +1 -1
- package/dist/core/ZipDecompress.d.ts +1 -3
- package/dist/core/ZipDecompress.d.ts.map +1 -1
- package/dist/core/ZipDecompress.js +14 -9
- package/dist/core/ZipDecompress.js.map +1 -1
- package/dist/core/encryption/AesCrypto.d.ts +55 -11
- package/dist/core/encryption/AesCrypto.d.ts.map +1 -1
- package/dist/core/encryption/AesCrypto.js +223 -68
- package/dist/core/encryption/AesCrypto.js.map +1 -1
- package/dist/core/encryption/index.d.ts +1 -1
- package/dist/core/encryption/index.d.ts.map +1 -1
- package/dist/core/encryption/index.js +3 -1
- package/dist/core/encryption/index.js.map +1 -1
- package/dist/core/version.js +1 -1
- package/dist/node/LegacyZstd.d.ts +67 -0
- package/dist/node/LegacyZstd.d.ts.map +1 -0
- package/dist/node/LegacyZstd.js +209 -0
- package/dist/node/LegacyZstd.js.map +1 -0
- package/dist/node/ZipCompressNode.d.ts +9 -7
- package/dist/node/ZipCompressNode.d.ts.map +1 -1
- package/dist/node/ZipCompressNode.js +85 -46
- package/dist/node/ZipCompressNode.js.map +1 -1
- package/dist/node/ZipDecompressNode.d.ts +7 -3
- package/dist/node/ZipDecompressNode.d.ts.map +1 -1
- package/dist/node/ZipDecompressNode.js +51 -55
- package/dist/node/ZipDecompressNode.js.map +1 -1
- package/dist/node/ZipkitNode.d.ts +14 -0
- package/dist/node/ZipkitNode.d.ts.map +1 -1
- package/dist/node/ZipkitNode.js +130 -39
- package/dist/node/ZipkitNode.js.map +1 -1
- package/dist/node/ZstdNode.d.ts +39 -0
- package/dist/node/ZstdNode.d.ts.map +1 -0
- package/dist/node/ZstdNode.js +171 -0
- package/dist/node/ZstdNode.js.map +1 -0
- package/dist/node/index.d.ts +3 -0
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +11 -1
- package/dist/node/index.js.map +1 -1
- package/package.json +10 -10
- package/src/browser/zstd-browser-shim.ts +76 -0
- package/src/core/ZipCompress.ts +57 -29
- package/src/core/ZipDecompress.ts +21 -10
- package/src/core/encryption/AesCrypto.ts +265 -86
- package/src/core/encryption/index.ts +1 -1
- package/src/core/version.ts +1 -1
- package/src/node/LegacyZstd.ts +270 -0
- package/src/node/ZipCompressNode.ts +107 -55
- package/src/node/ZipDecompressNode.ts +80 -66
- package/src/node/ZipkitNode.ts +150 -40
- package/src/node/ZstdNode.ts +155 -0
- package/src/node/index.ts +10 -0
- package/dist/core/ZstdManager.d.ts +0 -56
- package/dist/core/ZstdManager.d.ts.map +0 -1
- package/dist/core/ZstdManager.js +0 -144
- package/dist/core/ZstdManager.js.map +0 -1
- package/src/core/ZstdManager.ts +0 -165
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
//
|
|
15
15
|
|
|
16
16
|
const pako = require('pako');
|
|
17
|
-
import { ZstdManager } from '../core/ZstdManager';
|
|
18
17
|
import ZipkitNode from './ZipkitNode';
|
|
18
|
+
import { ZstdNode } from './ZstdNode';
|
|
19
|
+
import { repairLegacyWasmZstdPlaintext } from './LegacyZstd';
|
|
19
20
|
import { Logger } from '../core/components/Logger';
|
|
20
21
|
import ZipEntry from '../core/ZipEntry';
|
|
21
22
|
import Errors from '../core/constants/Errors';
|
|
@@ -125,6 +126,18 @@ export class ZipDecompressNode {
|
|
|
125
126
|
return await this.extractToBufferInternal(fileHandle, entry, options);
|
|
126
127
|
}
|
|
127
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Read the raw compressed payload for an entry (no decrypt / decompress).
|
|
131
|
+
*/
|
|
132
|
+
async readCompressedPayload(entry: ZipEntry): Promise<Buffer> {
|
|
133
|
+
const fileHandle = (this.zipkitNode as any).getFileHandle();
|
|
134
|
+
const chunks: Buffer[] = [];
|
|
135
|
+
for await (const chunk of this.readCompressedDataStream(fileHandle, entry)) {
|
|
136
|
+
chunks.push(chunk);
|
|
137
|
+
}
|
|
138
|
+
return Buffer.concat(chunks);
|
|
139
|
+
}
|
|
140
|
+
|
|
128
141
|
/**
|
|
129
142
|
* Test entry integrity without extracting to disk
|
|
130
143
|
* Validates CRC-32 or SHA-256 hash without writing decompressed data
|
|
@@ -228,19 +241,15 @@ export class ZipDecompressNode {
|
|
|
228
241
|
entry.aesVersion > 0 || entry.cmpMethod === CMP_METHOD.AES_ENCRYPT || entry.neoCryptoAlgorithm > 0;
|
|
229
242
|
|
|
230
243
|
if (isEncrypted && isFullPayloadCrypto) {
|
|
231
|
-
// WinZip AES
|
|
244
|
+
// WinZip AES / NeoEncrypt: stream decrypt (HMAC verified at end of ciphertext)
|
|
232
245
|
const dataStream = this.readCompressedDataStream(fileHandle, entry);
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
|
|
239
|
-
|
|
240
|
-
// Wrap decrypted data as a single-item generator for the decompression pipeline
|
|
246
|
+
const decrypted = AesCrypto.decryptStream(
|
|
247
|
+
(this.zipkitNode as any).password,
|
|
248
|
+
entry.compressedSize,
|
|
249
|
+
dataStream
|
|
250
|
+
);
|
|
241
251
|
const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
|
|
242
|
-
|
|
243
|
-
await this.unCompressToFile(singleChunk(), entry, writeStream, {
|
|
252
|
+
await this.unCompressToFile(decrypted, entry, writeStream, {
|
|
244
253
|
skipHashCheck: skipHash,
|
|
245
254
|
onProgress: options?.onProgress,
|
|
246
255
|
outputPath
|
|
@@ -297,16 +306,13 @@ export class ZipDecompressNode {
|
|
|
297
306
|
|
|
298
307
|
if (isEncrypted && isFullPayloadCrypto) {
|
|
299
308
|
const dataStream = this.readCompressedDataStream(fileHandle, entry);
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
|
|
306
|
-
|
|
309
|
+
const decrypted = AesCrypto.decryptStream(
|
|
310
|
+
(this.zipkitNode as any).password,
|
|
311
|
+
entry.compressedSize,
|
|
312
|
+
dataStream
|
|
313
|
+
);
|
|
307
314
|
const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
|
|
308
|
-
|
|
309
|
-
return await this.unCompressToBuffer(singleChunk(), entry, {
|
|
315
|
+
return await this.unCompressToBuffer(decrypted, entry, {
|
|
310
316
|
skipHashCheck: skipHash,
|
|
311
317
|
onProgress: options?.onProgress
|
|
312
318
|
});
|
|
@@ -357,16 +363,13 @@ export class ZipDecompressNode {
|
|
|
357
363
|
|
|
358
364
|
if (isEncrypted && isFullPayloadCrypto) {
|
|
359
365
|
const dataStream = this.readCompressedDataStream(fileHandle, entry);
|
|
360
|
-
const
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
|
|
366
|
-
|
|
366
|
+
const decrypted = AesCrypto.decryptStream(
|
|
367
|
+
(this.zipkitNode as any).password,
|
|
368
|
+
entry.compressedSize,
|
|
369
|
+
dataStream
|
|
370
|
+
);
|
|
367
371
|
const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
|
|
368
|
-
|
|
369
|
-
return await this.unCompressToTest(singleChunk(), entry, {
|
|
372
|
+
return await this.unCompressToTest(decrypted, entry, {
|
|
370
373
|
skipHashCheck: skipHash,
|
|
371
374
|
onProgress: options?.onProgress
|
|
372
375
|
});
|
|
@@ -662,8 +665,8 @@ export class ZipDecompressNode {
|
|
|
662
665
|
// Use pako streaming inflate - maintains state across chunks
|
|
663
666
|
yield* this.inflateStream(compressedStream);
|
|
664
667
|
} else if (method === CMP_METHOD.ZSTD) {
|
|
665
|
-
//
|
|
666
|
-
yield* this.zstdDecompressStream(compressedStream);
|
|
668
|
+
// Native Node zstd; bound output to entry size (repairs legacy WASM +18 padding)
|
|
669
|
+
yield* this.zstdDecompressStream(compressedStream, entry?.uncompressedSize ?? 0);
|
|
667
670
|
} else {
|
|
668
671
|
throw new Error(`Unsupported compression method: ${method}`);
|
|
669
672
|
}
|
|
@@ -707,36 +710,32 @@ export class ZipDecompressNode {
|
|
|
707
710
|
}
|
|
708
711
|
|
|
709
712
|
/**
|
|
710
|
-
* Streaming ZSTD decompression
|
|
713
|
+
* Streaming ZSTD decompression via Node zlib (native).
|
|
714
|
+
* When maxOut > 0, truncates output (legacy WASM emitted +18 zero bytes).
|
|
711
715
|
*/
|
|
712
716
|
private async *zstdDecompressStream(
|
|
713
|
-
compressedStream: AsyncGenerator<Buffer
|
|
717
|
+
compressedStream: AsyncGenerator<Buffer>,
|
|
718
|
+
maxOut: number = 0
|
|
714
719
|
): AsyncGenerator<Buffer> {
|
|
715
|
-
// ZSTD is guaranteed to be initialized via factory method
|
|
716
|
-
|
|
717
|
-
// Collect all compressed chunks first (ZSTD needs complete data)
|
|
718
|
-
const compressedChunks: Buffer[] = [];
|
|
719
|
-
for await (const chunk of compressedStream) {
|
|
720
|
-
compressedChunks.push(chunk);
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
const compressedData = Buffer.concat(compressedChunks);
|
|
724
|
-
|
|
725
720
|
try {
|
|
726
|
-
|
|
727
|
-
const
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
721
|
+
let remaining = maxOut > 0 ? maxOut : Number.POSITIVE_INFINITY;
|
|
722
|
+
for await (const chunk of ZstdNode.decompressStream(compressedStream)) {
|
|
723
|
+
if (remaining <= 0) {
|
|
724
|
+
break;
|
|
725
|
+
}
|
|
726
|
+
if (chunk.length <= remaining) {
|
|
727
|
+
remaining -= chunk.length;
|
|
728
|
+
yield chunk;
|
|
729
|
+
} else {
|
|
730
|
+
yield chunk.subarray(0, remaining);
|
|
731
|
+
remaining = 0;
|
|
732
|
+
break;
|
|
733
|
+
}
|
|
737
734
|
}
|
|
738
735
|
} catch (error) {
|
|
739
|
-
throw new Error(
|
|
736
|
+
throw new Error(
|
|
737
|
+
`ZSTD streaming decompression failed: ${error instanceof Error ? error.message : String(error)}`
|
|
738
|
+
);
|
|
740
739
|
}
|
|
741
740
|
}
|
|
742
741
|
|
|
@@ -750,20 +749,35 @@ export class ZipDecompressNode {
|
|
|
750
749
|
}
|
|
751
750
|
|
|
752
751
|
/**
|
|
753
|
-
* Zstd decompress
|
|
754
|
-
* Internal method only
|
|
752
|
+
* Zstd decompress via Node zlib (native), repairing legacy WASM +18 padding when present.
|
|
755
753
|
*/
|
|
756
|
-
private async zstdDecompressSync(data: Buffer): Promise<Buffer> {
|
|
754
|
+
private async zstdDecompressSync(data: Buffer, entry?: ZipEntry): Promise<Buffer> {
|
|
757
755
|
this.log(`zstdDecompressSync() called with ${data.length} bytes`);
|
|
758
|
-
|
|
756
|
+
|
|
759
757
|
try {
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
758
|
+
let decompressed = await ZstdNode.decompress(data);
|
|
759
|
+
if (entry && entry.uncompressedSize > 0) {
|
|
760
|
+
const { repaired, wasLegacy } = repairLegacyWasmZstdPlaintext(
|
|
761
|
+
decompressed,
|
|
762
|
+
entry.uncompressedSize,
|
|
763
|
+
entry.crc
|
|
764
|
+
);
|
|
765
|
+
if (wasLegacy) {
|
|
766
|
+
this.log(
|
|
767
|
+
`Legacy WASM zstd padding repaired: ${decompressed.length} -> ${repaired.length} bytes`
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
decompressed = repaired;
|
|
771
|
+
}
|
|
772
|
+
this.log(
|
|
773
|
+
`ZSTD decompression successful: ${data.length} bytes -> ${decompressed.length} bytes`
|
|
774
|
+
);
|
|
775
|
+
return decompressed;
|
|
764
776
|
} catch (error) {
|
|
765
777
|
this.log(`ZSTD decompression failed: ${error}`);
|
|
766
|
-
throw new Error(
|
|
778
|
+
throw new Error(
|
|
779
|
+
`ZSTD decompression failed: ${error instanceof Error ? error.message : String(error)}`
|
|
780
|
+
);
|
|
767
781
|
}
|
|
768
782
|
}
|
|
769
783
|
|
|
@@ -794,7 +808,7 @@ export class ZipDecompressNode {
|
|
|
794
808
|
} else if (method === CMP_METHOD.DEFLATED) {
|
|
795
809
|
outBuf = this.inflate(compressedData);
|
|
796
810
|
} else if (method === CMP_METHOD.ZSTD) {
|
|
797
|
-
outBuf = await this.zstdDecompressSync(compressedData);
|
|
811
|
+
outBuf = await this.zstdDecompressSync(compressedData, entry);
|
|
798
812
|
} else {
|
|
799
813
|
throw new Error(`Unsupported compression method: ${method}`);
|
|
800
814
|
}
|
package/src/node/ZipkitNode.ts
CHANGED
|
@@ -19,6 +19,11 @@ import {
|
|
|
19
19
|
ENCRYPT_HDR_SIZE
|
|
20
20
|
} from '../core/constants/Headers';
|
|
21
21
|
import { NEO_CRYPTO_ALGORITHM_AES256_V1 } from '../core/encryption/NeoCrypto';
|
|
22
|
+
import { AesCrypto } from '../core/encryption/AesCrypto';
|
|
23
|
+
import {
|
|
24
|
+
detectLegacyWasmZstd,
|
|
25
|
+
type LegacyZstdDetectResult,
|
|
26
|
+
} from './LegacyZstd';
|
|
22
27
|
import * as fs from 'fs';
|
|
23
28
|
import * as path from 'path';
|
|
24
29
|
import { minimatch } from 'minimatch';
|
|
@@ -345,6 +350,75 @@ export default class ZipkitNode extends Zipkit {
|
|
|
345
350
|
};
|
|
346
351
|
}
|
|
347
352
|
|
|
353
|
+
/**
|
|
354
|
+
* Scan the loaded archive (or `archivePath`) for entries compressed with the
|
|
355
|
+
* legacy NeoZipKit WASM zstd codec (`@oneidentity/zstd-js`).
|
|
356
|
+
*
|
|
357
|
+
* Encrypted entries are skipped unless you only need a header check after
|
|
358
|
+
* decrypting externally — this API inspects raw stored payloads.
|
|
359
|
+
*/
|
|
360
|
+
async detectLegacyZstdEntries(
|
|
361
|
+
archivePath?: string,
|
|
362
|
+
options?: { verify?: boolean }
|
|
363
|
+
): Promise<Array<{ entry: ZipEntry; detection: LegacyZstdDetectResult }>> {
|
|
364
|
+
if (archivePath && !this.filePath) {
|
|
365
|
+
await this.loadZipFile(archivePath);
|
|
366
|
+
}
|
|
367
|
+
if (!this.filePath) {
|
|
368
|
+
throw new Error('Archive not loaded. Call loadZipFile() first or provide archivePath parameter.');
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const verify = options?.verify !== false;
|
|
372
|
+
const results: Array<{ entry: ZipEntry; detection: LegacyZstdDetectResult }> = [];
|
|
373
|
+
const entries = this.getDirectory();
|
|
374
|
+
|
|
375
|
+
for (const entry of entries) {
|
|
376
|
+
if (entry.isDirectory) {
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
const method =
|
|
380
|
+
entry.cmpMethod === CMP_METHOD.AES_ENCRYPT && entry.realCmpMethod >= 0
|
|
381
|
+
? entry.realCmpMethod
|
|
382
|
+
: entry.cmpMethod;
|
|
383
|
+
if (method !== CMP_METHOD.ZSTD) {
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (entry.isEncrypted || (entry.bitFlags & GP_FLAG.ENCRYPTED)) {
|
|
387
|
+
results.push({
|
|
388
|
+
entry,
|
|
389
|
+
detection: {
|
|
390
|
+
isLegacyWasm: false,
|
|
391
|
+
confidence: 'none',
|
|
392
|
+
frameContentSize: null,
|
|
393
|
+
expectedUncompressedSize: entry.uncompressedSize,
|
|
394
|
+
paddingBytes: 18,
|
|
395
|
+
reason: 'Encrypted zstd entry — decrypt before detecting legacy WASM frames',
|
|
396
|
+
},
|
|
397
|
+
});
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
const compressed = await this.getZipDecompressNode().readCompressedPayload(entry);
|
|
402
|
+
|
|
403
|
+
if (!verify) {
|
|
404
|
+
const detection = await detectLegacyWasmZstd(compressed, entry.uncompressedSize, {
|
|
405
|
+
verify: false,
|
|
406
|
+
crc: entry.crc,
|
|
407
|
+
});
|
|
408
|
+
results.push({ entry, detection });
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const detection = await detectLegacyWasmZstd(compressed, entry.uncompressedSize, {
|
|
413
|
+
verify: true,
|
|
414
|
+
crc: entry.crc,
|
|
415
|
+
});
|
|
416
|
+
results.push({ entry, detection });
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return results;
|
|
420
|
+
}
|
|
421
|
+
|
|
348
422
|
/**
|
|
349
423
|
* Test entry integrity without extracting to disk
|
|
350
424
|
* Validates CRC-32 or SHA-256 hash without writing decompressed data
|
|
@@ -578,8 +652,12 @@ export default class ZipkitNode extends Zipkit {
|
|
|
578
652
|
// For AES encryption, pre-configure the entry so the initial local header
|
|
579
653
|
// includes the 0x9901 extra field and method 99. The actual compression
|
|
580
654
|
// method is preserved in realCmpMethod for the AES extra field.
|
|
581
|
-
|
|
582
|
-
const
|
|
655
|
+
// Default password encryption is WinZip AES-256 (matches compressData).
|
|
656
|
+
const encMethod = options?.password
|
|
657
|
+
? (options.encryptionMethod || 'aes256')
|
|
658
|
+
: undefined;
|
|
659
|
+
const isAesRequested = encMethod === 'aes256';
|
|
660
|
+
const isNeoRequested = encMethod === 'neo-aes256';
|
|
583
661
|
if (isAesRequested) {
|
|
584
662
|
entry.cmpMethod = CMP_METHOD.AES_ENCRYPT;
|
|
585
663
|
entry.aesVersion = 1;
|
|
@@ -615,55 +693,87 @@ export default class ZipkitNode extends Zipkit {
|
|
|
615
693
|
});
|
|
616
694
|
});
|
|
617
695
|
|
|
618
|
-
// Step 3: Compress file and write data
|
|
696
|
+
// Step 3: Compress file and write data (chunked when large).
|
|
697
|
+
// ZipCrypto still uses the buffer path (legacy header needs CRC up front).
|
|
698
|
+
// AES / NeoEncrypt use AesEncryptor so ciphertext is written in chunks.
|
|
619
699
|
const bufferSize = options?.bufferSize || this.getBufferSize();
|
|
620
|
-
const
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
const onOutputBuffer = async (data: Buffer) => {
|
|
631
|
-
await new Promise<void>((resolve, reject) => {
|
|
632
|
-
writer.outputStream.write(data, (error) => {
|
|
633
|
-
if (error) {
|
|
634
|
-
reject(error);
|
|
635
|
-
} else {
|
|
636
|
-
writer.currentPosition += data.length;
|
|
637
|
-
if (callbacks?.onProgress) {
|
|
638
|
-
callbacks.onProgress(entry, data.length);
|
|
639
|
-
}
|
|
640
|
-
resolve();
|
|
641
|
-
}
|
|
642
|
-
});
|
|
643
|
-
});
|
|
644
|
-
};
|
|
645
|
-
|
|
646
|
-
// compressFileStream will set entry.compressedSize and entry.crc
|
|
647
|
-
await this.compressFileStream(filePath, entry, options, onOutputBuffer);
|
|
648
|
-
} else {
|
|
649
|
-
// Use regular buffer compression for small files
|
|
650
|
-
// compressFile will set entry.compressedSize and entry.crc
|
|
651
|
-
const compressedData = await this.compressFile(filePath, entry, options);
|
|
652
|
-
|
|
653
|
-
// Write compressed data to file
|
|
700
|
+
const useZipCrypto = encMethod === 'zipcrypto';
|
|
701
|
+
const shouldUseChunked =
|
|
702
|
+
!!entry.uncompressedSize &&
|
|
703
|
+
entry.uncompressedSize > bufferSize &&
|
|
704
|
+
!useZipCrypto;
|
|
705
|
+
|
|
706
|
+
const writeChunk = async (data: Buffer): Promise<void> => {
|
|
707
|
+
if (data.length === 0) {
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
654
710
|
await new Promise<void>((resolve, reject) => {
|
|
655
|
-
writer.outputStream.write(
|
|
711
|
+
writer.outputStream.write(data, (error) => {
|
|
656
712
|
if (error) {
|
|
657
713
|
reject(error);
|
|
658
714
|
} else {
|
|
659
|
-
writer.currentPosition +=
|
|
715
|
+
writer.currentPosition += data.length;
|
|
660
716
|
if (callbacks?.onProgress) {
|
|
661
|
-
callbacks.onProgress(entry,
|
|
717
|
+
callbacks.onProgress(entry, data.length);
|
|
662
718
|
}
|
|
663
719
|
resolve();
|
|
664
720
|
}
|
|
665
721
|
});
|
|
666
722
|
});
|
|
723
|
+
};
|
|
724
|
+
|
|
725
|
+
if (shouldUseChunked) {
|
|
726
|
+
const useAesStream = isAesRequested || isNeoRequested;
|
|
727
|
+
const encryptor = useAesStream
|
|
728
|
+
? AesCrypto.createEncryptor(options!.password!)
|
|
729
|
+
: null;
|
|
730
|
+
let streamedBytes = 0;
|
|
731
|
+
|
|
732
|
+
if (encryptor) {
|
|
733
|
+
const hdr = encryptor.header();
|
|
734
|
+
await writeChunk(hdr);
|
|
735
|
+
streamedBytes += hdr.length;
|
|
736
|
+
entry.isEncrypted = true;
|
|
737
|
+
entry.bitFlags |= GP_FLAG.ENCRYPTED;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// Compress without applying buffer encrypt — we stream-encrypt below.
|
|
741
|
+
const streamOptions: CompressOptions = encryptor
|
|
742
|
+
? { ...options, password: undefined, encryptionMethod: undefined }
|
|
743
|
+
: { ...options };
|
|
744
|
+
|
|
745
|
+
const onOutputBuffer = async (data: Buffer) => {
|
|
746
|
+
const out = encryptor ? encryptor.update(data) : data;
|
|
747
|
+
await writeChunk(out);
|
|
748
|
+
streamedBytes += out.length;
|
|
749
|
+
};
|
|
750
|
+
|
|
751
|
+
await this.compressFileStream(filePath, entry, streamOptions, onOutputBuffer);
|
|
752
|
+
|
|
753
|
+
if (encryptor) {
|
|
754
|
+
const auth = encryptor.finish();
|
|
755
|
+
await writeChunk(auth);
|
|
756
|
+
streamedBytes += auth.length;
|
|
757
|
+
entry.compressedSize = streamedBytes;
|
|
758
|
+
entry.isEncrypted = true;
|
|
759
|
+
entry.bitFlags |= GP_FLAG.ENCRYPTED;
|
|
760
|
+
// compressFileStream overwrites cmpMethod with the real codec — restore AES/Neo metadata
|
|
761
|
+
if (isAesRequested) {
|
|
762
|
+
entry.realCmpMethod = realCmpMethod;
|
|
763
|
+
entry.cmpMethod = CMP_METHOD.AES_ENCRYPT;
|
|
764
|
+
entry.aesVersion = 1;
|
|
765
|
+
entry.aesStrength = 3;
|
|
766
|
+
} else if (isNeoRequested) {
|
|
767
|
+
entry.cmpMethod = realCmpMethod;
|
|
768
|
+
entry.neoCryptoPayloadVersion = 1;
|
|
769
|
+
entry.neoCryptoAlgorithm = NEO_CRYPTO_ALGORITHM_AES256_V1;
|
|
770
|
+
entry.neoCryptoFlags = 0;
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
} else {
|
|
774
|
+
// Small files / ZipCrypto: buffer compression (may encrypt inside compressFile)
|
|
775
|
+
const compressedData = await this.compressFile(filePath, entry, options);
|
|
776
|
+
await writeChunk(compressedData);
|
|
667
777
|
}
|
|
668
778
|
|
|
669
779
|
// Step 4: Patch local header in-place with final compressed size, CRC, and flags
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ZstdNode - Native Node.js Zstandard via zlib (streaming / chunked)
|
|
3
|
+
*
|
|
4
|
+
* This is the only zstd backend in NeoZipKit. There is no WASM codec.
|
|
5
|
+
* Requires Node.js with `zlib.createZstdCompress` (Node >= 22.15 or >= 23.8).
|
|
6
|
+
* Browser builds alias this module to a stub that throws.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import * as zlib from 'zlib';
|
|
10
|
+
import { Readable } from 'stream';
|
|
11
|
+
import { promisify } from 'util';
|
|
12
|
+
|
|
13
|
+
const zstdCompressAsync = promisify(zlib.zstdCompress);
|
|
14
|
+
const zstdDecompressAsync = promisify(zlib.zstdDecompress);
|
|
15
|
+
|
|
16
|
+
export function isNativeZstdAvailable(): boolean {
|
|
17
|
+
return (
|
|
18
|
+
typeof zlib.createZstdCompress === 'function' &&
|
|
19
|
+
typeof zlib.createZstdDecompress === 'function' &&
|
|
20
|
+
typeof zlib.zstdCompress === 'function' &&
|
|
21
|
+
typeof zlib.zstdDecompress === 'function'
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function assertNativeZstd(): void {
|
|
26
|
+
if (!isNativeZstdAvailable()) {
|
|
27
|
+
throw new Error(
|
|
28
|
+
'Native zstd requires Node.js with zlib ZSTD support ' +
|
|
29
|
+
'(Node.js >= 22.15 or >= 23.8).'
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Map NeoZipKit level 1–9 to a zstd compression level (1–19). */
|
|
35
|
+
export function mapCompressionLevel(level: number): number {
|
|
36
|
+
return Math.min(Math.max(1, Math.floor(level * 2.1)), 19);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function compressOptions(level: number): zlib.ZstdOptions {
|
|
40
|
+
return {
|
|
41
|
+
params: {
|
|
42
|
+
[zlib.constants.ZSTD_c_compressionLevel]: mapCompressionLevel(level),
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function createCompressTransform(level: number = 6): zlib.ZstdCompress {
|
|
48
|
+
assertNativeZstd();
|
|
49
|
+
return zlib.createZstdCompress(compressOptions(level));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function createDecompressTransform(): zlib.ZstdDecompress {
|
|
53
|
+
assertNativeZstd();
|
|
54
|
+
return zlib.createZstdDecompress();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export async function compress(data: Buffer, level: number = 6): Promise<Buffer> {
|
|
58
|
+
assertNativeZstd();
|
|
59
|
+
const result = await zstdCompressAsync(data, compressOptions(level));
|
|
60
|
+
return Buffer.from(result);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function decompress(data: Buffer): Promise<Buffer> {
|
|
64
|
+
assertNativeZstd();
|
|
65
|
+
const result = await zstdDecompressAsync(data);
|
|
66
|
+
return Buffer.from(result);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function compressSync(data: Buffer, level: number = 6): Buffer {
|
|
70
|
+
assertNativeZstd();
|
|
71
|
+
return zlib.zstdCompressSync(data, compressOptions(level));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function decompressSync(data: Buffer): Buffer {
|
|
75
|
+
assertNativeZstd();
|
|
76
|
+
return zlib.zstdDecompressSync(data);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Stream-compress plaintext from a chunked reader (primary path for all sizes).
|
|
81
|
+
*/
|
|
82
|
+
export async function compressChunks(
|
|
83
|
+
readChunk: (position: number, size: number) => Buffer,
|
|
84
|
+
totalSize: number,
|
|
85
|
+
chunkSize: number,
|
|
86
|
+
level: number,
|
|
87
|
+
onPlainChunk?: (chunk: Buffer) => void,
|
|
88
|
+
onCompressedChunk?: (chunk: Buffer) => void | Promise<void>
|
|
89
|
+
): Promise<number> {
|
|
90
|
+
async function* plainChunks(): AsyncGenerator<Buffer> {
|
|
91
|
+
let position = 0;
|
|
92
|
+
while (position < totalSize) {
|
|
93
|
+
const size = Math.min(chunkSize, totalSize - position);
|
|
94
|
+
const chunk = readChunk(position, size);
|
|
95
|
+
onPlainChunk?.(chunk);
|
|
96
|
+
position += chunk.length;
|
|
97
|
+
yield chunk;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const transform = createCompressTransform(level);
|
|
102
|
+
const readable = Readable.from(plainChunks());
|
|
103
|
+
readable.pipe(transform);
|
|
104
|
+
|
|
105
|
+
let compressedSize = 0;
|
|
106
|
+
try {
|
|
107
|
+
for await (const out of transform) {
|
|
108
|
+
const buf = Buffer.isBuffer(out) ? out : Buffer.from(out as Uint8Array);
|
|
109
|
+
compressedSize += buf.length;
|
|
110
|
+
if (onCompressedChunk) {
|
|
111
|
+
await onCompressedChunk(buf);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
} finally {
|
|
115
|
+
readable.destroy();
|
|
116
|
+
transform.destroy();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return compressedSize;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Decompress an async iterable of compressed chunks, yielding plaintext buffers.
|
|
124
|
+
*/
|
|
125
|
+
export async function* decompressStream(
|
|
126
|
+
compressedStream: AsyncIterable<Buffer>
|
|
127
|
+
): AsyncGenerator<Buffer> {
|
|
128
|
+
const transform = createDecompressTransform();
|
|
129
|
+
const readable = Readable.from(compressedStream);
|
|
130
|
+
readable.pipe(transform);
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
for await (const chunk of transform) {
|
|
134
|
+
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
readable.destroy();
|
|
138
|
+
transform.destroy();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export const ZstdNode = {
|
|
143
|
+
isNativeZstdAvailable,
|
|
144
|
+
mapCompressionLevel,
|
|
145
|
+
createCompressTransform,
|
|
146
|
+
createDecompressTransform,
|
|
147
|
+
compress,
|
|
148
|
+
decompress,
|
|
149
|
+
compressSync,
|
|
150
|
+
decompressSync,
|
|
151
|
+
compressChunks,
|
|
152
|
+
decompressStream,
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export default ZstdNode;
|
package/src/node/index.ts
CHANGED
|
@@ -21,6 +21,16 @@ export default ZipkitNodeDefault;
|
|
|
21
21
|
export type { ZipFileWriter } from './ZipkitNode';
|
|
22
22
|
export { ZipCompressNode } from './ZipCompressNode';
|
|
23
23
|
export { ZipDecompressNode } from './ZipDecompressNode';
|
|
24
|
+
export { ZstdNode, isNativeZstdAvailable } from './ZstdNode';
|
|
25
|
+
export {
|
|
26
|
+
LegacyZstd,
|
|
27
|
+
LEGACY_WASM_ZSTD_PADDING,
|
|
28
|
+
parseZstdFrameContentSize,
|
|
29
|
+
looksLikeLegacyWasmZstd,
|
|
30
|
+
detectLegacyWasmZstd,
|
|
31
|
+
repairLegacyWasmZstdPlaintext,
|
|
32
|
+
} from './LegacyZstd';
|
|
33
|
+
export type { LegacyZstdDetectResult, LegacyZstdConfidence, DetectLegacyZstdOptions } from './LegacyZstd';
|
|
24
34
|
|
|
25
35
|
// Efficient ZIP copying using ZipEntry directly
|
|
26
36
|
export { ZipCopyNode } from './ZipCopyNode';
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* ZstdManager - Global Zstd Codec Manager
|
|
3
|
-
*
|
|
4
|
-
* This manager ensures safe usage of the Zstd WASM module across multiple instances.
|
|
5
|
-
*
|
|
6
|
-
* ## Why This Is Necessary
|
|
7
|
-
*
|
|
8
|
-
* The `@oneidentity/zstd-js` library uses a shared WASM module internally.
|
|
9
|
-
* While `ZstdInit()` returns different object references, they all share the same
|
|
10
|
-
* `ZstdSimple` implementation and WASM memory/heap.
|
|
11
|
-
*
|
|
12
|
-
* This means concurrent or improperly managed sequential operations can cause
|
|
13
|
-
* memory corruption because they're all operating on the same underlying WASM state.
|
|
14
|
-
*
|
|
15
|
-
* ## How It Works
|
|
16
|
-
*
|
|
17
|
-
* 1. Single initialization: WASM module is initialized once globally
|
|
18
|
-
* 2. Operation queuing: All compress/decompress operations are queued
|
|
19
|
-
* 3. Sequential execution: JavaScript's single-threaded nature ensures safe execution
|
|
20
|
-
* 4. Automatic initialization: Lazy initialization on first use
|
|
21
|
-
*
|
|
22
|
-
* ## Usage
|
|
23
|
-
*
|
|
24
|
-
* ```typescript
|
|
25
|
-
* import { ZstdManager } from './ZstdManager';
|
|
26
|
-
*
|
|
27
|
-
* // Compress
|
|
28
|
-
* const compressed = await ZstdManager.compress(data, level);
|
|
29
|
-
*
|
|
30
|
-
* // Decompress
|
|
31
|
-
* const decompressed = await ZstdManager.decompress(compressedData);
|
|
32
|
-
* ```
|
|
33
|
-
*/
|
|
34
|
-
/**
|
|
35
|
-
* Global Zstd Manager
|
|
36
|
-
* Use this for all Zstd compression/decompression operations
|
|
37
|
-
*/
|
|
38
|
-
export declare const ZstdManager: {
|
|
39
|
-
/**
|
|
40
|
-
* Compress data using Zstd
|
|
41
|
-
* Operations are automatically queued to prevent interference
|
|
42
|
-
*/
|
|
43
|
-
compress: (data: Uint8Array, level?: number) => Promise<Uint8Array<ArrayBufferLike>>;
|
|
44
|
-
/**
|
|
45
|
-
* Decompress data using Zstd
|
|
46
|
-
* Operations are automatically queued to prevent interference
|
|
47
|
-
*/
|
|
48
|
-
decompress: (data: Uint8Array | Buffer) => Promise<Uint8Array<ArrayBufferLike>>;
|
|
49
|
-
/**
|
|
50
|
-
* Reset the manager (for testing)
|
|
51
|
-
* @internal
|
|
52
|
-
*/
|
|
53
|
-
reset: () => void;
|
|
54
|
-
};
|
|
55
|
-
export default ZstdManager;
|
|
56
|
-
//# sourceMappingURL=ZstdManager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ZstdManager.d.ts","sourceRoot":"","sources":["../../src/core/ZstdManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAyGH;;;GAGG;AACH,eAAO,MAAM,WAAW;IACtB;;;OAGG;qBACc,UAAU,UAAU,MAAM;IAG3C;;;OAGG;uBACgB,UAAU,GAAG,MAAM;IAGtC;;;OAGG;;CAEJ,CAAC;AAEF,eAAe,WAAW,CAAC"}
|