neozipkit 0.8.0 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NEOZIP_APPNOTE.md +527 -0
- package/README.md +57 -21
- package/WHATS_NEW.md +146 -0
- package/dist/browser/ZipkitBrowser.d.ts.map +1 -1
- package/dist/browser/ZipkitBrowser.js +4 -6
- package/dist/browser/ZipkitBrowser.js.map +1 -1
- package/dist/browser/zstd-browser-shim.d.ts +28 -0
- package/dist/browser/zstd-browser-shim.d.ts.map +1 -0
- package/dist/browser/zstd-browser-shim.js +66 -0
- package/dist/browser/zstd-browser-shim.js.map +1 -0
- package/dist/browser-esm/index.js +2110 -41755
- package/dist/browser-esm/index.js.map +4 -4
- package/dist/browser-umd/index.js +2110 -41755
- package/dist/browser-umd/index.js.map +4 -4
- package/dist/browser-umd/index.min.js +3 -21
- package/dist/core/ZipCompress.d.ts +2 -7
- package/dist/core/ZipCompress.d.ts.map +1 -1
- package/dist/core/ZipCompress.js +57 -33
- package/dist/core/ZipCompress.js.map +1 -1
- package/dist/core/ZipDecompress.d.ts +1 -3
- package/dist/core/ZipDecompress.d.ts.map +1 -1
- package/dist/core/ZipDecompress.js +27 -17
- package/dist/core/ZipDecompress.js.map +1 -1
- package/dist/core/ZipEntry.d.ts +6 -0
- package/dist/core/ZipEntry.d.ts.map +1 -1
- package/dist/core/ZipEntry.js +7 -1
- package/dist/core/ZipEntry.js.map +1 -1
- package/dist/core/Zipkit.d.ts +43 -11
- package/dist/core/Zipkit.d.ts.map +1 -1
- package/dist/core/Zipkit.js +106 -63
- package/dist/core/Zipkit.js.map +1 -1
- package/dist/core/components/HashCalculator.d.ts +22 -1
- package/dist/core/components/HashCalculator.d.ts.map +1 -1
- package/dist/core/components/HashCalculator.js +49 -6
- package/dist/core/components/HashCalculator.js.map +1 -1
- package/dist/core/components/Util.d.ts +3 -4
- package/dist/core/components/Util.d.ts.map +1 -1
- package/dist/core/components/Util.js +10 -8
- package/dist/core/components/Util.js.map +1 -1
- package/dist/core/constants/Headers.d.ts +1 -3
- package/dist/core/constants/Headers.d.ts.map +1 -1
- package/dist/core/constants/Headers.js +14 -5
- package/dist/core/constants/Headers.js.map +1 -1
- package/dist/core/constants/MetaPaths.d.ts +38 -0
- package/dist/core/constants/MetaPaths.d.ts.map +1 -0
- package/dist/core/constants/MetaPaths.js +79 -0
- package/dist/core/constants/MetaPaths.js.map +1 -0
- package/dist/core/encryption/AesCrypto.d.ts +55 -11
- package/dist/core/encryption/AesCrypto.d.ts.map +1 -1
- package/dist/core/encryption/AesCrypto.js +223 -68
- package/dist/core/encryption/AesCrypto.js.map +1 -1
- package/dist/core/encryption/index.d.ts +1 -1
- package/dist/core/encryption/index.d.ts.map +1 -1
- package/dist/core/encryption/index.js +3 -1
- package/dist/core/encryption/index.js.map +1 -1
- package/dist/core/index.d.ts +2 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +14 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/merkle/MerkleRoot.d.ts +109 -0
- package/dist/core/merkle/MerkleRoot.d.ts.map +1 -0
- package/dist/core/merkle/MerkleRoot.js +275 -0
- package/dist/core/merkle/MerkleRoot.js.map +1 -0
- package/dist/core/version.js +1 -1
- package/dist/node/LegacyZstd.d.ts +67 -0
- package/dist/node/LegacyZstd.d.ts.map +1 -0
- package/dist/node/LegacyZstd.js +209 -0
- package/dist/node/LegacyZstd.js.map +1 -0
- package/dist/node/ZipCompressNode.d.ts +9 -7
- package/dist/node/ZipCompressNode.d.ts.map +1 -1
- package/dist/node/ZipCompressNode.js +105 -75
- package/dist/node/ZipCompressNode.js.map +1 -1
- package/dist/node/ZipDecompressNode.d.ts +7 -3
- package/dist/node/ZipDecompressNode.d.ts.map +1 -1
- package/dist/node/ZipDecompressNode.js +68 -56
- package/dist/node/ZipDecompressNode.js.map +1 -1
- package/dist/node/ZipkitNode.d.ts +23 -0
- package/dist/node/ZipkitNode.d.ts.map +1 -1
- package/dist/node/ZipkitNode.js +195 -41
- package/dist/node/ZipkitNode.js.map +1 -1
- package/dist/node/ZstdNode.d.ts +39 -0
- package/dist/node/ZstdNode.d.ts.map +1 -0
- package/dist/node/ZstdNode.js +171 -0
- package/dist/node/ZstdNode.js.map +1 -0
- package/dist/node/index.d.ts +3 -0
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +11 -1
- package/dist/node/index.js.map +1 -1
- package/package.json +4 -3
- package/src/browser/ZipkitBrowser.ts +4 -6
- package/src/browser/zstd-browser-shim.ts +76 -0
- package/src/core/ZipCompress.ts +71 -42
- package/src/core/ZipDecompress.ts +35 -18
- package/src/core/ZipEntry.ts +8 -1
- package/src/core/Zipkit.ts +149 -76
- package/src/core/components/HashCalculator.ts +58 -6
- package/src/core/components/Util.ts +16 -7
- package/src/core/constants/Headers.ts +14 -4
- package/src/core/constants/MetaPaths.ts +86 -0
- package/src/core/encryption/AesCrypto.ts +265 -86
- package/src/core/encryption/index.ts +1 -1
- package/src/core/index.ts +14 -0
- package/src/core/merkle/MerkleRoot.ts +312 -0
- package/src/core/version.ts +1 -1
- package/src/node/LegacyZstd.ts +270 -0
- package/src/node/ZipCompressNode.ts +127 -84
- package/src/node/ZipDecompressNode.ts +93 -67
- package/src/node/ZipkitNode.ts +230 -42
- package/src/node/ZstdNode.ts +155 -0
- package/src/node/index.ts +10 -0
- package/dist/core/ZstdManager.d.ts +0 -56
- package/dist/core/ZstdManager.d.ts.map +0 -1
- package/dist/core/ZstdManager.js +0 -144
- package/dist/core/ZstdManager.js.map +0 -1
- package/src/core/ZstdManager.ts +0 -165
package/src/node/ZipkitNode.ts
CHANGED
|
@@ -19,9 +19,20 @@ import {
|
|
|
19
19
|
ENCRYPT_HDR_SIZE
|
|
20
20
|
} from '../core/constants/Headers';
|
|
21
21
|
import { NEO_CRYPTO_ALGORITHM_AES256_V1 } from '../core/encryption/NeoCrypto';
|
|
22
|
+
import { AesCrypto } from '../core/encryption/AesCrypto';
|
|
23
|
+
import {
|
|
24
|
+
detectLegacyWasmZstd,
|
|
25
|
+
type LegacyZstdDetectResult,
|
|
26
|
+
} from './LegacyZstd';
|
|
22
27
|
import * as fs from 'fs';
|
|
23
28
|
import * as path from 'path';
|
|
24
29
|
import { minimatch } from 'minimatch';
|
|
30
|
+
import {
|
|
31
|
+
computeMerkleRootV0FromDigests,
|
|
32
|
+
computeMerkleRootV1FromLeaves,
|
|
33
|
+
type MerkleAlgorithm,
|
|
34
|
+
} from '../core/merkle/MerkleRoot';
|
|
35
|
+
import { isMetaInfPath } from '../core/constants/MetaPaths';
|
|
25
36
|
|
|
26
37
|
// Re-export everything from core Zipkit
|
|
27
38
|
export * from '../core';
|
|
@@ -106,6 +117,79 @@ export default class ZipkitNode extends Zipkit {
|
|
|
106
117
|
return this._zipkitDeCmpNode;
|
|
107
118
|
}
|
|
108
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Streaming Merkle root for file-based archives.
|
|
122
|
+
* V1 leaves come from the same decompress→hash pass as integrity (testEntry / extract),
|
|
123
|
+
* chunked — never re-reads uncompressed sources or materializes whole entries for the root alone.
|
|
124
|
+
*/
|
|
125
|
+
async getMerkleRootAsync(options?: { algorithm?: MerkleAlgorithm }): Promise<string | null> {
|
|
126
|
+
const algorithm: MerkleAlgorithm = options?.algorithm ?? 'v1';
|
|
127
|
+
let zipEntries: ZipEntry[];
|
|
128
|
+
try {
|
|
129
|
+
zipEntries = this.getDirectory();
|
|
130
|
+
} catch {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
if (!zipEntries?.length) {
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const contentEntries = zipEntries.filter((e) => !isMetaInfPath(e.filename || ''));
|
|
138
|
+
if (contentEntries.length === 0) {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (algorithm === 'v0') {
|
|
143
|
+
const digests = contentEntries
|
|
144
|
+
.filter((e) => e.sha256)
|
|
145
|
+
.map((e) => ({ path: e.filename || '', sha256: e.sha256 as string }));
|
|
146
|
+
return computeMerkleRootV0FromDigests(digests);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (contentEntries.every((e) => !!e.merkleLeafV1)) {
|
|
150
|
+
return computeMerkleRootV1FromLeaves(
|
|
151
|
+
contentEntries.map((e) => ({
|
|
152
|
+
path: e.filename || '',
|
|
153
|
+
merkleLeafV1: e.merkleLeafV1 as string,
|
|
154
|
+
}))
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Buffer-loaded: base class extract path
|
|
159
|
+
if ((this as any).inBuffer) {
|
|
160
|
+
return super.getMerkleRootAsync(options);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// File-based: stream hash each content entry once via testEntry (no write, dual hash)
|
|
164
|
+
if (!this.fileHandle && !this.filePath) {
|
|
165
|
+
// Create-session only: may only have live ZipEntry[] with leaves from write
|
|
166
|
+
return super.getMerkleRootAsync(options);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
for (const entry of contentEntries) {
|
|
171
|
+
if (entry.merkleLeafV1) {
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
await this.getZipDecompressNode().testEntry(entry, { skipHashCheck: false });
|
|
175
|
+
}
|
|
176
|
+
} catch (err) {
|
|
177
|
+
console.error('[ZipkitNode.getMerkleRootAsync] stream hash failed:', err);
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (!contentEntries.every((e) => !!e.merkleLeafV1)) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
return computeMerkleRootV1FromLeaves(
|
|
186
|
+
contentEntries.map((e) => ({
|
|
187
|
+
path: e.filename || '',
|
|
188
|
+
merkleLeafV1: e.merkleLeafV1 as string,
|
|
189
|
+
}))
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
|
|
109
193
|
// ============================================================================
|
|
110
194
|
// File Loading Methods
|
|
111
195
|
// ============================================================================
|
|
@@ -345,6 +429,75 @@ export default class ZipkitNode extends Zipkit {
|
|
|
345
429
|
};
|
|
346
430
|
}
|
|
347
431
|
|
|
432
|
+
/**
|
|
433
|
+
* Scan the loaded archive (or `archivePath`) for entries compressed with the
|
|
434
|
+
* legacy NeoZipKit WASM zstd codec (`@oneidentity/zstd-js`).
|
|
435
|
+
*
|
|
436
|
+
* Encrypted entries are skipped unless you only need a header check after
|
|
437
|
+
* decrypting externally — this API inspects raw stored payloads.
|
|
438
|
+
*/
|
|
439
|
+
async detectLegacyZstdEntries(
|
|
440
|
+
archivePath?: string,
|
|
441
|
+
options?: { verify?: boolean }
|
|
442
|
+
): Promise<Array<{ entry: ZipEntry; detection: LegacyZstdDetectResult }>> {
|
|
443
|
+
if (archivePath && !this.filePath) {
|
|
444
|
+
await this.loadZipFile(archivePath);
|
|
445
|
+
}
|
|
446
|
+
if (!this.filePath) {
|
|
447
|
+
throw new Error('Archive not loaded. Call loadZipFile() first or provide archivePath parameter.');
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const verify = options?.verify !== false;
|
|
451
|
+
const results: Array<{ entry: ZipEntry; detection: LegacyZstdDetectResult }> = [];
|
|
452
|
+
const entries = this.getDirectory();
|
|
453
|
+
|
|
454
|
+
for (const entry of entries) {
|
|
455
|
+
if (entry.isDirectory) {
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
const method =
|
|
459
|
+
entry.cmpMethod === CMP_METHOD.AES_ENCRYPT && entry.realCmpMethod >= 0
|
|
460
|
+
? entry.realCmpMethod
|
|
461
|
+
: entry.cmpMethod;
|
|
462
|
+
if (method !== CMP_METHOD.ZSTD) {
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (entry.isEncrypted || (entry.bitFlags & GP_FLAG.ENCRYPTED)) {
|
|
466
|
+
results.push({
|
|
467
|
+
entry,
|
|
468
|
+
detection: {
|
|
469
|
+
isLegacyWasm: false,
|
|
470
|
+
confidence: 'none',
|
|
471
|
+
frameContentSize: null,
|
|
472
|
+
expectedUncompressedSize: entry.uncompressedSize,
|
|
473
|
+
paddingBytes: 18,
|
|
474
|
+
reason: 'Encrypted zstd entry — decrypt before detecting legacy WASM frames',
|
|
475
|
+
},
|
|
476
|
+
});
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
const compressed = await this.getZipDecompressNode().readCompressedPayload(entry);
|
|
481
|
+
|
|
482
|
+
if (!verify) {
|
|
483
|
+
const detection = await detectLegacyWasmZstd(compressed, entry.uncompressedSize, {
|
|
484
|
+
verify: false,
|
|
485
|
+
crc: entry.crc,
|
|
486
|
+
});
|
|
487
|
+
results.push({ entry, detection });
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const detection = await detectLegacyWasmZstd(compressed, entry.uncompressedSize, {
|
|
492
|
+
verify: true,
|
|
493
|
+
crc: entry.crc,
|
|
494
|
+
});
|
|
495
|
+
results.push({ entry, detection });
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
return results;
|
|
499
|
+
}
|
|
500
|
+
|
|
348
501
|
/**
|
|
349
502
|
* Test entry integrity without extracting to disk
|
|
350
503
|
* Validates CRC-32 or SHA-256 hash without writing decompressed data
|
|
@@ -576,10 +729,13 @@ export default class ZipkitNode extends Zipkit {
|
|
|
576
729
|
}
|
|
577
730
|
|
|
578
731
|
// For AES encryption, pre-configure the entry so the initial local header
|
|
579
|
-
//
|
|
580
|
-
//
|
|
581
|
-
const
|
|
582
|
-
|
|
732
|
+
// matches the chosen scheme (WinZip method 99 + 0x9901, or NeoEncrypt + 0x024E).
|
|
733
|
+
// Default password encryption is NeoEncrypt (matches NeoZip product AES-256).
|
|
734
|
+
const encMethod = options?.password
|
|
735
|
+
? (options.encryptionMethod || 'neo-aes256')
|
|
736
|
+
: undefined;
|
|
737
|
+
const isAesRequested = encMethod === 'aes256';
|
|
738
|
+
const isNeoRequested = encMethod === 'neo-aes256';
|
|
583
739
|
if (isAesRequested) {
|
|
584
740
|
entry.cmpMethod = CMP_METHOD.AES_ENCRYPT;
|
|
585
741
|
entry.aesVersion = 1;
|
|
@@ -615,55 +771,87 @@ export default class ZipkitNode extends Zipkit {
|
|
|
615
771
|
});
|
|
616
772
|
});
|
|
617
773
|
|
|
618
|
-
// Step 3: Compress file and write data
|
|
774
|
+
// Step 3: Compress file and write data (chunked when large).
|
|
775
|
+
// ZipCrypto still uses the buffer path (legacy header needs CRC up front).
|
|
776
|
+
// AES / NeoEncrypt use AesEncryptor so ciphertext is written in chunks.
|
|
619
777
|
const bufferSize = options?.bufferSize || this.getBufferSize();
|
|
620
|
-
const
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
const onOutputBuffer = async (data: Buffer) => {
|
|
631
|
-
await new Promise<void>((resolve, reject) => {
|
|
632
|
-
writer.outputStream.write(data, (error) => {
|
|
633
|
-
if (error) {
|
|
634
|
-
reject(error);
|
|
635
|
-
} else {
|
|
636
|
-
writer.currentPosition += data.length;
|
|
637
|
-
if (callbacks?.onProgress) {
|
|
638
|
-
callbacks.onProgress(entry, data.length);
|
|
639
|
-
}
|
|
640
|
-
resolve();
|
|
641
|
-
}
|
|
642
|
-
});
|
|
643
|
-
});
|
|
644
|
-
};
|
|
645
|
-
|
|
646
|
-
// compressFileStream will set entry.compressedSize and entry.crc
|
|
647
|
-
await this.compressFileStream(filePath, entry, options, onOutputBuffer);
|
|
648
|
-
} else {
|
|
649
|
-
// Use regular buffer compression for small files
|
|
650
|
-
// compressFile will set entry.compressedSize and entry.crc
|
|
651
|
-
const compressedData = await this.compressFile(filePath, entry, options);
|
|
652
|
-
|
|
653
|
-
// Write compressed data to file
|
|
778
|
+
const useZipCrypto = encMethod === 'zipcrypto';
|
|
779
|
+
const shouldUseChunked =
|
|
780
|
+
!!entry.uncompressedSize &&
|
|
781
|
+
entry.uncompressedSize > bufferSize &&
|
|
782
|
+
!useZipCrypto;
|
|
783
|
+
|
|
784
|
+
const writeChunk = async (data: Buffer): Promise<void> => {
|
|
785
|
+
if (data.length === 0) {
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
654
788
|
await new Promise<void>((resolve, reject) => {
|
|
655
|
-
writer.outputStream.write(
|
|
789
|
+
writer.outputStream.write(data, (error) => {
|
|
656
790
|
if (error) {
|
|
657
791
|
reject(error);
|
|
658
792
|
} else {
|
|
659
|
-
writer.currentPosition +=
|
|
793
|
+
writer.currentPosition += data.length;
|
|
660
794
|
if (callbacks?.onProgress) {
|
|
661
|
-
callbacks.onProgress(entry,
|
|
795
|
+
callbacks.onProgress(entry, data.length);
|
|
662
796
|
}
|
|
663
797
|
resolve();
|
|
664
798
|
}
|
|
665
799
|
});
|
|
666
800
|
});
|
|
801
|
+
};
|
|
802
|
+
|
|
803
|
+
if (shouldUseChunked) {
|
|
804
|
+
const useAesStream = isAesRequested || isNeoRequested;
|
|
805
|
+
const encryptor = useAesStream
|
|
806
|
+
? AesCrypto.createEncryptor(options!.password!)
|
|
807
|
+
: null;
|
|
808
|
+
let streamedBytes = 0;
|
|
809
|
+
|
|
810
|
+
if (encryptor) {
|
|
811
|
+
const hdr = encryptor.header();
|
|
812
|
+
await writeChunk(hdr);
|
|
813
|
+
streamedBytes += hdr.length;
|
|
814
|
+
entry.isEncrypted = true;
|
|
815
|
+
entry.bitFlags |= GP_FLAG.ENCRYPTED;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
// Compress without applying buffer encrypt — we stream-encrypt below.
|
|
819
|
+
const streamOptions: CompressOptions = encryptor
|
|
820
|
+
? { ...options, password: undefined, encryptionMethod: undefined }
|
|
821
|
+
: { ...options };
|
|
822
|
+
|
|
823
|
+
const onOutputBuffer = async (data: Buffer) => {
|
|
824
|
+
const out = encryptor ? encryptor.update(data) : data;
|
|
825
|
+
await writeChunk(out);
|
|
826
|
+
streamedBytes += out.length;
|
|
827
|
+
};
|
|
828
|
+
|
|
829
|
+
await this.compressFileStream(filePath, entry, streamOptions, onOutputBuffer);
|
|
830
|
+
|
|
831
|
+
if (encryptor) {
|
|
832
|
+
const auth = encryptor.finish();
|
|
833
|
+
await writeChunk(auth);
|
|
834
|
+
streamedBytes += auth.length;
|
|
835
|
+
entry.compressedSize = streamedBytes;
|
|
836
|
+
entry.isEncrypted = true;
|
|
837
|
+
entry.bitFlags |= GP_FLAG.ENCRYPTED;
|
|
838
|
+
// compressFileStream overwrites cmpMethod with the real codec — restore AES/Neo metadata
|
|
839
|
+
if (isAesRequested) {
|
|
840
|
+
entry.realCmpMethod = realCmpMethod;
|
|
841
|
+
entry.cmpMethod = CMP_METHOD.AES_ENCRYPT;
|
|
842
|
+
entry.aesVersion = 1;
|
|
843
|
+
entry.aesStrength = 3;
|
|
844
|
+
} else if (isNeoRequested) {
|
|
845
|
+
entry.cmpMethod = realCmpMethod;
|
|
846
|
+
entry.neoCryptoPayloadVersion = 1;
|
|
847
|
+
entry.neoCryptoAlgorithm = NEO_CRYPTO_ALGORITHM_AES256_V1;
|
|
848
|
+
entry.neoCryptoFlags = 0;
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
} else {
|
|
852
|
+
// Small files / ZipCrypto: buffer compression (may encrypt inside compressFile)
|
|
853
|
+
const compressedData = await this.compressFile(filePath, entry, options);
|
|
854
|
+
await writeChunk(compressedData);
|
|
667
855
|
}
|
|
668
856
|
|
|
669
857
|
// Step 4: Patch local header in-place with final compressed size, CRC, and flags
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ZstdNode - Native Node.js Zstandard via zlib (streaming / chunked)
|
|
3
|
+
*
|
|
4
|
+
* This is the only zstd backend in NeoZipKit. There is no WASM codec.
|
|
5
|
+
* Requires Node.js with `zlib.createZstdCompress` (Node >= 22.15 or >= 23.8).
|
|
6
|
+
* Browser builds alias this module to a stub that throws.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import * as zlib from 'zlib';
|
|
10
|
+
import { Readable } from 'stream';
|
|
11
|
+
import { promisify } from 'util';
|
|
12
|
+
|
|
13
|
+
const zstdCompressAsync = promisify(zlib.zstdCompress);
|
|
14
|
+
const zstdDecompressAsync = promisify(zlib.zstdDecompress);
|
|
15
|
+
|
|
16
|
+
export function isNativeZstdAvailable(): boolean {
|
|
17
|
+
return (
|
|
18
|
+
typeof zlib.createZstdCompress === 'function' &&
|
|
19
|
+
typeof zlib.createZstdDecompress === 'function' &&
|
|
20
|
+
typeof zlib.zstdCompress === 'function' &&
|
|
21
|
+
typeof zlib.zstdDecompress === 'function'
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function assertNativeZstd(): void {
|
|
26
|
+
if (!isNativeZstdAvailable()) {
|
|
27
|
+
throw new Error(
|
|
28
|
+
'Native zstd requires Node.js with zlib ZSTD support ' +
|
|
29
|
+
'(Node.js >= 22.15 or >= 23.8).'
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Map NeoZipKit level 1–9 to a zstd compression level (1–19). */
|
|
35
|
+
export function mapCompressionLevel(level: number): number {
|
|
36
|
+
return Math.min(Math.max(1, Math.floor(level * 2.1)), 19);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function compressOptions(level: number): zlib.ZstdOptions {
|
|
40
|
+
return {
|
|
41
|
+
params: {
|
|
42
|
+
[zlib.constants.ZSTD_c_compressionLevel]: mapCompressionLevel(level),
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function createCompressTransform(level: number = 6): zlib.ZstdCompress {
|
|
48
|
+
assertNativeZstd();
|
|
49
|
+
return zlib.createZstdCompress(compressOptions(level));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function createDecompressTransform(): zlib.ZstdDecompress {
|
|
53
|
+
assertNativeZstd();
|
|
54
|
+
return zlib.createZstdDecompress();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export async function compress(data: Buffer, level: number = 6): Promise<Buffer> {
|
|
58
|
+
assertNativeZstd();
|
|
59
|
+
const result = await zstdCompressAsync(data, compressOptions(level));
|
|
60
|
+
return Buffer.from(result);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function decompress(data: Buffer): Promise<Buffer> {
|
|
64
|
+
assertNativeZstd();
|
|
65
|
+
const result = await zstdDecompressAsync(data);
|
|
66
|
+
return Buffer.from(result);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function compressSync(data: Buffer, level: number = 6): Buffer {
|
|
70
|
+
assertNativeZstd();
|
|
71
|
+
return zlib.zstdCompressSync(data, compressOptions(level));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function decompressSync(data: Buffer): Buffer {
|
|
75
|
+
assertNativeZstd();
|
|
76
|
+
return zlib.zstdDecompressSync(data);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Stream-compress plaintext from a chunked reader (primary path for all sizes).
|
|
81
|
+
*/
|
|
82
|
+
export async function compressChunks(
|
|
83
|
+
readChunk: (position: number, size: number) => Buffer,
|
|
84
|
+
totalSize: number,
|
|
85
|
+
chunkSize: number,
|
|
86
|
+
level: number,
|
|
87
|
+
onPlainChunk?: (chunk: Buffer) => void,
|
|
88
|
+
onCompressedChunk?: (chunk: Buffer) => void | Promise<void>
|
|
89
|
+
): Promise<number> {
|
|
90
|
+
async function* plainChunks(): AsyncGenerator<Buffer> {
|
|
91
|
+
let position = 0;
|
|
92
|
+
while (position < totalSize) {
|
|
93
|
+
const size = Math.min(chunkSize, totalSize - position);
|
|
94
|
+
const chunk = readChunk(position, size);
|
|
95
|
+
onPlainChunk?.(chunk);
|
|
96
|
+
position += chunk.length;
|
|
97
|
+
yield chunk;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const transform = createCompressTransform(level);
|
|
102
|
+
const readable = Readable.from(plainChunks());
|
|
103
|
+
readable.pipe(transform);
|
|
104
|
+
|
|
105
|
+
let compressedSize = 0;
|
|
106
|
+
try {
|
|
107
|
+
for await (const out of transform) {
|
|
108
|
+
const buf = Buffer.isBuffer(out) ? out : Buffer.from(out as Uint8Array);
|
|
109
|
+
compressedSize += buf.length;
|
|
110
|
+
if (onCompressedChunk) {
|
|
111
|
+
await onCompressedChunk(buf);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
} finally {
|
|
115
|
+
readable.destroy();
|
|
116
|
+
transform.destroy();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return compressedSize;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Decompress an async iterable of compressed chunks, yielding plaintext buffers.
|
|
124
|
+
*/
|
|
125
|
+
export async function* decompressStream(
|
|
126
|
+
compressedStream: AsyncIterable<Buffer>
|
|
127
|
+
): AsyncGenerator<Buffer> {
|
|
128
|
+
const transform = createDecompressTransform();
|
|
129
|
+
const readable = Readable.from(compressedStream);
|
|
130
|
+
readable.pipe(transform);
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
for await (const chunk of transform) {
|
|
134
|
+
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
readable.destroy();
|
|
138
|
+
transform.destroy();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export const ZstdNode = {
|
|
143
|
+
isNativeZstdAvailable,
|
|
144
|
+
mapCompressionLevel,
|
|
145
|
+
createCompressTransform,
|
|
146
|
+
createDecompressTransform,
|
|
147
|
+
compress,
|
|
148
|
+
decompress,
|
|
149
|
+
compressSync,
|
|
150
|
+
decompressSync,
|
|
151
|
+
compressChunks,
|
|
152
|
+
decompressStream,
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
export default ZstdNode;
|
package/src/node/index.ts
CHANGED
|
@@ -21,6 +21,16 @@ export default ZipkitNodeDefault;
|
|
|
21
21
|
export type { ZipFileWriter } from './ZipkitNode';
|
|
22
22
|
export { ZipCompressNode } from './ZipCompressNode';
|
|
23
23
|
export { ZipDecompressNode } from './ZipDecompressNode';
|
|
24
|
+
export { ZstdNode, isNativeZstdAvailable } from './ZstdNode';
|
|
25
|
+
export {
|
|
26
|
+
LegacyZstd,
|
|
27
|
+
LEGACY_WASM_ZSTD_PADDING,
|
|
28
|
+
parseZstdFrameContentSize,
|
|
29
|
+
looksLikeLegacyWasmZstd,
|
|
30
|
+
detectLegacyWasmZstd,
|
|
31
|
+
repairLegacyWasmZstdPlaintext,
|
|
32
|
+
} from './LegacyZstd';
|
|
33
|
+
export type { LegacyZstdDetectResult, LegacyZstdConfidence, DetectLegacyZstdOptions } from './LegacyZstd';
|
|
24
34
|
|
|
25
35
|
// Efficient ZIP copying using ZipEntry directly
|
|
26
36
|
export { ZipCopyNode } from './ZipCopyNode';
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* ZstdManager - Global Zstd Codec Manager
|
|
3
|
-
*
|
|
4
|
-
* This manager ensures safe usage of the Zstd WASM module across multiple instances.
|
|
5
|
-
*
|
|
6
|
-
* ## Why This Is Necessary
|
|
7
|
-
*
|
|
8
|
-
* The `@oneidentity/zstd-js` library uses a shared WASM module internally.
|
|
9
|
-
* While `ZstdInit()` returns different object references, they all share the same
|
|
10
|
-
* `ZstdSimple` implementation and WASM memory/heap.
|
|
11
|
-
*
|
|
12
|
-
* This means concurrent or improperly managed sequential operations can cause
|
|
13
|
-
* memory corruption because they're all operating on the same underlying WASM state.
|
|
14
|
-
*
|
|
15
|
-
* ## How It Works
|
|
16
|
-
*
|
|
17
|
-
* 1. Single initialization: WASM module is initialized once globally
|
|
18
|
-
* 2. Operation queuing: All compress/decompress operations are queued
|
|
19
|
-
* 3. Sequential execution: JavaScript's single-threaded nature ensures safe execution
|
|
20
|
-
* 4. Automatic initialization: Lazy initialization on first use
|
|
21
|
-
*
|
|
22
|
-
* ## Usage
|
|
23
|
-
*
|
|
24
|
-
* ```typescript
|
|
25
|
-
* import { ZstdManager } from './ZstdManager';
|
|
26
|
-
*
|
|
27
|
-
* // Compress
|
|
28
|
-
* const compressed = await ZstdManager.compress(data, level);
|
|
29
|
-
*
|
|
30
|
-
* // Decompress
|
|
31
|
-
* const decompressed = await ZstdManager.decompress(compressedData);
|
|
32
|
-
* ```
|
|
33
|
-
*/
|
|
34
|
-
/**
|
|
35
|
-
* Global Zstd Manager
|
|
36
|
-
* Use this for all Zstd compression/decompression operations
|
|
37
|
-
*/
|
|
38
|
-
export declare const ZstdManager: {
|
|
39
|
-
/**
|
|
40
|
-
* Compress data using Zstd
|
|
41
|
-
* Operations are automatically queued to prevent interference
|
|
42
|
-
*/
|
|
43
|
-
compress: (data: Uint8Array, level?: number) => Promise<Uint8Array<ArrayBufferLike>>;
|
|
44
|
-
/**
|
|
45
|
-
* Decompress data using Zstd
|
|
46
|
-
* Operations are automatically queued to prevent interference
|
|
47
|
-
*/
|
|
48
|
-
decompress: (data: Uint8Array | Buffer) => Promise<Uint8Array<ArrayBufferLike>>;
|
|
49
|
-
/**
|
|
50
|
-
* Reset the manager (for testing)
|
|
51
|
-
* @internal
|
|
52
|
-
*/
|
|
53
|
-
reset: () => void;
|
|
54
|
-
};
|
|
55
|
-
export default ZstdManager;
|
|
56
|
-
//# sourceMappingURL=ZstdManager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ZstdManager.d.ts","sourceRoot":"","sources":["../../src/core/ZstdManager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAyGH;;;GAGG;AACH,eAAO,MAAM,WAAW;IACtB;;;OAGG;qBACc,UAAU,UAAU,MAAM;IAG3C;;;OAGG;uBACgB,UAAU,GAAG,MAAM;IAGtC;;;OAGG;;CAEJ,CAAC;AAEF,eAAe,WAAW,CAAC"}
|
package/dist/core/ZstdManager.js
DELETED
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* ZstdManager - Global Zstd Codec Manager
|
|
4
|
-
*
|
|
5
|
-
* This manager ensures safe usage of the Zstd WASM module across multiple instances.
|
|
6
|
-
*
|
|
7
|
-
* ## Why This Is Necessary
|
|
8
|
-
*
|
|
9
|
-
* The `@oneidentity/zstd-js` library uses a shared WASM module internally.
|
|
10
|
-
* While `ZstdInit()` returns different object references, they all share the same
|
|
11
|
-
* `ZstdSimple` implementation and WASM memory/heap.
|
|
12
|
-
*
|
|
13
|
-
* This means concurrent or improperly managed sequential operations can cause
|
|
14
|
-
* memory corruption because they're all operating on the same underlying WASM state.
|
|
15
|
-
*
|
|
16
|
-
* ## How It Works
|
|
17
|
-
*
|
|
18
|
-
* 1. Single initialization: WASM module is initialized once globally
|
|
19
|
-
* 2. Operation queuing: All compress/decompress operations are queued
|
|
20
|
-
* 3. Sequential execution: JavaScript's single-threaded nature ensures safe execution
|
|
21
|
-
* 4. Automatic initialization: Lazy initialization on first use
|
|
22
|
-
*
|
|
23
|
-
* ## Usage
|
|
24
|
-
*
|
|
25
|
-
* ```typescript
|
|
26
|
-
* import { ZstdManager } from './ZstdManager';
|
|
27
|
-
*
|
|
28
|
-
* // Compress
|
|
29
|
-
* const compressed = await ZstdManager.compress(data, level);
|
|
30
|
-
*
|
|
31
|
-
* // Decompress
|
|
32
|
-
* const decompressed = await ZstdManager.decompress(compressedData);
|
|
33
|
-
* ```
|
|
34
|
-
*/
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.ZstdManager = void 0;
|
|
37
|
-
const zstd_js_1 = require("@oneidentity/zstd-js");
|
|
38
|
-
class ZstdCodecManager {
|
|
39
|
-
constructor() {
|
|
40
|
-
this.codec = null;
|
|
41
|
-
this.initPromise = null;
|
|
42
|
-
this.operationQueue = Promise.resolve();
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* Get the singleton instance of ZstdCodecManager
|
|
46
|
-
*/
|
|
47
|
-
static getInstance() {
|
|
48
|
-
if (!ZstdCodecManager.instance) {
|
|
49
|
-
ZstdCodecManager.instance = new ZstdCodecManager();
|
|
50
|
-
}
|
|
51
|
-
return ZstdCodecManager.instance;
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Ensure the Zstd codec is initialized
|
|
55
|
-
* This is called automatically before any compress/decompress operation
|
|
56
|
-
*/
|
|
57
|
-
async ensureInitialized() {
|
|
58
|
-
if (this.codec) {
|
|
59
|
-
return;
|
|
60
|
-
}
|
|
61
|
-
if (this.initPromise) {
|
|
62
|
-
return this.initPromise;
|
|
63
|
-
}
|
|
64
|
-
this.initPromise = (async () => {
|
|
65
|
-
this.codec = await (0, zstd_js_1.ZstdInit)();
|
|
66
|
-
})();
|
|
67
|
-
return this.initPromise;
|
|
68
|
-
}
|
|
69
|
-
/**
|
|
70
|
-
* Queue an operation to ensure sequential execution
|
|
71
|
-
* This prevents concurrent operations from interfering with each other
|
|
72
|
-
*/
|
|
73
|
-
async queueOperation(operation) {
|
|
74
|
-
// Chain this operation after the previous one
|
|
75
|
-
const promise = this.operationQueue.then(operation, operation);
|
|
76
|
-
// Update the queue to point to this operation
|
|
77
|
-
this.operationQueue = promise.catch(() => { }); // Catch to prevent unhandled rejections in queue
|
|
78
|
-
return promise;
|
|
79
|
-
}
|
|
80
|
-
/**
|
|
81
|
-
* Compress data using Zstd
|
|
82
|
-
* @param data - Data to compress (Uint8Array)
|
|
83
|
-
* @param level - Compression level (1-22, default 6)
|
|
84
|
-
* @returns Compressed data as Uint8Array
|
|
85
|
-
*/
|
|
86
|
-
async compress(data, level = 6) {
|
|
87
|
-
return this.queueOperation(async () => {
|
|
88
|
-
await this.ensureInitialized();
|
|
89
|
-
if (!this.codec) {
|
|
90
|
-
throw new Error('Zstd codec not initialized');
|
|
91
|
-
}
|
|
92
|
-
return this.codec.ZstdSimple.compress(data, level);
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Decompress data using Zstd
|
|
97
|
-
* @param data - Compressed data (Uint8Array or Buffer)
|
|
98
|
-
* @returns Decompressed data as Uint8Array
|
|
99
|
-
*/
|
|
100
|
-
async decompress(data) {
|
|
101
|
-
return this.queueOperation(async () => {
|
|
102
|
-
await this.ensureInitialized();
|
|
103
|
-
if (!this.codec) {
|
|
104
|
-
throw new Error('Zstd codec not initialized');
|
|
105
|
-
}
|
|
106
|
-
// Convert Buffer to Uint8Array if needed
|
|
107
|
-
const inputData = data instanceof Buffer
|
|
108
|
-
? new Uint8Array(data.buffer, data.byteOffset, data.byteLength)
|
|
109
|
-
: data;
|
|
110
|
-
return this.codec.ZstdSimple.decompress(inputData);
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Reset the manager (for testing purposes)
|
|
115
|
-
* @internal
|
|
116
|
-
*/
|
|
117
|
-
static reset() {
|
|
118
|
-
ZstdCodecManager.instance = null;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
ZstdCodecManager.instance = null;
|
|
122
|
-
/**
|
|
123
|
-
* Global Zstd Manager
|
|
124
|
-
* Use this for all Zstd compression/decompression operations
|
|
125
|
-
*/
|
|
126
|
-
exports.ZstdManager = {
|
|
127
|
-
/**
|
|
128
|
-
* Compress data using Zstd
|
|
129
|
-
* Operations are automatically queued to prevent interference
|
|
130
|
-
*/
|
|
131
|
-
compress: (data, level) => ZstdCodecManager.getInstance().compress(data, level),
|
|
132
|
-
/**
|
|
133
|
-
* Decompress data using Zstd
|
|
134
|
-
* Operations are automatically queued to prevent interference
|
|
135
|
-
*/
|
|
136
|
-
decompress: (data) => ZstdCodecManager.getInstance().decompress(data),
|
|
137
|
-
/**
|
|
138
|
-
* Reset the manager (for testing)
|
|
139
|
-
* @internal
|
|
140
|
-
*/
|
|
141
|
-
reset: () => ZstdCodecManager.reset(),
|
|
142
|
-
};
|
|
143
|
-
exports.default = exports.ZstdManager;
|
|
144
|
-
//# sourceMappingURL=ZstdManager.js.map
|