neozipkit 0.8.0 → 1.0.5

package/src/node/ZipCompressNode.ts

@@ -22,10 +22,10 @@ import { HashCalculator } from '../core/components/HashCalculator';
 import { ZipCrypto } from '../core/encryption/ZipCrypto';
 import { AesCrypto } from '../core/encryption/AesCrypto';
 import { NeoCrypto, NEO_CRYPTO_ALGORITHM_AES256_V1 } from '../core/encryption/NeoCrypto';
-import { ZstdManager } from '../core/ZstdManager';
 import Errors from '../core/constants/Errors';
 import * as fs from 'fs';
 import * as path from 'path';
+import { ZstdNode } from './ZstdNode';
 
 const pako = require('pako');
 
@@ -119,12 +119,10 @@ export class ZipCompressNode {
     
     if (hashCalculator) {
       hashCalculator.update(data);
-      if (!entry.crc || entry.crc === 0) {
-        entry.crc = hashCalculator.finalizeCRC32();
-      }
-      if (options?.useSHA256 && !entry.sha256) {
-        entry.sha256 = hashCalculator.finalizeSHA256();
-      }
+      hashCalculator.applyToEntry(entry, {
+            setCrc: !entry.crc || entry.crc === 0,
+            setSha256: !!(options?.useSHA256 && !entry.sha256),
+          });
     }
     
     // Compress based on method
@@ -148,10 +146,11 @@ export class ZipCompressNode {
       const enc = options.encryptionMethod;
       if (enc === 'zipcrypto') {
         buffer = this.encryptCompressedData(buffer, entry, options.password);
-      } else if (enc === 'neo-aes256') {
-        buffer = this.encryptCompressedDataNeo(buffer, entry, options.password);
-      } else {
+      } else if (enc === 'aes256') {
         buffer = this.encryptCompressedDataAes(buffer, entry, options.password);
+      } else {
+        // Default / 'neo-aes256': NeoEncrypt (0x024E)
+        buffer = this.encryptCompressedDataNeo(buffer, entry, options.password);
       }
     }
 
@@ -187,12 +186,10 @@ export class ZipCompressNode {
         // For buffer, calculate hashes if needed
         if (hashCalculator && entry) {
           hashCalculator.update(data);
-          if (!entry.crc || entry.crc === 0) {
-            entry.crc = hashCalculator.finalizeCRC32();
-          }
-          if (options?.useSHA256 && !entry.sha256) {
-            entry.sha256 = hashCalculator.finalizeSHA256();
-          }
+          hashCalculator.applyToEntry(entry, {
+            setCrc: !entry.crc || entry.crc === 0,
+            setSha256: !!(options?.useSHA256 && !entry.sha256),
+          });
         }
         return data;
       } else {
@@ -219,12 +216,10 @@ export class ZipCompressNode {
         
         // Finalize hashes
         if (hashCalculator && entry) {
-          if (!entry.crc || entry.crc === 0) {
-            entry.crc = hashCalculator.finalizeCRC32();
-          }
-          if (options?.useSHA256 && !entry.sha256) {
-            entry.sha256 = hashCalculator.finalizeSHA256();
-          }
+          hashCalculator.applyToEntry(entry, {
+            setCrc: !entry.crc || entry.crc === 0,
+            setSha256: !!(options?.useSHA256 && !entry.sha256),
+          });
         }
         
         return Buffer.alloc(0);
@@ -245,10 +240,10 @@ export class ZipCompressNode {
         if (hashCalculator && entry) {
           hashCalculator.update(data);
           // Always set CRC if hash calculator was used (it calculated the correct CRC)
-          entry.crc = hashCalculator.finalizeCRC32();
-          if (options?.useSHA256 && !entry.sha256) {
-            entry.sha256 = hashCalculator.finalizeSHA256();
-          }
+          hashCalculator.applyToEntry(entry, {
+          setCrc: true,
+          setSha256: !!(options?.useSHA256 && !entry.sha256),
+        });
         }
         
         const result = pako.deflateRaw(data, { level: level ?? 6 });
@@ -298,10 +293,10 @@ export class ZipCompressNode {
         // Finalize hashes after all chunks processed and compressed
         if (hashCalculator && entry) {
           // Always set CRC if hash calculator was used (it calculated the correct CRC)
-          entry.crc = hashCalculator.finalizeCRC32();
-          if (options?.useSHA256 && !entry.sha256) {
-            entry.sha256 = hashCalculator.finalizeSHA256();
-          }
+          hashCalculator.applyToEntry(entry, {
+          setCrc: true,
+          setSha256: !!(options?.useSHA256 && !entry.sha256),
+        });
         }
         
         // For chunked processing, return empty buffer (data already written via onOutputBuffer)
@@ -339,16 +334,23 @@ export class ZipCompressNode {
   }
 
   /**
-   * Compresses data using Zstandard (zstd) algorithm
-   * @param input - Buffer to compress OR chunked reader object with totalSize and readChunk callback
+   * Compresses data using Zstandard via Node native zlib chunked streaming.
+   *
+   * @param input - Buffer OR chunked reader (`onReadChunk` preferred; `readChunk` accepted)
    * @param options - Compression options
-   * @param bufferSize - Size of buffer to read if using chunked reader (default: 512KB)
-   * @param entry - Optional ZIP entry for hash calculation
-   * @param onOutputBuffer - Optional callback for streaming output
-   * @returns Compressed data buffer
+   * @param bufferSize - Chunk size for streaming reads (default: 512KB)
+   * @param entry - Optional ZIP entry for hash / compressedSize
+   * @param onOutputBuffer - Optional callback for streaming compressed output
+   * @returns Compressed buffer, or empty buffer when streamed via onOutputBuffer
    */
   async zstdCompress(
-    input: Buffer | { totalSize: number, readChunk: (position: number, size: number) => Buffer },
+    input:
+      | Buffer
+      | {
+          totalSize: number;
+          onReadChunk?: (position: number, size: number) => Buffer;
+          readChunk?: (position: number, size: number) => Buffer;
+        },
     options?: CompressOptions,
     bufferSize?: number,
     entry?: ZipEntry,
@@ -357,66 +359,108 @@ export class ZipCompressNode {
     const effectiveBufferSize = bufferSize || options?.bufferSize || this.zipkitNode.getBufferSize();
     const isBuffer = Buffer.isBuffer(input);
     const totalSize = isBuffer ? input.length : input.totalSize;
-    
+
+    const needsHashCalculation =
+      entry && ((!entry.crc || entry.crc === 0) || (options?.useSHA256 && !entry.sha256));
+    const hashCalculator = needsHashCalculation
+      ? new HashCalculator({ useSHA256: !!(options?.useSHA256 && entry && !entry.sha256) })
+      : null;
+
     if (options?.level == 0) {
-      // For store mode, return as-is
       if (isBuffer) {
+        if (hashCalculator && entry) {
+          hashCalculator.update(input);
+          hashCalculator.applyToEntry(entry, {
+            setCrc: !entry.crc || entry.crc === 0,
+            setSha256: !!(options?.useSHA256 && !entry.sha256),
+          });
+        }
         return input;
-      } else {
-        // Read all chunks
-        const chunks: Buffer[] = [];
-        let position = 0;
-        while (position < totalSize) {
-          const size = Math.min(effectiveBufferSize, totalSize - position);
-          const chunk = input.readChunk(position, size);
+      }
+
+      const chunks: Buffer[] = [];
+      const read = input.onReadChunk ?? input.readChunk;
+      if (!read) {
+        throw new Error('ZSTD store mode: chunked reader missing read callback');
+      }
+      let position = 0;
+      while (position < totalSize) {
+        const size = Math.min(effectiveBufferSize, totalSize - position);
+        const chunk = read(position, size);
+        if (hashCalculator) {
+          hashCalculator.update(chunk);
+        }
+        if (onOutputBuffer) {
+          await onOutputBuffer(chunk);
+        } else {
           chunks.push(chunk);
-          position += size;
         }
-        return Buffer.concat(chunks);
+        position += chunk.length;
+      }
+      if (hashCalculator && entry) {
+        hashCalculator.applyToEntry(entry, {
+            setCrc: !entry.crc || entry.crc === 0,
+            setSha256: !!(options?.useSHA256 && !entry.sha256),
+          });
       }
+      return onOutputBuffer ? Buffer.alloc(0) : Buffer.concat(chunks);
     }
-    
+
     try {
-      // Zstd compression levels range from 1 (fastest) to 22 (highest compression)
-      // Map our 1-9 level to a reasonable zstd range (1-19)
       const level = options?.level ?? 6;
-      const zstdLevel = Math.min(Math.max(1, Math.floor(level * 2.1)), 19);
 
-      // Get the full buffer (zstd doesn't support true streaming compression)
-      const inbuf = isBuffer ? input : (() => {
-        const chunks: Buffer[] = [];
-        let position = 0;
-        while (position < totalSize) {
-          const size = Math.min(effectiveBufferSize, totalSize - position);
-          const chunk = input.readChunk(position, size);
-          chunks.push(chunk);
-          position += size;
-        }
-        return Buffer.concat(chunks);
-      })();
-      
-      // Validate input
-      if (!inbuf || inbuf.length === 0) {
+      const read = isBuffer
+        ? (position: number, size: number) => input.subarray(position, position + size)
+        : (input.onReadChunk ?? input.readChunk);
+      if (!read) {
+        throw new Error('ZSTD compression: chunked reader missing read callback');
+      }
+      if (!totalSize) {
         throw new Error('ZSTD compression: empty input buffer');
       }
-      
-      // Convert Buffer to Uint8Array for WASM module
-      const inputArray = new Uint8Array(inbuf.buffer, inbuf.byteOffset, inbuf.byteLength);
-      
-      // Compress the data with zstd using global ZstdManager
-      const compressedData = await ZstdManager.compress(inputArray, zstdLevel);
-      const compressedBuffer = Buffer.from(compressedData);
-      
-      // Set the compressed size in the entry for ZIP file structure
+
+      if (isBuffer && hashCalculator && entry) {
+        hashCalculator.update(input);
+        hashCalculator.applyToEntry(entry, {
+          setCrc: true,
+          setSha256: !!(options?.useSHA256 && !entry.sha256),
+        });
+      }
+
+      const collected: Buffer[] = [];
+      const compressedSize = await ZstdNode.compressChunks(
+        read,
+        totalSize,
+        effectiveBufferSize,
+        level,
+        isBuffer
+          ? undefined
+          : (plain) => {
+              hashCalculator?.update(plain);
+            },
+        async (compressed) => {
+          if (onOutputBuffer) {
+            await onOutputBuffer(compressed);
+          } else {
+            collected.push(compressed);
+          }
+        }
+      );
+
+      if (!isBuffer && hashCalculator && entry) {
+        hashCalculator.applyToEntry(entry, {
+          setCrc: true,
+          setSha256: !!(options?.useSHA256 && !entry.sha256),
+        });
+      }
       if (entry) {
-        entry.compressedSize = compressedBuffer.length;
+        entry.compressedSize = compressedSize;
       }
-      
+
       if (onOutputBuffer) {
-        await onOutputBuffer(compressedBuffer);
+        return Buffer.alloc(0);
       }
-      
-      return compressedBuffer;
+      return Buffer.concat(collected);
     } catch (e) {
       Logger.error('Error during zstd compression:', e);
       throw new Error(Errors.COMPRESSION_ERROR);
@@ -591,9 +635,8 @@ export class ZipCompressNode {
       const fileData = fs.readFileSync(filePath);
       return await this.compressData(entry, fileData, options, onOutputBuffer);
     } else if (compressionMethod === 'ZSTD') {
-      // ZSTD requires full buffer, so read file first
-      const fileData = fs.readFileSync(filePath);
-      return await this.compressData(entry, fileData, options, onOutputBuffer);
+      // Native Node zstd chunked streaming
+      return await this.zstdCompress(chunkedReader, options, bufferSize, entry, onOutputBuffer);
     } else {
       // DEFLATED: Use deflateCompress with chunked reader
       return await this.deflateCompress(chunkedReader, options, bufferSize, entry, onOutputBuffer);

package/src/node/ZipDecompressNode.ts

@@ -14,8 +14,9 @@
 //
 
 const pako = require('pako');
-import { ZstdManager } from '../core/ZstdManager';
 import ZipkitNode from './ZipkitNode';
+import { ZstdNode } from './ZstdNode';
+import { repairLegacyWasmZstdPlaintext } from './LegacyZstd';
 import { Logger } from '../core/components/Logger';
 import ZipEntry from '../core/ZipEntry';
 import Errors from '../core/constants/Errors';
@@ -125,6 +126,18 @@ export class ZipDecompressNode {
     return await this.extractToBufferInternal(fileHandle, entry, options);
   }
 
+  /**
+   * Read the raw compressed payload for an entry (no decrypt / decompress).
+   */
+  async readCompressedPayload(entry: ZipEntry): Promise<Buffer> {
+    const fileHandle = (this.zipkitNode as any).getFileHandle();
+    const chunks: Buffer[] = [];
+    for await (const chunk of this.readCompressedDataStream(fileHandle, entry)) {
+      chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+  }
+
   /**
    * Test entry integrity without extracting to disk
    * Validates CRC-32 or SHA-256 hash without writing decompressed data
@@ -228,19 +241,15 @@ export class ZipDecompressNode {
         entry.aesVersion > 0 || entry.cmpMethod === CMP_METHOD.AES_ENCRYPT || entry.neoCryptoAlgorithm > 0;
 
       if (isEncrypted && isFullPayloadCrypto) {
-        // WinZip AES or NeoEncrypt: read full payload, decrypt in one shot (HMAC must be verified first)
+        // WinZip AES / NeoEncrypt: stream decrypt (HMAC verified at end of ciphertext)
         const dataStream = this.readCompressedDataStream(fileHandle, entry);
-        const chunks: Buffer[] = [];
-        for await (const chunk of dataStream) {
-          chunks.push(chunk);
-        }
-        const fullPayload = Buffer.concat(chunks);
-        const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
-
-        // Wrap decrypted data as a single-item generator for the decompression pipeline
+        const decrypted = AesCrypto.decryptStream(
+          (this.zipkitNode as any).password,
+          entry.compressedSize,
+          dataStream
+        );
         const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
-        async function* singleChunk() { yield decrypted; }
-        await this.unCompressToFile(singleChunk(), entry, writeStream, {
+        await this.unCompressToFile(decrypted, entry, writeStream, {
           skipHashCheck: skipHash,
           onProgress: options?.onProgress,
           outputPath
@@ -297,16 +306,13 @@ export class ZipDecompressNode {
 
       if (isEncrypted && isFullPayloadCrypto) {
         const dataStream = this.readCompressedDataStream(fileHandle, entry);
-        const chunks: Buffer[] = [];
-        for await (const chunk of dataStream) {
-          chunks.push(chunk);
-        }
-        const fullPayload = Buffer.concat(chunks);
-        const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
-
+        const decrypted = AesCrypto.decryptStream(
+          (this.zipkitNode as any).password,
+          entry.compressedSize,
+          dataStream
+        );
         const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
-        async function* singleChunk() { yield decrypted; }
-        return await this.unCompressToBuffer(singleChunk(), entry, {
+        return await this.unCompressToBuffer(decrypted, entry, {
           skipHashCheck: skipHash,
           onProgress: options?.onProgress
         });
@@ -357,16 +363,13 @@ export class ZipDecompressNode {
 
       if (isEncrypted && isFullPayloadCrypto) {
         const dataStream = this.readCompressedDataStream(fileHandle, entry);
-        const chunks: Buffer[] = [];
-        for await (const chunk of dataStream) {
-          chunks.push(chunk);
-        }
-        const fullPayload = Buffer.concat(chunks);
-        const decrypted = AesCrypto.decryptBuffer(entry, fullPayload, (this.zipkitNode as any).password);
-
+        const decrypted = AesCrypto.decryptStream(
+          (this.zipkitNode as any).password,
+          entry.compressedSize,
+          dataStream
+        );
         const skipHash = entry.aesVersion === 2 ? true : options?.skipHashCheck;
-        async function* singleChunk() { yield decrypted; }
-        return await this.unCompressToTest(singleChunk(), entry, {
+        return await this.unCompressToTest(decrypted, entry, {
           skipHashCheck: skipHash,
           onProgress: options?.onProgress
         });
@@ -448,6 +451,8 @@ export class ZipDecompressNode {
       if (!options?.skipHashCheck) {
         if (entry.sha256) {
           const calculatedHash = hashCalc.finalizeSHA256();
+          const leaf = hashCalc.finalizeMerkleLeafV1();
+          if (leaf) entry.merkleLeafV1 = leaf;
           this.log(`SHA-256 comparison: calculated=${calculatedHash}, stored=${entry.sha256}`);
           if (calculatedHash !== entry.sha256) {
             if (options?.outputPath && fs) {
@@ -537,6 +542,8 @@ export class ZipDecompressNode {
       if (!options?.skipHashCheck) {
         if (entry.sha256) {
           const calculatedHash = hashCalc.finalizeSHA256();
+          const leaf = hashCalc.finalizeMerkleLeafV1();
+          if (leaf) entry.merkleLeafV1 = leaf;
           this.log(`SHA-256 comparison: calculated=${calculatedHash}, stored=${entry.sha256}`);
           if (calculatedHash !== entry.sha256) {
             throw new Error(Errors.INVALID_SHA256);
@@ -604,6 +611,8 @@ export class ZipDecompressNode {
       if (!options?.skipHashCheck) {
         if (entry.sha256) {
           const calculatedHash = hashCalc.finalizeSHA256();
+          const leaf = hashCalc.finalizeMerkleLeafV1();
+          if (leaf) entry.merkleLeafV1 = leaf;
           this.log(`SHA-256 comparison: calculated=${calculatedHash}, stored=${entry.sha256}`);
           if (calculatedHash !== entry.sha256) {
             throw new Error(Errors.INVALID_SHA256);
@@ -622,7 +631,13 @@ export class ZipDecompressNode {
           return { verifiedHash: undefined };
         }
       } else {
-        // Hash check skipped - return undefined
+        // Hash check skipped — still finalize leaf/digest once from this stream when SHA was enabled
+        if (entry.sha256 && hashCalc) {
+          const calculatedHash = hashCalc.finalizeSHA256();
+          const leaf = hashCalc.finalizeMerkleLeafV1();
+          if (leaf) entry.merkleLeafV1 = leaf;
+          return { verifiedHash: calculatedHash || undefined };
+        }
         return { verifiedHash: undefined };
       }
     } catch (error) {
@@ -662,8 +677,8 @@ export class ZipDecompressNode {
       // Use pako streaming inflate - maintains state across chunks
       yield* this.inflateStream(compressedStream);
     } else if (method === CMP_METHOD.ZSTD) {
-      // Use ZSTD streaming decompression - note: ZSTD requires full buffer
-      yield* this.zstdDecompressStream(compressedStream);
+      // Native Node zstd; bound output to entry size (repairs legacy WASM +18 padding)
+      yield* this.zstdDecompressStream(compressedStream, entry?.uncompressedSize ?? 0);
     } else {
       throw new Error(`Unsupported compression method: ${method}`);
     }
@@ -707,36 +722,32 @@ export class ZipDecompressNode {
   }
 
   /**
-   * Streaming ZSTD decompression
+   * Streaming ZSTD decompression via Node zlib (native).
+   * When maxOut > 0, truncates output (legacy WASM emitted +18 zero bytes).
    */
   private async *zstdDecompressStream(
-    compressedStream: AsyncGenerator<Buffer>
+    compressedStream: AsyncGenerator<Buffer>,
+    maxOut: number = 0
   ): AsyncGenerator<Buffer> {
-    // ZSTD is guaranteed to be initialized via factory method
-    
-    // Collect all compressed chunks first (ZSTD needs complete data)
-    const compressedChunks: Buffer[] = [];
-    for await (const chunk of compressedStream) {
-      compressedChunks.push(chunk);
-    }
-    
-    const compressedData = Buffer.concat(compressedChunks);
-    
     try {
-      // Use global ZstdManager for decompression
-      const decompressed = await ZstdManager.decompress(compressedData);
-      const decompressedBuffer = Buffer.from(decompressed);
-      
-      // Yield decompressed data in chunks using ZipkitServer's bufferSize
-      const chunkSize = this.zipkitNode.getBufferSize();
-      let offset = 0;
-      while (offset < decompressedBuffer.length) {
-        const end = Math.min(offset + chunkSize, decompressedBuffer.length);
-        yield decompressedBuffer.slice(offset, end);
-        offset = end;
+      let remaining = maxOut > 0 ? maxOut : Number.POSITIVE_INFINITY;
+      for await (const chunk of ZstdNode.decompressStream(compressedStream)) {
+        if (remaining <= 0) {
+          break;
+        }
+        if (chunk.length <= remaining) {
+          remaining -= chunk.length;
+          yield chunk;
+        } else {
+          yield chunk.subarray(0, remaining);
+          remaining = 0;
+          break;
+        }
       }
     } catch (error) {
-      throw new Error(`ZSTD streaming decompression failed: ${error instanceof Error ? error.message : String(error)}`);
+      throw new Error(
+        `ZSTD streaming decompression failed: ${error instanceof Error ? error.message : String(error)}`
+      );
     }
   }
 
@@ -750,20 +761,35 @@ export class ZipDecompressNode {
   }
 
   /**
-   * Zstd decompress method (now async with ZstdManager)
-   * Internal method only
+   * Zstd decompress via Node zlib (native), repairing legacy WASM +18 padding when present.
    */
-  private async zstdDecompressSync(data: Buffer): Promise<Buffer> {
+  private async zstdDecompressSync(data: Buffer, entry?: ZipEntry): Promise<Buffer> {
     this.log(`zstdDecompressSync() called with ${data.length} bytes`);
-    
+
     try {
-      // Use global ZstdManager for decompression
-      const decompressed = await ZstdManager.decompress(data);
-      this.log(`ZSTD decompression successful: ${data.length} bytes -> ${decompressed.length} bytes`);
-      return Buffer.from(decompressed);
+      let decompressed = await ZstdNode.decompress(data);
+      if (entry && entry.uncompressedSize > 0) {
+        const { repaired, wasLegacy } = repairLegacyWasmZstdPlaintext(
+          decompressed,
+          entry.uncompressedSize,
+          entry.crc
+        );
+        if (wasLegacy) {
+          this.log(
+            `Legacy WASM zstd padding repaired: ${decompressed.length} -> ${repaired.length} bytes`
+          );
+        }
+        decompressed = repaired;
+      }
+      this.log(
+        `ZSTD decompression successful: ${data.length} bytes -> ${decompressed.length} bytes`
+      );
+      return decompressed;
     } catch (error) {
       this.log(`ZSTD decompression failed: ${error}`);
-      throw new Error(`ZSTD decompression failed: ${error instanceof Error ? error.message : String(error)}`);
+      throw new Error(
+        `ZSTD decompression failed: ${error instanceof Error ? error.message : String(error)}`
+      );
     }
   }
 
@@ -794,7 +820,7 @@ export class ZipDecompressNode {
     } else if (method === CMP_METHOD.DEFLATED) {
       outBuf = this.inflate(compressedData);
     } else if (method === CMP_METHOD.ZSTD) {
-      outBuf = await this.zstdDecompressSync(compressedData);
+      outBuf = await this.zstdDecompressSync(compressedData, entry);
     } else {
       throw new Error(`Unsupported compression method: ${method}`);
     }