neozipkit 0.8.0 → 1.0.5

package/src/core/ZipCompress.ts

@@ -14,7 +14,7 @@
 //
 
 const pako = require('pako');
-import { ZstdManager } from './ZstdManager';
+import { ZstdNode } from '../node/ZstdNode';
 import Zipkit from './Zipkit';
 import { Logger } from './components/Logger';
 import ZipEntry from './ZipEntry';
@@ -31,9 +31,9 @@ import { NeoCrypto, NEO_CRYPTO_ALGORITHM_AES256_V1 } from './encryption/NeoCrypt
 export interface CompressOptions {
   level?: number;             // Compression level (1-9, 0=store)
   password?: string | null;   // Password for encryption
-  encryptionMethod?: 'aes256' | 'zipcrypto' | 'neo-aes256'; // default WinZip AES when password set; 'neo-aes256' = NeoEncrypt extra 0x024E
+  encryptionMethod?: 'aes256' | 'zipcrypto' | 'neo-aes256'; // default NeoEncrypt when password set; 'aes256' = WinZip AE (method 99); 'neo-aes256' = Extra Field 0x024E
   useSHA256?: boolean;        // Whether to calculate SHA256 hash default is false
-  useZstd?: boolean;          // Whether to use Zstandard compression default is true
+  useZstd?: boolean;          // Use Zstandard (method 93); default false (Deflate). Node ≥ 22.15 required when true.
   bufferSize?: number;        // Override default buffer size
 }
 
@@ -132,20 +132,19 @@ export class ZipCompress {
     // Calculate hashes if needed
     if (hashCalculator) {
       hashCalculator.update(data);
-      if (!entry.crc || entry.crc === 0) {
-        entry.crc = hashCalculator.finalizeCRC32();
-      }
-      if (options?.useSHA256 && !entry.sha256) {
-        entry.sha256 = hashCalculator.finalizeSHA256();
-      }
+      hashCalculator.applyToEntry(entry, {
+        setCrc: !entry.crc || entry.crc === 0,
+        setSha256: !!(options?.useSHA256 && !entry.sha256),
+      });
       this.log(`Final hashes: CRC32=0x${entry.crc.toString(16).padStart(8, '0')}, SHA256=${entry.sha256 || 'N/A'}`);
     }
     
     // Encrypt if password provided
     const enc = options?.encryptionMethod;
     const useZipCrypto = enc === 'zipcrypto';
-    const useNeo = enc === 'neo-aes256';
-    const useAes = options?.password && !useZipCrypto && !useNeo;
+    // Explicit WinZip AE only when encryptionMethod is 'aes256'. Otherwise NeoEncrypt (default).
+    const useAes = enc === 'aes256';
+    const useNeo = !!options?.password && !useZipCrypto && !useAes;
     if (options?.password) {
       const label = useZipCrypto ? 'ZipCrypto' : useNeo ? 'NeoEncrypt (AES-256)' : 'WinZip AES-256';
       this.log(`Encrypting compressed data for entry: ${entry.filename} (method: ${label})`);
@@ -275,13 +274,8 @@ export class ZipCompress {
   }
 
   /**
-   * Compress data using Zstandard (zstd) algorithm
-   * @param input - Input data to compress (Buffer or chunked reader)
-   * @param options - Compression options
-   * @param bufferSize - Buffer size for chunked processing
-   * @param entry - ZIP entry being compressed
-   * @param onOutputBuffer - Optional callback for streaming output
-   * @returns Buffer containing compressed data
+   * Compress data using Zstandard via Node native zlib chunked streaming.
+   * Not available in browser builds (throws).
    */
   async zstdCompress(
     input: Buffer | { totalSize: number, readChunk: (position: number, size: number) => Buffer },
@@ -291,33 +285,66 @@ export class ZipCompress {
     onOutputBuffer?: (data: Buffer) => Promise<void>
   ): Promise<Buffer> {
     this.log(`zstdCompress() called - entry: ${entry?.filename ?? 'unknown'}, bufferSize: ${bufferSize}, level: ${options?.level ?? 6}`);
-    
+
     const effectiveBufferSize = bufferSize || this.zipkit.getBufferSize();
     const level = options?.level ?? 6;
-    
-    // Handle chunked reader
-    if (typeof input === 'object' && 'totalSize' in input && 'readChunk' in input) {
-      // Chunked reader mode - not implemented in simplified version
-      throw new Error('Chunked reader mode not supported in ZipCompress');
-    }
-    
-    // Validate input
-    if (!input || input.length === 0) {
+    const isBuffer = Buffer.isBuffer(input);
+    const totalSize = isBuffer ? input.length : input.totalSize;
+
+    if (!totalSize) {
       throw new Error('ZSTD compression: empty input buffer');
     }
-    
-    // Convert Buffer to Uint8Array for WASM module
-    const inputArray = new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
-    
-    // Use global ZstdManager for compression (handles queuing and initialization)
-    const compressed = await ZstdManager.compress(inputArray, level);
-    const compressedBuffer = Buffer.from(compressed);
-    
+
+    if (options?.level === 0) {
+      if (isBuffer) {
+        if (onOutputBuffer) {
+          await onOutputBuffer(input);
+        }
+        return input;
+      }
+      const chunks: Buffer[] = [];
+      let position = 0;
+      while (position < totalSize) {
+        const size = Math.min(effectiveBufferSize, totalSize - position);
+        const chunk = input.readChunk(position, size);
+        if (onOutputBuffer) {
+          await onOutputBuffer(chunk);
+        } else {
+          chunks.push(chunk);
+        }
+        position += chunk.length;
+      }
+      return onOutputBuffer ? Buffer.alloc(0) : Buffer.concat(chunks);
+    }
+
+    const readChunk = isBuffer
+      ? (position: number, size: number) => input.subarray(position, position + size)
+      : input.readChunk.bind(input);
+
+    const collected: Buffer[] = [];
+    const compressedSize = await ZstdNode.compressChunks(
+      readChunk,
+      totalSize,
+      effectiveBufferSize,
+      level,
+      undefined,
+      async (compressed) => {
+        if (onOutputBuffer) {
+          await onOutputBuffer(compressed);
+        } else {
+          collected.push(compressed);
+        }
+      }
+    );
+
+    if (entry) {
+      entry.compressedSize = compressedSize;
+    }
+
     if (onOutputBuffer) {
-      await onOutputBuffer(compressedBuffer);
+      return Buffer.alloc(0);
     }
-    
-    return compressedBuffer;
+    return Buffer.concat(collected);
   }
 
   /**
@@ -397,9 +424,11 @@ export class ZipCompress {
     hashCalculator.update(fileData);
     
     // Set hashes
-    entry.crc = hashCalculator.finalizeCRC32();
-    if (cmpOptions?.useSHA256) {
-      entry.sha256 = hashCalculator.finalizeSHA256();
+    hashCalculator.applyToEntry(entry, {
+      setCrc: true,
+      setSha256: !!cmpOptions?.useSHA256,
+    });
+    if (entry.sha256) {
       this.log(`SHA-256 calculated: ${entry.sha256}`);
     }
     

package/src/core/ZipDecompress.ts

@@ -14,7 +14,8 @@
 //
 
 const pako = require('pako');
-import { ZstdManager } from './ZstdManager';
+import { ZstdNode } from '../node/ZstdNode';
+import { repairLegacyWasmZstdPlaintext } from '../node/LegacyZstd';
 import Zipkit from './Zipkit';
 import { Logger } from './components/Logger';
 import ZipEntry from './ZipEntry';
@@ -22,6 +23,7 @@ import Errors from './constants/Errors';
 import { CMP_METHOD } from './constants/Headers';
 import { ZipCrypto } from './encryption/ZipCrypto';
 import { AesCrypto } from './encryption/AesCrypto';
+import { HashCalculator } from './components/HashCalculator';
 
 export interface DecompressionResult {
   success: boolean;
@@ -128,18 +130,28 @@ class ZipDecompress {
 
 
   /**
-   * Synchronous zstd decompress method for in-memory mode
-   * ZSTD codec is guaranteed to be initialized via factory method
-   * Internal method only
+   * Zstd decompress via Node native zlib, repairing legacy WASM +18 padding when present.
    */
-  private async zstdDecompressSync(data: Buffer): Promise<Buffer> {
+  private async zstdDecompressSync(data: Buffer, entry?: ZipEntry): Promise<Buffer> {
     this.log(`zstdDecompressSync() called with ${data.length} bytes`);
-    
+
     try {
-      // Use global ZstdManager for decompression (handles queuing and initialization)
-      const decompressed = await ZstdManager.decompress(data);
+      let decompressed = await ZstdNode.decompress(data);
+      if (entry && entry.uncompressedSize > 0) {
+        const { repaired, wasLegacy } = repairLegacyWasmZstdPlaintext(
+          decompressed,
+          entry.uncompressedSize,
+          entry.crc
+        );
+        if (wasLegacy) {
+          this.log(
+            `Legacy WASM zstd padding repaired: ${decompressed.length} -> ${repaired.length} bytes`
+          );
+        }
+        decompressed = repaired;
+      }
       this.log(`ZSTD decompression successful: ${data.length} bytes -> ${decompressed.length} bytes`);
-      return Buffer.from(decompressed);
+      return decompressed;
     } catch (error) {
       this.log(`ZSTD decompression failed: ${error}`);
       throw new Error(`ZSTD decompression failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -173,23 +185,28 @@ class ZipDecompress {
     } else if (method === CMP_METHOD.DEFLATED) {
       outBuf = this.inflate(compressedData);
     } else if (method === CMP_METHOD.ZSTD) {
-      outBuf = await this.zstdDecompressSync(compressedData);
+      outBuf = await this.zstdDecompressSync(compressedData, entry);
     } else {
       throw new Error(`Unsupported compression method: ${method}`);
     }
 
-    // Verify hash
+    // One pass: CRC + bare SHA-256 (0x014E) + v1 Merkle leaf — no second payload read
+    const hashCalc = new HashCalculator({ useSHA256: true });
+    hashCalc.update(outBuf);
+    const calculatedCRC = hashCalc.finalizeCRC32();
+    const calculatedHash = hashCalc.finalizeSHA256();
+    const leaf = hashCalc.finalizeMerkleLeafV1();
+    if (leaf) {
+      entry.merkleLeafV1 = leaf;
+    }
+
     if (!skipHashCheck) {
       if (entry.sha256) {
-        const isValid = this.zipkit.testSHA256(entry, outBuf);
-        if (!isValid) {
+        if (calculatedHash !== entry.sha256) {
           throw new Error(Errors.INVALID_SHA256);
         }
-      } else {
-        const isValid = this.zipkit.testCRC32(entry, outBuf);
-        if (!isValid) {
-          throw new Error(Errors.INVALID_CRC);
-        }
+      } else if (calculatedCRC !== entry.crc) {
+        throw new Error(Errors.INVALID_CRC);
       }
     }
 

package/src/core/ZipEntry.ts

@@ -82,7 +82,14 @@ export default class ZipEntry implements ZipFileEntry {
   universalTime: number | null = null; // Universal Time
   uid: number | null = null;        // User ID
   gid: number | null = null;        // Group ID
-  sha256: string | null = null;     // SHA-256 hash of the file
+  sha256: string | null = null;     // SHA-256 hash of the file (Extra Field 0x014E, bare content digest)
+
+  /**
+   * APPNOTE §6.3 v1 Merkle leaf: SHA-256(0x00 ‖ uncompressed payload).
+   * In-memory only (not written as a ZIP Extra today). Filled in the same streaming hash
+   * pass as `sha256` during create/extract — never reconstruct by re-reading payload when set.
+   */
+  merkleLeafV1: string | null = null;
   
   // Symbolic link data
   isSymlink: boolean = false;       // Entry is a symbolic link

package/src/core/Zipkit.ts

@@ -10,7 +10,14 @@ import { NEOZIPKIT_INFO } from '../types';
 import { ZipCompress, CompressOptions } from './ZipCompress';
 import ZipDecompress from './ZipDecompress';
 import { sha256, crc32 } from './encryption/ZipCrypto';
-import HashCalculator from './components/HashCalculator';
+import {
+  computeArchiveMerkleRoot,
+  computeMerkleRootV0FromDigests,
+  computeMerkleRootV1FromContents,
+  computeMerkleRootV1FromLeaves,
+  leafHashV1,
+  type MerkleAlgorithm,
+} from './merkle/MerkleRoot';
 import { 
   LOCAL_HDR,
   ENCRYPT_HDR_SIZE,
@@ -28,6 +35,11 @@ import {
   CENTRAL_DIRECTORY_END,
   HDR_ID
 } from './constants/Headers';
+import {
+  findReservedMetaEntry,
+  isMetaInfPath,
+  isReservedMetaPath,
+} from './constants/MetaPaths';
 
 /**
  * Streaming file handle interface
@@ -371,15 +383,22 @@ export default class Zipkit {
 
 
   /**
-   * Get specific ZIP entry by filename from zipEntries[] array
-   * 
+   * Get specific ZIP entry by filename from zipEntries[] array.
+   * Reserved META-INF paths (APPNOTE §2.3) also match ASCII case-insensitively
+   * when an exact spelling is not found.
+   *
    * @param filename - The name/path of the file/directory to find
    * @returns ZipEntry object if found, null if not found
    */
   getZipEntry(filename: string): ZipEntry | null {
     try {
       const centralDir = this.getDirectory();
-      return centralDir.find((entry: ZipEntry) => entry.filename === filename) || null;
+      const exact = centralDir.find((entry: ZipEntry) => entry.filename === filename);
+      if (exact) return exact;
+      if (isReservedMetaPath(filename)) {
+        return findReservedMetaEntry(centralDir, filename);
+      }
+      return null;
     } catch (error) {
       return null;
     }
@@ -686,112 +705,166 @@ export default class Zipkit {
   }
 
   /**
-   * Calculate Merkle Root of the ZIP file
-   * 
-   * Excludes metadata files (META-INF) to ensure consistent calculation.
-   * Simply calls getDirectory() and returns null if empty.
-   * 
-   * @returns Merkle root string or null if calculation fails
+   * Calculate Merkle root of archive content (non-META-INF entries).
+   *
+   * Per NEOZIP_APPNOTE.md §6:
+   * - Default **v1**: domain-separated leaves SHA-256(0x00 ‖ uncompressed bytes) and
+   *   SHA-256(0x01 ‖ left ‖ right) parents; paths NFC-normalized before sort.
+   * - Legacy **v0**: bare 0x014E digests as leaves, duplicate-odd pairing, path as stored.
+   *
+   * When only Extra Field digests are available (typical after open), **v1 requires
+   * payload bytes**. This sync method therefore returns null for v1; use
+   * {@link getMerkleRootAsync} (extracts content) or `{ algorithm: 'v0' }`.
+   * When you already hold payloads, use {@link merkleRootFromEntries}.
+   *
+   * @param options.algorithm - `v1` (default) or legacy `v0`
+   * @returns Lowercase hex Merkle root, or null if calculation fails
    */
-  getMerkleRoot(): string | null {
-    // Simply get directory - no loading logic
+  getMerkleRoot(options?: { algorithm?: MerkleAlgorithm }): string | null {
+    const algorithm: MerkleAlgorithm = options?.algorithm ?? 'v1';
+
     let zipEntries: ZipEntry[];
     try {
       zipEntries = this.getDirectory();
     } catch (error) {
-      // Catch any errors from getDirectory() - this should never happen
-      // but if it does, log it and return null
       Logger.error(`getMerkleRoot(): Error calling getDirectory(): ${error instanceof Error ? error.message : String(error)}`);
       Logger.error(`getMerkleRoot(): Stack trace: ${error instanceof Error ? error.stack : 'No stack trace'}`);
       return null;
     }
-    
+
     if (!zipEntries || zipEntries.length === 0) {
       return null;
     }
-  
-    const hashAccumulator = new HashCalculator({ enableAccumulation: true });
-    
-    // Filter out metadata files (META-INF) to ensure consistent Merkle Root calculation
-    // Inline check to avoid conflict with ZipkitBrowser's isMetadataFile() method
-    const contentEntries = zipEntries.filter(entry => {
-      const filename = entry.filename || '';
-      return filename !== TIMESTAMP_SUBMITTED &&
-             filename !== TIMESTAMP_METADATA &&
-             filename !== TOKENIZED_METADATA;
-    });
-    
-    for (const entry of contentEntries) {
-      if (entry.sha256) {
-        // Convert hex string to Buffer
-        const hashBuffer = Buffer.from(entry.sha256, 'hex');
-        hashAccumulator.addHash(hashBuffer);
-      }
+
+    const contentEntries = zipEntries.filter((entry) => !isMetaInfPath(entry.filename || ''));
+
+    if (algorithm === 'v0') {
+      const digests = contentEntries
+        .filter((e) => e.sha256)
+        .map((e) => ({ path: e.filename || '', sha256: e.sha256 as string }));
+      return computeMerkleRootV0FromDigests(digests);
     }
-    
-    const merkleRoot = hashAccumulator.merkleRoot();
-    if (!merkleRoot) {
-      return null;
+
+    // v1: prefer leaves captured in the same create/extract hash stream (no re-hash)
+    if (
+      contentEntries.length > 0 &&
+      contentEntries.every((e) => !!e.merkleLeafV1)
+    ) {
+      return computeMerkleRootV1FromLeaves(
+        contentEntries.map((e) => ({
+          path: e.filename || '',
+          merkleLeafV1: e.merkleLeafV1 as string,
+        }))
+      );
     }
 
-    return merkleRoot;
+    // v1: bare digests are not leaves — need getMerkleRootAsync() or stream leaves
+    return null;
   }
 
   /**
-   * Calculate Merkle Root of the ZIP file asynchronously
-   * 
-   * Excludes metadata files (META-INF) to ensure consistent calculation.
-   * Works with both buffer-based and file-based ZIPs.
-   * 
-   * @returns Promise resolving to Merkle root string or null if calculation fails
+   * Calculate Merkle root (preferred for v1 / new archives).
+   *
+   * **v1 (default):** uses precomputed stream leaves when available; otherwise extracts
+   * each content entry once and folds APPNOTE §6.3 root (no second content hash pass
+   * when extract captures merkleLeafV1).
+   * **v0:** uses Extra Field digests only (no extract), APPNOTE §6.2.
+   *
+   * META-INF/** is excluded. File-streaming mode: use ZipkitNode override (testEntry
+   * pipeline) rather than buffering the whole archive.
+   *
+   * @param options.algorithm - `v1` (default) or legacy `v0`
+   * @returns Promise resolving to lowercase hex root, or null
    */
-  async getMerkleRootAsync(): Promise<string | null> {
+  async getMerkleRootAsync(options?: { algorithm?: MerkleAlgorithm }): Promise<string | null> {
+    const algorithm: MerkleAlgorithm = options?.algorithm ?? 'v1';
+
     let zipEntries: ZipEntry[] = [];
-    
+
     try {
-      // Get entries based on ZIP type
-      if (this.inBuffer) {
-        zipEntries = this.getDirectory();
-      } else {
-        // File-based: cannot get entries synchronously - return null
-        // Caller should use ZipkitServer.getMerkleRootAsync() for file-based ZIPs
-        Logger.error('getMerkleRootAsync() called on file-based ZIP. Use ZipkitServer.getMerkleRootAsync() instead.');
-        return null;
-      }
+      zipEntries = this.getDirectory();
     } catch (error) {
       Logger.error(`Failed to get directory for merkle root calculation: ${error instanceof Error ? error.message : String(error)}`);
       return null;
     }
-    
+
     if (!zipEntries || zipEntries.length === 0) {
       return null;
     }
-  
-    const hashAccumulator = new HashCalculator({ enableAccumulation: true });
-    
-    // Filter out metadata files (META-INF) to ensure consistent Merkle Root calculation
-    // Inline check to avoid conflict with ZipkitBrowser's isMetadataFile() method
-    const contentEntries = zipEntries.filter(entry => {
-      const filename = entry.filename || '';
-      return filename !== TIMESTAMP_SUBMITTED &&
-             filename !== TIMESTAMP_METADATA &&
-             filename !== TOKENIZED_METADATA;
-    });
-    
+
+    const contentEntries = zipEntries.filter((entry) => !isMetaInfPath(entry.filename || ''));
+
+    if (algorithm === 'v0') {
+      const digests = contentEntries
+        .filter((e) => e.sha256)
+        .map((e) => ({ path: e.filename || '', sha256: e.sha256 as string }));
+      return computeMerkleRootV0FromDigests(digests);
+    }
+
+    // Reuse leaves already produced during write/extract (one stream hash pass)
+    if (
+      contentEntries.length > 0 &&
+      contentEntries.every((e) => !!e.merkleLeafV1)
+    ) {
+      return computeMerkleRootV1FromLeaves(
+        contentEntries.map((e) => ({
+          path: e.filename || '',
+          merkleLeafV1: e.merkleLeafV1 as string,
+        }))
+      );
+    }
+
+    // Buffer path: one decompress + dual hash inside extract (ZipDecompress)
+    if (!this.inBuffer) {
+      Logger.error(
+        'getMerkleRootAsync() called on file-based ZIP without precomputed merkle leaves. ' +
+          'Use ZipkitNode (streaming testEntry) or ensure create captured merkleLeafV1.'
+      );
+      return null;
+    }
+
     for (const entry of contentEntries) {
-      if (entry.sha256) {
-        // Convert hex string to Buffer
-        const hashBuffer = Buffer.from(entry.sha256, 'hex');
-        hashAccumulator.addHash(hashBuffer);
+      try {
+        const data = await this.extract(entry, false);
+        if (entry.merkleLeafV1) {
+          continue;
+        }
+        // Legacy decompress without dual leaf: hash payload once, cache leaf
+        if (data === null || data.length === 0) {
+          entry.merkleLeafV1 = leafHashV1(Buffer.alloc(0)).toString('hex');
+        } else {
+          entry.merkleLeafV1 = leafHashV1(data).toString('hex');
+        }
+      } catch (error) {
+        Logger.error(
+          `getMerkleRootAsync(): failed to extract ${entry.filename}: ${error instanceof Error ? error.message : String(error)}`
+        );
+        return null;
       }
     }
-    
-    const merkleRoot = hashAccumulator.merkleRoot();
-    if (!merkleRoot) {
+
+    if (!contentEntries.every((e) => !!e.merkleLeafV1)) {
       return null;
     }
 
-    return merkleRoot;
+    return computeMerkleRootV1FromLeaves(
+      contentEntries.map((e) => ({
+        path: e.filename || '',
+        merkleLeafV1: e.merkleLeafV1 as string,
+      }))
+    );
+  }
+
+  /**
+   * Compute Merkle root from known digests and/or payloads (APPNOTE §6).
+   * Prefer this during create when uncompressed bytes are still in hand, or when
+   * stream leaves (`merkleLeafV1`) were already produced.
+   */
+  static merkleRootFromEntries(
+    entries: Array<{ path: string; content?: Buffer; contentSha256?: string; merkleLeafV1?: string }>,
+    algorithm: MerkleAlgorithm = 'v1'
+  ): string | null {
+    return computeArchiveMerkleRoot(entries, algorithm);
   }
 
 }