neozipkit 1.0.2 → 1.0.5

package/src/core/Zipkit.ts

@@ -10,7 +10,14 @@ import { NEOZIPKIT_INFO } from '../types';
 import { ZipCompress, CompressOptions } from './ZipCompress';
 import ZipDecompress from './ZipDecompress';
 import { sha256, crc32 } from './encryption/ZipCrypto';
-import HashCalculator from './components/HashCalculator';
+import {
+  computeArchiveMerkleRoot,
+  computeMerkleRootV0FromDigests,
+  computeMerkleRootV1FromContents,
+  computeMerkleRootV1FromLeaves,
+  leafHashV1,
+  type MerkleAlgorithm,
+} from './merkle/MerkleRoot';
 import { 
   LOCAL_HDR,
   ENCRYPT_HDR_SIZE,
@@ -28,6 +35,11 @@ import {
   CENTRAL_DIRECTORY_END,
   HDR_ID
 } from './constants/Headers';
+import {
+  findReservedMetaEntry,
+  isMetaInfPath,
+  isReservedMetaPath,
+} from './constants/MetaPaths';
 
 /**
  * Streaming file handle interface
@@ -371,15 +383,22 @@ export default class Zipkit {
 
 
   /**
-   * Get specific ZIP entry by filename from zipEntries[] array
-   * 
+   * Get specific ZIP entry by filename from zipEntries[] array.
+   * Reserved META-INF paths (APPNOTE §2.3) also match ASCII case-insensitively
+   * when an exact spelling is not found.
+   *
    * @param filename - The name/path of the file/directory to find
    * @returns ZipEntry object if found, null if not found
    */
   getZipEntry(filename: string): ZipEntry | null {
     try {
       const centralDir = this.getDirectory();
-      return centralDir.find((entry: ZipEntry) => entry.filename === filename) || null;
+      const exact = centralDir.find((entry: ZipEntry) => entry.filename === filename);
+      if (exact) return exact;
+      if (isReservedMetaPath(filename)) {
+        return findReservedMetaEntry(centralDir, filename);
+      }
+      return null;
     } catch (error) {
       return null;
     }
@@ -686,112 +705,166 @@ export default class Zipkit {
   }
 
   /**
-   * Calculate Merkle Root of the ZIP file
-   * 
-   * Excludes metadata files (META-INF) to ensure consistent calculation.
-   * Simply calls getDirectory() and returns null if empty.
-   * 
-   * @returns Merkle root string or null if calculation fails
+   * Calculate Merkle root of archive content (non-META-INF entries).
+   *
+   * Per NEOZIP_APPNOTE.md §6:
+   * - Default **v1**: domain-separated leaves SHA-256(0x00 ‖ uncompressed bytes) and
+   *   SHA-256(0x01 ‖ left ‖ right) parents; paths NFC-normalized before sort.
+   * - Legacy **v0**: bare 0x014E digests as leaves, duplicate-odd pairing, path as stored.
+   *
+   * When only Extra Field digests are available (typical after open), **v1 requires
+   * payload bytes**. This sync method therefore returns null for v1; use
+   * {@link getMerkleRootAsync} (extracts content) or `{ algorithm: 'v0' }`.
+   * When you already hold payloads, use {@link merkleRootFromEntries}.
+   *
+   * @param options.algorithm - `v1` (default) or legacy `v0`
+   * @returns Lowercase hex Merkle root, or null if calculation fails
    */
-  getMerkleRoot(): string | null {
-    // Simply get directory - no loading logic
+  getMerkleRoot(options?: { algorithm?: MerkleAlgorithm }): string | null {
+    const algorithm: MerkleAlgorithm = options?.algorithm ?? 'v1';
+
     let zipEntries: ZipEntry[];
     try {
       zipEntries = this.getDirectory();
     } catch (error) {
-      // Catch any errors from getDirectory() - this should never happen
-      // but if it does, log it and return null
       Logger.error(`getMerkleRoot(): Error calling getDirectory(): ${error instanceof Error ? error.message : String(error)}`);
       Logger.error(`getMerkleRoot(): Stack trace: ${error instanceof Error ? error.stack : 'No stack trace'}`);
       return null;
     }
-    
+
     if (!zipEntries || zipEntries.length === 0) {
       return null;
     }
-  
-    const hashAccumulator = new HashCalculator({ enableAccumulation: true });
-    
-    // Filter out metadata files (META-INF) to ensure consistent Merkle Root calculation
-    // Inline check to avoid conflict with ZipkitBrowser's isMetadataFile() method
-    const contentEntries = zipEntries.filter(entry => {
-      const filename = entry.filename || '';
-      return filename !== TIMESTAMP_SUBMITTED &&
-             filename !== TIMESTAMP_METADATA &&
-             filename !== TOKENIZED_METADATA;
-    });
-    
-    for (const entry of contentEntries) {
-      if (entry.sha256) {
-        // Convert hex string to Buffer
-        const hashBuffer = Buffer.from(entry.sha256, 'hex');
-        hashAccumulator.addHash(hashBuffer);
-      }
+
+    const contentEntries = zipEntries.filter((entry) => !isMetaInfPath(entry.filename || ''));
+
+    if (algorithm === 'v0') {
+      const digests = contentEntries
+        .filter((e) => e.sha256)
+        .map((e) => ({ path: e.filename || '', sha256: e.sha256 as string }));
+      return computeMerkleRootV0FromDigests(digests);
     }
-    
-    const merkleRoot = hashAccumulator.merkleRoot();
-    if (!merkleRoot) {
-      return null;
+
+    // v1: prefer leaves captured in the same create/extract hash stream (no re-hash)
+    if (
+      contentEntries.length > 0 &&
+      contentEntries.every((e) => !!e.merkleLeafV1)
+    ) {
+      return computeMerkleRootV1FromLeaves(
+        contentEntries.map((e) => ({
+          path: e.filename || '',
+          merkleLeafV1: e.merkleLeafV1 as string,
+        }))
+      );
     }
 
-    return merkleRoot;
+    // v1: bare digests are not leaves — need getMerkleRootAsync() or stream leaves
+    return null;
   }
 
   /**
-   * Calculate Merkle Root of the ZIP file asynchronously
-   * 
-   * Excludes metadata files (META-INF) to ensure consistent calculation.
-   * Works with both buffer-based and file-based ZIPs.
-   * 
-   * @returns Promise resolving to Merkle root string or null if calculation fails
+   * Calculate Merkle root (preferred for v1 / new archives).
+   *
+   * **v1 (default):** uses precomputed stream leaves when available; otherwise extracts
+   * each content entry once and folds APPNOTE §6.3 root (no second content hash pass
+   * when extract captures merkleLeafV1).
+   * **v0:** uses Extra Field digests only (no extract), APPNOTE §6.2.
+   *
+   * META-INF/** is excluded. File-streaming mode: use ZipkitNode override (testEntry
+   * pipeline) rather than buffering the whole archive.
+   *
+   * @param options.algorithm - `v1` (default) or legacy `v0`
+   * @returns Promise resolving to lowercase hex root, or null
    */
-  async getMerkleRootAsync(): Promise<string | null> {
+  async getMerkleRootAsync(options?: { algorithm?: MerkleAlgorithm }): Promise<string | null> {
+    const algorithm: MerkleAlgorithm = options?.algorithm ?? 'v1';
+
     let zipEntries: ZipEntry[] = [];
-    
+
     try {
-      // Get entries based on ZIP type
-      if (this.inBuffer) {
-        zipEntries = this.getDirectory();
-      } else {
-        // File-based: cannot get entries synchronously - return null
-        // Caller should use ZipkitServer.getMerkleRootAsync() for file-based ZIPs
-        Logger.error('getMerkleRootAsync() called on file-based ZIP. Use ZipkitServer.getMerkleRootAsync() instead.');
-        return null;
-      }
+      zipEntries = this.getDirectory();
     } catch (error) {
       Logger.error(`Failed to get directory for merkle root calculation: ${error instanceof Error ? error.message : String(error)}`);
       return null;
     }
-    
+
     if (!zipEntries || zipEntries.length === 0) {
       return null;
     }
-  
-    const hashAccumulator = new HashCalculator({ enableAccumulation: true });
-    
-    // Filter out metadata files (META-INF) to ensure consistent Merkle Root calculation
-    // Inline check to avoid conflict with ZipkitBrowser's isMetadataFile() method
-    const contentEntries = zipEntries.filter(entry => {
-      const filename = entry.filename || '';
-      return filename !== TIMESTAMP_SUBMITTED &&
-             filename !== TIMESTAMP_METADATA &&
-             filename !== TOKENIZED_METADATA;
-    });
-    
+
+    const contentEntries = zipEntries.filter((entry) => !isMetaInfPath(entry.filename || ''));
+
+    if (algorithm === 'v0') {
+      const digests = contentEntries
+        .filter((e) => e.sha256)
+        .map((e) => ({ path: e.filename || '', sha256: e.sha256 as string }));
+      return computeMerkleRootV0FromDigests(digests);
+    }
+
+    // Reuse leaves already produced during write/extract (one stream hash pass)
+    if (
+      contentEntries.length > 0 &&
+      contentEntries.every((e) => !!e.merkleLeafV1)
+    ) {
+      return computeMerkleRootV1FromLeaves(
+        contentEntries.map((e) => ({
+          path: e.filename || '',
+          merkleLeafV1: e.merkleLeafV1 as string,
+        }))
+      );
+    }
+
+    // Buffer path: one decompress + dual hash inside extract (ZipDecompress)
+    if (!this.inBuffer) {
+      Logger.error(
+        'getMerkleRootAsync() called on file-based ZIP without precomputed merkle leaves. ' +
+          'Use ZipkitNode (streaming testEntry) or ensure create captured merkleLeafV1.'
+      );
+      return null;
+    }
+
     for (const entry of contentEntries) {
-      if (entry.sha256) {
-        // Convert hex string to Buffer
-        const hashBuffer = Buffer.from(entry.sha256, 'hex');
-        hashAccumulator.addHash(hashBuffer);
+      try {
+        const data = await this.extract(entry, false);
+        if (entry.merkleLeafV1) {
+          continue;
+        }
+        // Legacy decompress without dual leaf: hash payload once, cache leaf
+        if (data === null || data.length === 0) {
+          entry.merkleLeafV1 = leafHashV1(Buffer.alloc(0)).toString('hex');
+        } else {
+          entry.merkleLeafV1 = leafHashV1(data).toString('hex');
+        }
+      } catch (error) {
+        Logger.error(
+          `getMerkleRootAsync(): failed to extract ${entry.filename}: ${error instanceof Error ? error.message : String(error)}`
+        );
+        return null;
       }
     }
-    
-    const merkleRoot = hashAccumulator.merkleRoot();
-    if (!merkleRoot) {
+
+    if (!contentEntries.every((e) => !!e.merkleLeafV1)) {
       return null;
     }
 
-    return merkleRoot;
+    return computeMerkleRootV1FromLeaves(
+      contentEntries.map((e) => ({
+        path: e.filename || '',
+        merkleLeafV1: e.merkleLeafV1 as string,
+      }))
+    );
+  }
+
+  /**
+   * Compute Merkle root from known digests and/or payloads (APPNOTE §6).
+   * Prefer this during create when uncompressed bytes are still in hand, or when
+   * stream leaves (`merkleLeafV1`) were already produced.
+   */
+  static merkleRootFromEntries(
+    entries: Array<{ path: string; content?: Buffer; contentSha256?: string; merkleLeafV1?: string }>,
+    algorithm: MerkleAlgorithm = 'v1'
+  ): string | null {
+    return computeArchiveMerkleRoot(entries, algorithm);
   }
 
 }

package/src/core/components/HashCalculator.ts

@@ -45,6 +45,8 @@ export class HashCalculator {
   // Incremental hash calculation state (from StreamHashCalculator)
   private crc32State: number = ~0;
   private sha256Hash: crypto.Hash | null = null;
+  /** Domain-separated leaf hasher (APPNOTE §6.3): SHA-256(0x00 ‖ payload). */
+  private merkleLeafHash: crypto.Hash | null = null;
   private useSHA256: boolean = false;
 
   // Hash accumulation state (from HashAccumulator)
@@ -52,13 +54,15 @@ export class HashCalculator {
   private xorResult: Buffer = Buffer.alloc(32, 0);
   private enableAccumulation: boolean = false;
 
-  // Merkle tree state (from MerkleTree)
+  // Merkle tree state (from MerkleTree) — generic digest folding; archive roots use merkle/MerkleRoot.ts
   private merkleLeaves: Buffer[] = [];
   private merkleLayers: Buffer[][] = [];
   private merkleOptions: MerkleTreeOptions = {
     hashLeaves: false,
-    sortLeaves: true,
-    sortPairs: true,
+    // Archive content roots sort by path in MerkleRoot; generic accumulator keeps insertion order.
+    sortLeaves: false,
+    sortPairs: false,
+    // match APPNOTE v0 odd-leaf rule when used as a generic Bitcoin-style tree
     duplicateOdd: true
   };
 
@@ -77,6 +81,8 @@ export class HashCalculator {
 
     if (this.useSHA256) {
       this.sha256Hash = crypto.createHash('sha256');
+      this.merkleLeafHash = crypto.createHash('sha256');
+      this.merkleLeafHash.update(Buffer.from([0x00]));
     }
   }
 
@@ -95,10 +101,13 @@ export class HashCalculator {
       this.crc32State = crc32update(this.crc32State, chunk[i]);
     }
     
-    // Update SHA-256 incrementally
+    // Update SHA-256 and v1 Merkle leaf hash incrementally
     if (this.sha256Hash) {
       this.sha256Hash.update(chunk);
     }
+    if (this.merkleLeafHash) {
+      this.merkleLeafHash.update(chunk);
+    }
   }
 
   /**
@@ -110,7 +119,7 @@ export class HashCalculator {
   }
 
   /**
-   * Get final SHA-256 hash as hex string
+   * Get final SHA-256 hash as hex string (Extra Field 0x014E — bare content digest)
    * @returns SHA-256 hash as hex string, or null if SHA-256 not enabled
    */
   finalizeSHA256(): string | null {
@@ -120,13 +129,55 @@ export class HashCalculator {
     return null;
   }
 
+  /**
+   * Get APPNOTE §6.3 v1 Merkle leaf: SHA-256(0x00 ‖ uncompressed payload) as hex.
+   * Consumes the leaf hasher; call after or instead of finalizeSHA256 for dual streams.
+   */
+  finalizeMerkleLeafV1(): string | null {
+    if (this.merkleLeafHash) {
+      return this.merkleLeafHash.digest('hex');
+    }
+    return null;
+  }
+
+  /**
+   * Apply stream digests to a ZIP entry in one place.
+   *
+   * When SHA-256 was enabled, both Extra Field 0x014E (`sha256`) and the v1 Merkle leaf
+   * (`merkleLeafV1`) are finalized from the **same** chunk stream — no second payload pass.
+   */
+  applyToEntry(
+    entry: {
+      crc: number;
+      sha256: string | null;
+      merkleLeafV1?: string | null;
+    },
+    options: { setCrc?: boolean; setSha256?: boolean }
+  ): void {
+    if (options.setCrc) {
+      entry.crc = this.finalizeCRC32();
+    }
+    if (options.setSha256) {
+      const digest = this.finalizeSHA256();
+      const leaf = this.finalizeMerkleLeafV1();
+      if (digest) {
+        entry.sha256 = digest;
+      }
+      if (leaf) {
+        entry.merkleLeafV1 = leaf;
+      }
+    }
+  }
+
   /**
    * Reset the incremental hash calculation state
    */
   reset(): void {
     this.crc32State = ~0;
-    if (this.sha256Hash) {
+    if (this.useSHA256) {
       this.sha256Hash = crypto.createHash('sha256');
+      this.merkleLeafHash = crypto.createHash('sha256');
+      this.merkleLeafHash.update(Buffer.from([0x00]));
     }
   }
 
@@ -264,6 +315,7 @@ export class HashCalculator {
     if (this.merkleOptions.sortPairs && Buffer.compare(left, right) > 0) {
       [left, right] = [right, left];
     }
+    // Generic accumulator tree (not APPNOTE archive root — see merkle/MerkleRoot.ts)
     return this.hash(Buffer.concat([left, right]));
   }
 

package/src/core/components/Util.ts

@@ -6,7 +6,19 @@
 
 import Support from "./Support";
 import Errors from "../constants/Errors";
-import { TIMESTAMP_SUBMITTED, TIMESTAMP_METADATA, TOKENIZED_METADATA } from "../constants/Headers";
+import {
+  isMetaInfPath,
+  findReservedMetaEntry,
+  isReservedMetaPath,
+  asciiPathEqualsIgnoreCase,
+} from "../constants/MetaPaths";
+
+export {
+  findReservedMetaEntry,
+  isReservedMetaPath,
+  isMetaInfPath,
+  asciiPathEqualsIgnoreCase,
+} from "../constants/MetaPaths";
 
 export const DATATYPE = {
   STRING: "string",
@@ -80,12 +92,9 @@ export function getTypeOf(input: string|Buffer|Blob|Array<any>|ArrayBuffer|Uint8
 };
 
 /**
- * Check if a filename is a metadata file
- * @param filename The filename to check
- * @returns True if the file is a metadata file (META-INF)
+ * Check if a filename is NeoZip metadata (any META-INF/** path).
+ * Case-insensitive on the META-INF/ prefix (NEOZIP_APPNOTE.md §2.3 / §6.2).
  */
 export const isMetadataFile = (filename: string): boolean => {
-  return filename === TIMESTAMP_SUBMITTED || 
-         filename === TIMESTAMP_METADATA || 
-         filename === TOKENIZED_METADATA;
+  return isMetaInfPath(filename);
 };

package/src/core/constants/Headers.ts

@@ -11,9 +11,19 @@ export const ZIP64_CENTRAL_DIRECTORY_LOCATOR = 'PK\x06\x07';
 export const ZIP64_CENTRAL_DIRECTORY_END = 'PK\x06\x06';
 export const DATA_DESCRIPTOR = 'PK\x07\x08';
 
-export const TIMESTAMP_SUBMITTED = 'META-INF/TS-SUBMIT.OTS';
-export const TIMESTAMP_METADATA = 'META-INF/TIMESTAMP.OTS';
-export const TOKENIZED_METADATA = 'META-INF/NZIP.TOKEN';
+// Reserved META-INF paths — see constants/MetaPaths.ts and NEOZIP_APPNOTE.md §2
+export {
+  META_TOKEN_NZIP,
+  META_TOKEN_NZIP_LEGACY,
+  META_TS_SUBMIT_NZIP,
+  META_TIMESTAMP_NZIP,
+  META_TS_SUBMIT_OTS,
+  META_TIMESTAMP_OTS,
+  META_MANIFEST_JSON,
+  TIMESTAMP_SUBMITTED,
+  TIMESTAMP_METADATA,
+  TOKENIZED_METADATA,
+} from './MetaPaths';
 
 // Local file header
 export const LOCAL_HDR = {
@@ -176,7 +186,7 @@ export const HDR_ID = {
   x509PKCS7RL:     0x0019,    // X.509 Certificate Revocation List (PKCS#7)
   IBM1:            0x0065,    // IBM S/390 (Z390) - Attribute
   IBM2:            0x0066,    // IBM S/390 (Z390) - Attribute
-  SHA256:          0x014E,    // SHA256 "N\01" with 64 bytes (256 bit) of SHA256 hash data
+  SHA256:          0x014E,    // NeoZip Extra Field: 32-byte SHA-256 of uncompressed payload (APPNOTE §5)
   NEO_CRYPTO:      0x024E,    // NeoEncrypt metadata (see docs/NEO_CRYPTO_FORMAT.md); LE on wire: 4E 02
   POSZIP:          0x4690,    // POSZIP 4690
   UNV_TIME:        0x5455,    // Universal Time (UT) (32-bit) 5 data bytes

package/src/core/constants/MetaPaths.ts

@@ -0,0 +1,86 @@
+/**
+ * NeoZip reserved META-INF paths and ASCII case-insensitive discovery.
+ * Matches NEOZIP_APPNOTE.md §2.3: writers emit canonical spellings; readers fold case.
+ */
+
+/** Canonical writers MUST emit these spellings. */
+export const META_TOKEN_NZIP = 'META-INF/TOKEN.NZIP';
+export const META_TOKEN_NZIP_LEGACY = 'META-INF/NZIP.TOKEN';
+export const META_TS_SUBMIT_NZIP = 'META-INF/TS-SUBMIT.NZIP';
+export const META_TIMESTAMP_NZIP = 'META-INF/TIMESTAMP.NZIP';
+export const META_TS_SUBMIT_OTS = 'META-INF/TS-SUBMIT.OTS';
+export const META_TIMESTAMP_OTS = 'META-INF/TIMESTAMP.OTS';
+export const META_MANIFEST_JSON = 'META-INF/manifest.json';
+
+/** @deprecated Prefer META_TS_SUBMIT_OTS — kept for existing imports. */
+export const TIMESTAMP_SUBMITTED = META_TS_SUBMIT_OTS;
+/** @deprecated Prefer META_TIMESTAMP_OTS — kept for existing imports. */
+export const TIMESTAMP_METADATA = META_TIMESTAMP_OTS;
+/** @deprecated Prefer META_TOKEN_NZIP — kept for existing imports (canonical TOKEN.NZIP). */
+export const TOKENIZED_METADATA = META_TOKEN_NZIP;
+
+const RESERVED_CANONICAL = [
+  META_TOKEN_NZIP,
+  META_TOKEN_NZIP_LEGACY,
+  META_TS_SUBMIT_NZIP,
+  META_TIMESTAMP_NZIP,
+  META_TS_SUBMIT_OTS,
+  META_TIMESTAMP_OTS,
+  META_MANIFEST_JSON,
+] as const;
+
+export type ZipEntryLikeFilename = { filename?: string };
+
+/** ASCII case-insensitive path compare (APPNOTE §2.3). */
+export function asciiPathEqualsIgnoreCase(a: string, b: string): boolean {
+  if (a.length !== b.length) {
+    // Still allow length mismatch only via toLowerCase for UTF-8 safety on ASCII paths
+  }
+  return a.toLowerCase() === b.toLowerCase();
+}
+
+/** True when path is under META-INF/ (ASCII case-insensitive). */
+export function isMetaInfPath(filename: string): boolean {
+  return filename.toLowerCase().startsWith('meta-inf/');
+}
+
+/** True when filename is a reserved NeoZip meta path (any casing). */
+export function isReservedMetaPath(filename: string): boolean {
+  const lower = filename.toLowerCase();
+  return RESERVED_CANONICAL.some((c) => c.toLowerCase() === lower);
+}
+
+/**
+ * True for any META-INF/** entry (integrity merkle leaves exclude these per APPNOTE §6.2).
+ */
+export function isMetadataFile(filename: string): boolean {
+  return isMetaInfPath(filename);
+}
+
+/**
+ * Find a reserved meta entry among ZIP directory entries.
+ * Prefers an exact canonical match, then ASCII case-insensitive match.
+ * When multiple preferred names are given, earlier names win.
+ */
+export function findReservedMetaEntry<T extends ZipEntryLikeFilename>(
+  entries: T[],
+  preferredCanonicalNames: string | readonly string[]
+): T | null {
+  const preferred = Array.isArray(preferredCanonicalNames)
+    ? preferredCanonicalNames
+    : [preferredCanonicalNames];
+
+  for (const name of preferred) {
+    const exact = entries.find((e) => e.filename === name);
+    if (exact) return exact;
+  }
+
+  for (const name of preferred) {
+    const folded = entries.find(
+      (e) => typeof e.filename === 'string' && asciiPathEqualsIgnoreCase(e.filename, name)
+    );
+    if (folded) return folded;
+  }
+
+  return null;
+}

package/src/core/index.ts

@@ -24,6 +24,7 @@ export { default as HashCalculator, HashCalculator as HashCalculatorClass } from
 export * from './components/Util';
 export * from './components/Support';
 export { ProgressTracker } from './components/ProgressTracker';
+export * from './merkle/MerkleRoot';
 
 // Encryption functionality
 export { EncryptionManager } from './encryption/Manager';
@@ -33,6 +34,19 @@ export type { EncryptionMethod, EncryptionOptions, EncryptionResult, DecryptionR
 // Types and constants
 export * from '../types';
 export * from './constants/Headers';
+export {
+  META_TOKEN_NZIP,
+  META_TOKEN_NZIP_LEGACY,
+  META_TS_SUBMIT_NZIP,
+  META_TIMESTAMP_NZIP,
+  META_TS_SUBMIT_OTS,
+  META_TIMESTAMP_OTS,
+  META_MANIFEST_JSON,
+  asciiPathEqualsIgnoreCase,
+  isMetaInfPath,
+  isReservedMetaPath,
+  findReservedMetaEntry,
+} from './constants/MetaPaths';
 export * from './constants/Errors';
 export * from './version';