neozipkit 0.8.0 → 1.0.5

package/src/core/merkle/MerkleRoot.ts

@@ -0,0 +1,312 @@
+// ======================================
+//  MerkleRoot.ts — NeoZip APPNOTE §6
+//  v0 (legacy Bitcoin-style) and v1 (RFC 6962–style domain separation)
+// ======================================
+
+import * as crypto from 'crypto';
+import { isMetaInfPath } from '../constants/MetaPaths';
+
+/** Merkle algorithms defined in NEOZIP_APPNOTE.md §6. */
+export type MerkleAlgorithm = 'v0' | 'v1';
+
+export interface MerkleContentEntry {
+  /** Central-directory path (UTF-8 string as stored or as known to the writer). */
+  path: string;
+  /**
+   * Uncompressed payload bytes (same bytes hashed into Extra Field 0x014E).
+   * Required for **v1** leaves when `merkleLeafV1` is absent. Optional for **v0** when `contentSha256` is set.
+   */
+  content?: Buffer | Uint8Array;
+  /**
+   * Bare SHA-256 of uncompressed payload (= Extra Field 0x014E), hex or 32-byte Buffer.
+   * Used as v0 leaf; for v1 this alone is **not** a leaf (see APPNOTE §6.3).
+   */
+  contentSha256?: string | Buffer;
+  /**
+   * Precomputed APPNOTE §6.3 v1 leaf: SHA-256(0x00 ‖ uncompressed_bytes), hex or 32-byte Buffer.
+   * Produced in the same streaming pass as `contentSha256` (create/extract via HashCalculator).
+   * Preferred for v1 when present — no second content read.
+   */
+  merkleLeafV1?: string | Buffer;
+}
+
+const LEAF_DOMAIN = Buffer.from([0x00]);
+const NODE_DOMAIN = Buffer.from([0x01]);
+
+function sha256(data: Buffer | Uint8Array | Buffer[]): Buffer {
+  const h = crypto.createHash('sha256');
+  if (Array.isArray(data)) {
+    for (const part of data) {
+      h.update(part);
+    }
+  } else {
+    h.update(data);
+  }
+  return h.digest();
+}
+
+function asBuffer(data: Buffer | Uint8Array | string, encoding?: BufferEncoding): Buffer {
+  if (typeof data === 'string') {
+    return Buffer.from(data, encoding ?? 'hex');
+  }
+  return Buffer.isBuffer(data) ? data : Buffer.from(data);
+}
+
+/**
+ * Normalize a ZIP entry path for v1 leaf ordering (APPNOTE §6.3):
+ * POSIX separators, strip leading `./` and `/`, Unicode NFC, UTF-8 byte order when compared.
+ */
+export function normalizeMerklePath(path: string): string {
+  let p = path.replace(/\\/g, '/');
+  if (p.startsWith('./')) {
+    p = p.slice(2);
+  }
+  while (p.startsWith('/')) {
+    p = p.slice(1);
+  }
+  if (typeof p.normalize === 'function') {
+    p = p.normalize('NFC');
+  }
+  return p;
+}
+
+/** Sort key for path comparison (UTF-8 byte order). */
+function pathSortKey(path: string, algorithm: MerkleAlgorithm): Buffer {
+  const normalized = algorithm === 'v1' ? normalizeMerklePath(path) : path;
+  return Buffer.from(normalized, 'utf8');
+}
+
+/** Bare content digest (= 0x014E value). */
+export function contentDigest(content: Buffer | Uint8Array): Buffer {
+  return sha256(asBuffer(content));
+}
+
+/**
+ * v0 leaf: SHA-256(uncompressed bytes) — same as Extra Field 0x014E.
+ */
+export function leafHashV0(contentOrDigest: Buffer | Uint8Array, isDigest = false): Buffer {
+  const buf = asBuffer(contentOrDigest);
+  return isDigest ? buf : sha256(buf);
+}
+
+/**
+ * v1 leaf: SHA-256(0x00 ‖ uncompressed_bytes).
+ * Must not be confused with the bare 0x014E digest (APPNOTE §6.3).
+ */
+export function leafHashV1(content: Buffer | Uint8Array): Buffer {
+  return sha256([LEAF_DOMAIN, asBuffer(content)]);
+}
+
+/**
+ * Parent node hashes.
+ * v0: SHA-256(H_left ‖ H_right) — order preserved, no pair sorting.
+ * v1: SHA-256(0x01 ‖ H_left ‖ H_right).
+ */
+export function parentHash(left: Buffer, right: Buffer, algorithm: MerkleAlgorithm): Buffer {
+  if (algorithm === 'v1') {
+    return sha256([NODE_DOMAIN, left, right]);
+  }
+  return sha256([left, right]);
+}
+
+/**
+ * Fold an ordered list of leaf hashes into a single root (hex lowercase).
+ * Odd-count rule:
+ *  - v0: duplicate last leaf (Bitcoin-style)
+ *  - v1: promote last node unhashed (RFC 6962–style)
+ */
+export function merkleRootFromLeaves(leaves: Buffer[], algorithm: MerkleAlgorithm): string | null {
+  if (leaves.length === 0) {
+    return null;
+  }
+
+  let level: Buffer[] = leaves.map((l) => Buffer.from(l));
+
+  while (level.length > 1) {
+    const next: Buffer[] = [];
+    for (let i = 0; i + 1 < level.length; i += 2) {
+      next.push(parentHash(level[i], level[i + 1], algorithm));
+    }
+    if (level.length % 2 === 1) {
+      const last = level[level.length - 1];
+      if (algorithm === 'v0') {
+        // Duplicate last leaf, then hash the pair
+        next.push(parentHash(last, last, algorithm));
+      } else {
+        // Promote unhashed
+        next.push(last);
+      }
+    }
+    level = next;
+  }
+
+  return level[0].toString('hex');
+}
+
+function resolveLeaf(
+  entry: MerkleContentEntry,
+  algorithm: MerkleAlgorithm
+): Buffer | null {
+  if (algorithm === 'v0') {
+    if (entry.contentSha256) {
+      const d = asBuffer(entry.contentSha256, typeof entry.contentSha256 === 'string' ? 'hex' : undefined);
+      if (d.length !== 32) {
+        return null;
+      }
+      return d;
+    }
+    if (entry.content) {
+      return contentDigest(entry.content);
+    }
+    return null;
+  }
+
+  // v1: prefer precomputed domain-separated leaf from the same hashing pass as 0x014E
+  if (entry.merkleLeafV1) {
+    const d = asBuffer(
+      entry.merkleLeafV1,
+      typeof entry.merkleLeafV1 === 'string' ? 'hex' : undefined
+    );
+    if (d.length !== 32) {
+      return null;
+    }
+    return d;
+  }
+  // Domain-separated hash of uncompressed payload (re-hash only if leaf not available)
+  if (entry.content) {
+    return leafHashV1(entry.content);
+  }
+  return null;
+}
+
+/**
+ * Compute archive Merkle root per APPNOTE §6 from content entries
+ * (caller already excluded META-INF/** if desired).
+ *
+ * @returns lowercase hex root, or null if no usable leaves
+ */
+export function computeArchiveMerkleRoot(
+  entries: MerkleContentEntry[],
+  algorithm: MerkleAlgorithm = 'v1'
+): string | null {
+  const prepared: { key: Buffer; leaf: Buffer }[] = [];
+
+  for (const entry of entries) {
+    if (isMetaInfPath(entry.path || '')) {
+      continue;
+    }
+    const leaf = resolveLeaf(entry, algorithm);
+    if (!leaf) {
+      continue;
+    }
+    prepared.push({
+      key: pathSortKey(entry.path || '', algorithm),
+      leaf,
+    });
+  }
+
+  if (prepared.length === 0) {
+    return null;
+  }
+
+  prepared.sort((a, b) => Buffer.compare(a.key, b.key));
+  return merkleRootFromLeaves(
+    prepared.map((p) => p.leaf),
+    algorithm
+  );
+}
+
+/**
+ * Build v0 root from path + bare content digests only (0x014E extras).
+ * Path order: central-directory bytes as stored (no NFC / strip).
+ */
+export function computeMerkleRootV0FromDigests(
+  entries: Array<{ path: string; sha256: string | Buffer }>
+): string | null {
+  return computeArchiveMerkleRoot(
+    entries.map((e) => ({ path: e.path, contentSha256: e.sha256 })),
+    'v0'
+  );
+}
+
+/**
+ * Build v1 root from path + uncompressed payloads (APPNOTE §6.3).
+ * Paths are normalized before sorting.
+ */
+export function computeMerkleRootV1FromContents(
+  entries: Array<{ path: string; content: Buffer | Uint8Array }>
+): string | null {
+  return computeArchiveMerkleRoot(
+    entries.map((e) => ({ path: e.path, content: e.content })),
+    'v1'
+  );
+}
+
+/**
+ * Build v1 root from path + precomputed domain-separated leaves (APPNOTE §6.3).
+ * Prefer this after create/extract when HashCalculator produced merkleLeafV1 in one stream pass.
+ */
+export function computeMerkleRootV1FromLeaves(
+  entries: Array<{ path: string; merkleLeafV1: string | Buffer }>
+): string | null {
+  return computeArchiveMerkleRoot(
+    entries.map((e) => ({ path: e.path, merkleLeafV1: e.merkleLeafV1 })),
+    'v1'
+  );
+}
+
+/**
+ * Verify a declared root (APPNOTE §6.4): try v1 first (when content or leaves available),
+ * then v0 digests. Returns which algorithm matched, or null on failure.
+ *
+ * When only bare 0x014E digests are available, only v0 can match APPNOTE correctly.
+ * Pass `merkleLeafV1` or content for full v1 verification without re-hashing from scratch
+ * when leaves were computed in the decompress stream.
+ */
+export function matchMerkleRoot(
+  entries: MerkleContentEntry[],
+  declaredRoot: string
+): { algorithm: MerkleAlgorithm; security: 'high' | 'legacy' } | null {
+  const expected = declaredRoot.toLowerCase().replace(/^0x/, '');
+  if (!/^[0-9a-f]{64}$/.test(expected)) {
+    return null;
+  }
+
+  const hasV1Material = entries.some(
+    (e) =>
+      !isMetaInfPath(e.path || '') &&
+      (!!e.merkleLeafV1 || !!e.content)
+  );
+  if (hasV1Material) {
+    const v1 = computeArchiveMerkleRoot(entries, 'v1');
+    if (v1 && v1 === expected) {
+      return { algorithm: 'v1', security: 'high' };
+    }
+  }
+
+  const v0 = computeArchiveMerkleRoot(entries, 'v0');
+  if (v0 && v0 === expected) {
+    return { algorithm: 'v0', security: 'legacy' };
+  }
+
+  // If v1 was not tried (no leaves/content), v0 was the only attempt.
+  if (hasV1Material) {
+    return null;
+  }
+
+  return null;
+}
+
+export default {
+  normalizeMerklePath,
+  contentDigest,
+  leafHashV0,
+  leafHashV1,
+  parentHash,
+  merkleRootFromLeaves,
+  computeArchiveMerkleRoot,
+  computeMerkleRootV0FromDigests,
+  computeMerkleRootV1FromContents,
+  computeMerkleRootV1FromLeaves,
+  matchMerkleRoot,
+};

package/src/core/version.ts

@@ -28,6 +28,6 @@ const currentDate = new Date();
 const releaseDate = `${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}-${currentDate.getFullYear()}`;
 
 export const VERSION = {
-  number: '0.7.0', // packageJson.version
+  number: '1.0.5', // packageJson.version
   date: releaseDate
 }; 

package/src/node/LegacyZstd.ts

@@ -0,0 +1,270 @@
+/**
+ * Detect legacy NeoZipKit WASM zstd (`@oneidentity/zstd-js`).
+ *
+ * That codec wrote valid-looking zstd frames but declared Frame_Content_Size as
+ * `uncompressed_size + 18`. Standard libzstd / Node zlib decode to the real
+ * payload plus 18 trailing zero bytes. CRC of the truncated payload matches
+ * the ZIP entry CRC.
+ *
+ * Header-only checks do not require decompressing. Verified checks decompress
+ * with native zlib and confirm trailing zeros + optional CRC.
+ */
+
+import { ZstdNode } from './ZstdNode';
+import { HashCalculator } from '../core/components/HashCalculator';
+
+/** Fixed extra bytes declared/emitted by the broken WASM codec. */
+export const LEGACY_WASM_ZSTD_PADDING = 18;
+
+const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
+
+export type LegacyZstdConfidence = 'none' | 'header' | 'verified';
+
+export interface LegacyZstdDetectResult {
+  /** True when this looks like (or verifies as) legacy WASM zstd. */
+  isLegacyWasm: boolean;
+  /** How the decision was made. */
+  confidence: LegacyZstdConfidence;
+  /** Parsed Frame_Content_Size, if present in the frame header. */
+  frameContentSize: number | null;
+  /** Expected plaintext size from the ZIP entry (or caller). */
+  expectedUncompressedSize: number;
+  /** Declared/observed padding past expected size (typically 18). */
+  paddingBytes: number;
+  /** Human-readable reason. */
+  reason: string;
+}
+
+export interface DetectLegacyZstdOptions {
+  /**
+   * When true (default), decompress with native zlib and confirm
+   * trailing zeros (+ optional CRC) instead of trusting the header alone.
+   */
+  verify?: boolean;
+  /** ZIP CRC-32 of the original uncompressed data (strengthens verified check). */
+  crc?: number;
+}
+
+/**
+ * Read Frame_Content_Size from a zstd frame header, or null if absent / unparseable.
+ * @see https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md
+ */
+export function parseZstdFrameContentSize(compressed: Buffer): number | null {
+  if (!compressed || compressed.length < 6) {
+    return null;
+  }
+  if (
+    compressed[0] !== ZSTD_MAGIC[0] ||
+    compressed[1] !== ZSTD_MAGIC[1] ||
+    compressed[2] !== ZSTD_MAGIC[2] ||
+    compressed[3] !== ZSTD_MAGIC[3]
+  ) {
+    return null;
+  }
+
+  const descriptor = compressed[4];
+  const fcsFlag = (descriptor >> 6) & 0x3;
+  const singleSegment = ((descriptor >> 5) & 0x1) === 1;
+  const dictIdFlag = descriptor & 0x3;
+  const dictIdSizes = [0, 1, 2, 4];
+  let pos = 5 + dictIdSizes[dictIdFlag];
+
+  if (fcsFlag === 0) {
+    if (!singleSegment) {
+      return null;
+    }
+    if (pos >= compressed.length) {
+      return null;
+    }
+    return compressed[pos];
+  }
+
+  const fcsSizes = [0, 2, 4, 8];
+  const n = fcsSizes[fcsFlag];
+  if (pos + n > compressed.length) {
+    return null;
+  }
+
+  let fcs: number;
+  if (n === 2) {
+    fcs = compressed.readUInt16LE(pos);
+  } else if (n === 4) {
+    fcs = compressed.readUInt32LE(pos);
+  } else {
+    const lo = compressed.readUInt32LE(pos);
+    const hi = compressed.readUInt32LE(pos + 4);
+    // Content sizes that do not fit in JS safe integers are not our legacy case.
+    if (hi !== 0) {
+      return null;
+    }
+    fcs = lo;
+  }
+
+  // Spec: when FCS_Flag == 1, stored value is (size - 256).
+  if (fcsFlag === 1) {
+    fcs += 256;
+  }
+  return fcs;
+}
+
+/**
+ * Fast header heuristic: FCS === expectedUncompressedSize + 18.
+ */
+export function looksLikeLegacyWasmZstd(
+  compressed: Buffer,
+  expectedUncompressedSize: number
+): boolean {
+  if (expectedUncompressedSize < 0 || !Number.isFinite(expectedUncompressedSize)) {
+    return false;
+  }
+  const fcs = parseZstdFrameContentSize(compressed);
+  if (fcs === null) {
+    return false;
+  }
+  return fcs === expectedUncompressedSize + LEGACY_WASM_ZSTD_PADDING;
+}
+
+function crc32Of(data: Buffer): number {
+  const h = new HashCalculator();
+  h.update(data);
+  return h.finalizeCRC32();
+}
+
+/**
+ * Detect legacy WASM zstd frames for a single compressed payload.
+ */
+export async function detectLegacyWasmZstd(
+  compressed: Buffer,
+  expectedUncompressedSize: number,
+  options?: DetectLegacyZstdOptions
+): Promise<LegacyZstdDetectResult> {
+  const expected = expectedUncompressedSize >>> 0;
+  const fcs = parseZstdFrameContentSize(compressed);
+  const headerMatch =
+    fcs !== null && fcs === expected + LEGACY_WASM_ZSTD_PADDING;
+
+  const base = {
+    frameContentSize: fcs,
+    expectedUncompressedSize: expected,
+    paddingBytes: LEGACY_WASM_ZSTD_PADDING,
+  };
+
+  if (!headerMatch && options?.verify === false) {
+    return {
+      ...base,
+      isLegacyWasm: false,
+      confidence: 'none',
+      reason:
+        fcs === null
+          ? 'No Frame_Content_Size in zstd header (or not a zstd frame)'
+          : `Frame_Content_Size ${fcs} does not equal expected ${expected} + ${LEGACY_WASM_ZSTD_PADDING}`,
+    };
+  }
+
+  if (options?.verify === false) {
+    return {
+      ...base,
+      isLegacyWasm: true,
+      confidence: 'header',
+      reason: `Frame_Content_Size is expected+${LEGACY_WASM_ZSTD_PADDING} (legacy WASM heuristic)`,
+    };
+  }
+
+  // Verified path (default): decompress and check padding / CRC.
+  let decoded: Buffer;
+  try {
+    decoded = await ZstdNode.decompress(compressed);
+  } catch (e) {
+    return {
+      ...base,
+      isLegacyWasm: false,
+      confidence: 'none',
+      reason: `Native decompress failed: ${e instanceof Error ? e.message : String(e)}`,
+    };
+  }
+
+  const pad = decoded.length - expected;
+  const trailingZeros =
+    pad === LEGACY_WASM_ZSTD_PADDING &&
+    decoded.subarray(expected).equals(Buffer.alloc(LEGACY_WASM_ZSTD_PADDING));
+  const truncated = decoded.subarray(0, Math.min(expected, decoded.length));
+  const crcOk =
+    options?.crc === undefined ||
+    options.crc === 0 ||
+    crc32Of(truncated) === (options.crc >>> 0);
+
+  if (trailingZeros && crcOk && decoded.length === expected + LEGACY_WASM_ZSTD_PADDING) {
+    return {
+      ...base,
+      isLegacyWasm: true,
+      confidence: 'verified',
+      reason:
+        options?.crc && options.crc !== 0
+          ? `Decoded length is +${LEGACY_WASM_ZSTD_PADDING} with trailing zeros; truncated CRC matches`
+          : `Decoded length is +${LEGACY_WASM_ZSTD_PADDING} with trailing zeros`,
+    };
+  }
+
+  if (headerMatch && !trailingZeros) {
+    return {
+      ...base,
+      isLegacyWasm: false,
+      confidence: 'none',
+      reason:
+        'Header suggested legacy WASM but decoded payload did not have 18 trailing zeros',
+    };
+  }
+
+  return {
+    ...base,
+    isLegacyWasm: false,
+    confidence: 'none',
+    paddingBytes: Math.max(0, pad),
+    reason:
+      decoded.length === expected
+        ? 'Standard zstd frame (decoded size matches expected)'
+        : `Decoded size ${decoded.length} does not match legacy pattern (expected ${expected} or ${expected + LEGACY_WASM_ZSTD_PADDING})`,
+  };
+}
+
+/**
+ * Truncate plaintext if it matches the legacy +18 zero padding pattern.
+ * Returns the original buffer when not legacy-shaped.
+ */
+export function repairLegacyWasmZstdPlaintext(
+  decoded: Buffer,
+  expectedUncompressedSize: number,
+  crc?: number
+): { repaired: Buffer; wasLegacy: boolean } {
+  const expected = expectedUncompressedSize >>> 0;
+  if (
+    decoded.length === expected + LEGACY_WASM_ZSTD_PADDING &&
+    decoded.subarray(expected).equals(Buffer.alloc(LEGACY_WASM_ZSTD_PADDING))
+  ) {
+    const repaired = decoded.subarray(0, expected);
+    if (crc !== undefined && crc !== 0) {
+      if (crc32Of(repaired) !== (crc >>> 0)) {
+        return { repaired: decoded, wasLegacy: false };
+      }
+    }
+    return { repaired, wasLegacy: true };
+  }
+  if (decoded.length > expected && expected > 0) {
+    // Match Rust extract behavior: bound to declared uncompressed size for zstd.
+    const repaired = decoded.subarray(0, expected);
+    if (crc !== undefined && crc !== 0 && crc32Of(repaired) === (crc >>> 0)) {
+      return { repaired, wasLegacy: true };
+    }
+  }
+  return { repaired: decoded, wasLegacy: false };
+}
+
+export const LegacyZstd = {
+  LEGACY_WASM_ZSTD_PADDING,
+  parseZstdFrameContentSize,
+  looksLikeLegacyWasmZstd,
+  detectLegacyWasmZstd,
+  repairLegacyWasmZstdPlaintext,
+};
+
+export default LegacyZstd;