neozipkit 1.0.5 → 1.5.0

package/src/node/ZipkitNode.ts

@@ -11,19 +11,31 @@ import { ZipDecompressNode } from './ZipDecompressNode';
 import { 
   CENTRAL_END, 
   CENTRAL_DIR, 
-  ZIP64_CENTRAL_END, 
-  ZIP64_CENTRAL_DIR,
   LOCAL_HDR,
   GP_FLAG,
   CMP_METHOD,
-  ENCRYPT_HDR_SIZE
+  ENCRYPT_HDR_SIZE,
+  HDR_ID,
 } from '../core/constants/Headers';
+import {
+  buildEndRecords,
+  classicEocdNeedsZip64,
+  exceedsU32,
+  parseZip64Eocd,
+  parseZip64Locator,
+  writeU64,
+  ZIP64_EOCD_RECORD_SIZE,
+  ZIP64_LOCATOR_SIZE,
+  ZIP64_U32,
+} from '../core/zip64/Zip64';
 import { NEO_CRYPTO_ALGORITHM_AES256_V1 } from '../core/encryption/NeoCrypto';
 import { AesCrypto } from '../core/encryption/AesCrypto';
 import {
   detectLegacyWasmZstd,
   type LegacyZstdDetectResult,
 } from './LegacyZstd';
+import { writeZipFileSync, type ZipBufferMember, type BuildZipBufferOptions } from './buildZipBuffer';
+export type { ZipBufferMember };
 import * as fs from 'fs';
 import * as path from 'path';
 import { minimatch } from 'minimatch';
@@ -717,6 +729,10 @@ export default class ZipkitNode extends Zipkit {
       onHashCalculated?: (entry: ZipEntry, hash: Buffer) => void;
     }
   ): Promise<void> {
+    if (options?.forceZip64) {
+      entry.forceZip64 = true;
+    }
+
     // Set compression method based on options
     const level = options?.level ?? 6;
     let realCmpMethod: number;
@@ -859,9 +875,55 @@ export default class ZipkitNode extends Zipkit {
       throw new Error(`Compressed size not set for entry: ${entry.filename}`);
     }
 
-    const sizeBuffer = Buffer.alloc(4);
-    sizeBuffer.writeUInt32LE(entry.compressedSize, 0);
-    fs.writeSync(writer.outputFd, sizeBuffer, 0, 4, entry.localHdrOffset + 18);
+    const classicCmp = entry.localHdrOffset + LOCAL_HDR.CMP_SIZE;
+    const classicUncmp = entry.localHdrOffset + LOCAL_HDR.UNCMP_SIZE;
+    const useZip64Sizes =
+      entry.forceZip64 ||
+      exceedsU32(entry.uncompressedSize) ||
+      exceedsU32(entry.compressedSize);
+
+    if (useZip64Sizes) {
+      // Classic fields stay at 0xFFFFFFFF; update u64 sizes inside local extra 0x0001
+      const sentinel = Buffer.alloc(4);
+      sentinel.writeUInt32LE(ZIP64_U32, 0);
+      fs.writeSync(writer.outputFd, sentinel, 0, 4, classicCmp);
+      fs.writeSync(writer.outputFd, sentinel, 0, 4, classicUncmp);
+
+      const fnameLenBuf = Buffer.alloc(2);
+      fs.readSync(writer.outputFd, fnameLenBuf, 0, 2, entry.localHdrOffset + LOCAL_HDR.FNAME_LEN);
+      const fnameLen = fnameLenBuf.readUInt16LE(0);
+      const extraLenBuf = Buffer.alloc(2);
+      fs.readSync(writer.outputFd, extraLenBuf, 0, 2, entry.localHdrOffset + LOCAL_HDR.EXTRA_LEN);
+      const extraLen = extraLenBuf.readUInt16LE(0);
+      const extraStart = entry.localHdrOffset + LOCAL_HDR.SIZE + fnameLen;
+      const extraBuf = Buffer.alloc(extraLen);
+      if (extraLen > 0) {
+        fs.readSync(writer.outputFd, extraBuf, 0, extraLen, extraStart);
+      }
+      let patched = false;
+      for (let i = 0; i + 4 <= extraBuf.length; ) {
+        const id = extraBuf.readUInt16LE(i);
+        const len = extraBuf.readUInt16LE(i + 2);
+        if (id === HDR_ID.ZIP64 && len >= 16) {
+          // Local Zip64 always has uncompressed (8) then compressed (8)
+          writeU64(extraBuf, i + 4, entry.uncompressedSize);
+          writeU64(extraBuf, i + 12, entry.compressedSize);
+          fs.writeSync(writer.outputFd, extraBuf, i + 4, 16, extraStart + i + 4);
+          patched = true;
+          break;
+        }
+        i += 4 + len;
+      }
+      if (!patched) {
+        throw new Error(
+          `Zip64 local extra (0x0001) missing for large entry: ${entry.filename}`
+        );
+      }
+    } else {
+      const sizeBuffer = Buffer.alloc(4);
+      sizeBuffer.writeUInt32LE(entry.compressedSize, 0);
+      fs.writeSync(writer.outputFd, sizeBuffer, 0, 4, classicCmp);
+    }
 
     if (entry.crc !== undefined) {
       const crcBuffer = Buffer.alloc(4);
@@ -938,6 +1000,7 @@ export default class ZipkitNode extends Zipkit {
    * @param centralDirSize - Size of central directory in bytes
    * @param centralDirOffset - Offset to start of central directory
    * @param archiveComment - Optional archive comment (max 65535 bytes)
+   * @param options - Optional Zip64 force and related EOCD flags
    * @returns Promise that resolves when EOCD is written
    */
   async writeEndOfCentralDirectory(
@@ -945,53 +1008,20 @@ export default class ZipkitNode extends Zipkit {
     totalEntries: number,
     centralDirSize: number,
     centralDirOffset: number,
-    archiveComment?: string
+    archiveComment?: string,
+    options?: { forceZip64?: boolean }
   ): Promise<void> {
-    const comment = archiveComment || '';
-    const commentBytes = Buffer.from(comment, 'utf8');
-    const commentLength = Math.min(commentBytes.length, 0xFFFF); // Max 65535 bytes
-
-    const buffer = Buffer.alloc(22 + commentLength);
-    let offset = 0;
-
-    // End of central directory signature (4 bytes)
-    buffer.writeUInt32LE(0x06054b50, offset);
-    offset += 4;
-
-    // Number of this disk (2 bytes)
-    buffer.writeUInt16LE(0, offset);
-    offset += 2;
-
-    // Number of the disk with the start of the central directory (2 bytes)
-    buffer.writeUInt16LE(0, offset);
-    offset += 2;
-
-    // Total number of entries in the central directory on this disk (2 bytes)
-    buffer.writeUInt16LE(totalEntries, offset);
-    offset += 2;
-
-    // Total number of entries in the central directory (2 bytes)
-    buffer.writeUInt16LE(totalEntries, offset);
-    offset += 2;
-
-    // Size of the central directory (4 bytes)
-    buffer.writeUInt32LE(centralDirSize, offset);
-    offset += 4;
-
-    // Offset of start of central directory with respect to the starting disk number (4 bytes)
-    buffer.writeUInt32LE(centralDirOffset, offset);
-    offset += 4;
-
-    // ZIP file comment length (2 bytes)
-    buffer.writeUInt16LE(commentLength, offset);
-    offset += 2;
-
-    // ZIP file comment (variable length)
-    if (commentLength > 0) {
-      commentBytes.copy(buffer, offset, 0, commentLength);
-    }
+    const buffer = buildEndRecords({
+      totalEntries,
+      centralDirSize,
+      centralDirOffset,
+      zip64EocdOffset: centralDirOffset + centralDirSize,
+      archiveComment,
+      allowSizeOffsetZip64: true,
+      forceZip64: options?.forceZip64 === true,
+    });
 
-    // Write EOCD to file
+    // Write EOCD (and Zip64 EOCD + locator when needed) to file
     await new Promise<void>((resolve, reject) => {
       writer.outputStream.write(buffer, (error) => {
         if (error) {
@@ -1067,7 +1097,9 @@ export default class ZipkitNode extends Zipkit {
         writer,
         entries.length,
         centralDirSize,
-        centralDirOffset
+        centralDirOffset,
+        undefined,
+        { forceZip64: options?.forceZip64 === true }
       );
     } finally {
       await this.finalizeZipFile(writer);
@@ -1210,6 +1242,21 @@ export default class ZipkitNode extends Zipkit {
     throw new Error('updateZipFile() - Full implementation pending. Use neozip CLI for now.');
   }
 
+  /**
+   * Stream members to `filePath` (local headers, payloads, central directory, EOCD).
+   * Pass `useSHA256` for Extra Field 0x014E and `additionalExtra` for caller blocks
+   * such as ZipWiki 0x014F. `method` 93 uses zlib zstd at `level` (default 7).
+   * NeoEncrypt is not enabled. Precompressed members are copied without recompression.
+   * Pass `{ forceZip64: true }` to emit Zip64 even for small members (testing).
+   */
+  writeMembersSync(
+    filePath: string,
+    members: ZipBufferMember[],
+    options?: BuildZipBufferOptions
+  ): void {
+    writeZipFileSync(filePath, members, options);
+  }
+
   // ============================================================================
   // File-based ZIP Loading Methods (merged from ZipLoadEntriesServer)
   // ============================================================================
@@ -1275,9 +1322,17 @@ export default class ZipkitNode extends Zipkit {
         const zipkit = this as any;
         zipkit.centralDirSize = eocdBuffer.readUInt32LE(CENTRAL_END.CENTRAL_DIR_SIZE);
         zipkit.centralDirOffset = eocdBuffer.readUInt32LE(CENTRAL_END.CENTRAL_DIR_OFFSET);
-        
-        // Handle ZIP64
-        if (zipkit.centralDirOffset === 0xFFFFFFFF) {
+        const volEntries = eocdBuffer.readUInt16LE(CENTRAL_END.VOL_ENTRIES);
+        const totalEntries = eocdBuffer.readUInt16LE(CENTRAL_END.TOTAL_ENTRIES);
+
+        if (
+          classicEocdNeedsZip64({
+            volEntries,
+            totalEntries,
+            centralDirSize: zipkit.centralDirSize,
+            centralDirOffset: zipkit.centralDirOffset,
+          })
+        ) {
           await this.loadZIP64EOCD(eocdOffset);
         }
       } else {
@@ -1307,20 +1362,18 @@ export default class ZipkitNode extends Zipkit {
     }
     
     // Look for ZIP64 locator
-    const locatorOffset = eocdOffset - 20;
-    const locatorBuffer = Buffer.alloc(20);
-    await this.fileHandle.read(locatorBuffer, 0, 20, locatorOffset);
+    const locatorOffset = eocdOffset - ZIP64_LOCATOR_SIZE;
+    const locatorBuffer = Buffer.alloc(ZIP64_LOCATOR_SIZE);
+    await this.fileHandle.read(locatorBuffer, 0, ZIP64_LOCATOR_SIZE, locatorOffset);
     
-    if (locatorBuffer.readUInt32LE(0) === ZIP64_CENTRAL_END.SIGNATURE) {
-      // Read ZIP64 EOCD
-      const zip64Offset = locatorBuffer.readBigUInt64LE(8);
-      const zip64Buffer = Buffer.alloc(56);
-      await this.fileHandle.read(zip64Buffer, 0, 56, Number(zip64Offset));
-      
-      const zipkit = this as any;
-      zipkit.centralDirSize = Number(zip64Buffer.readBigUInt64LE(ZIP64_CENTRAL_DIR.CENTRAL_DIR_SIZE));
-      zipkit.centralDirOffset = Number(zip64Buffer.readBigUInt64LE(ZIP64_CENTRAL_DIR.CENTRAL_DIR_OFFSET));
-    }
+    const zip64Offset = parseZip64Locator(locatorBuffer);
+    const zip64Buffer = Buffer.alloc(ZIP64_EOCD_RECORD_SIZE);
+    await this.fileHandle.read(zip64Buffer, 0, ZIP64_EOCD_RECORD_SIZE, zip64Offset);
+    
+    const z64 = parseZip64Eocd(zip64Buffer);
+    const zipkit = this as any;
+    zipkit.centralDirSize = z64.centralDirSize;
+    zipkit.centralDirOffset = z64.centralDirOffset;
   }
 
   /**

package/src/node/ZstdNode.ts

@@ -54,26 +54,49 @@ export function createDecompressTransform(): zlib.ZstdDecompress {
   return zlib.createZstdDecompress();
 }
 
-export async function compress(data: Buffer, level: number = 6): Promise<Buffer> {
+/** Bytes accepted by the buffer zstd codec (Node Buffer or browser ArrayBuffer). */
+export type ZstdInput = Buffer | Uint8Array | ArrayBuffer;
+
+export function asZstdBuffer(data: ZstdInput): Buffer {
+  if (Buffer.isBuffer(data)) return data;
+  if (data instanceof ArrayBuffer) return Buffer.from(data);
+  return Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+}
+
+export async function compress(data: ZstdInput, level: number = 6): Promise<Buffer> {
   assertNativeZstd();
-  const result = await zstdCompressAsync(data, compressOptions(level));
+  const result = await zstdCompressAsync(asZstdBuffer(data), compressOptions(level));
   return Buffer.from(result);
 }
 
-export async function decompress(data: Buffer): Promise<Buffer> {
+export async function decompress(data: ZstdInput): Promise<Buffer> {
   assertNativeZstd();
-  const result = await zstdDecompressAsync(data);
+  const result = await zstdDecompressAsync(asZstdBuffer(data));
   return Buffer.from(result);
 }
 
-export function compressSync(data: Buffer, level: number = 6): Buffer {
+export function compressSync(data: ZstdInput, level: number = 6): Buffer {
+  assertNativeZstd();
+  return zlib.zstdCompressSync(asZstdBuffer(data), compressOptions(level));
+}
+
+/**
+ * Compress with a raw zlib zstd level (1–19), without the 1–9 kit scale.
+ * ZipWiki pack uses level 7 on this scale.
+ */
+export function compressSyncAtLevel(data: ZstdInput, zstdLevel: number): Buffer {
   assertNativeZstd();
-  return zlib.zstdCompressSync(data, compressOptions(level));
+  const level = Math.min(19, Math.max(1, Math.trunc(zstdLevel)));
+  return zlib.zstdCompressSync(asZstdBuffer(data), {
+    params: {
+      [zlib.constants.ZSTD_c_compressionLevel]: level,
+    },
+  });
 }
 
-export function decompressSync(data: Buffer): Buffer {
+export function decompressSync(data: ZstdInput): Buffer {
   assertNativeZstd();
-  return zlib.zstdDecompressSync(data);
+  return zlib.zstdDecompressSync(asZstdBuffer(data));
 }
 
 /**
@@ -147,7 +170,9 @@ export const ZstdNode = {
   compress,
   decompress,
   compressSync,
+  compressSyncAtLevel,
   decompressSync,
+  asZstdBuffer,
   compressChunks,
   decompressStream,
 };

package/src/node/buildZipBuffer.ts

@@ -0,0 +1,221 @@
+/**
+ * In-memory and streaming ZIP writer.
+ * Zstd level is the raw zlib level (ZipWiki default 7). NeoEncrypt stays off.
+ * Caller extra blocks (ZipWiki 0x014F) are written beside optional 0x014E.
+ */
+
+import { closeSync, openSync, writeSync } from 'fs';
+import { deflateRawSync } from 'zlib';
+import ZipEntry from '../core/ZipEntry';
+import { crc32, sha256 } from '../core/encryption/ZipCrypto';
+import { compressSyncAtLevel } from './ZstdNode';
+import {
+  assertBufferAllowsEntryZip64,
+  buildEndRecords,
+} from '../core/zip64/Zip64';
+
+export type ZipPrecompressedMember = {
+  method: number;
+  data: Buffer;
+  crc32: number;
+  uncompressedSize: number;
+  /** Complete extra-field bytes copied onto the local and central headers. */
+  extra?: Buffer;
+};
+
+export type ZipBufferMember = {
+  name: string;
+  /** Uncompressed payload. Unused as stored bytes when `precompressed` is set. */
+  data: Buffer;
+  mtime?: Date;
+  /** DOS time word. Wins over `mtime` when set together with `dosDate`. */
+  dosTime?: number;
+  dosDate?: number;
+  /**
+   * Complete extra-field records (id, size, payload), e.g. ZipWiki 0x014F.
+   * Ignored when `precompressed.extra` is set — that blob is copied as-is.
+   */
+  additionalExtra?: Buffer;
+  /** Write Extra Field 0x014E (SHA-256 of uncompressed `data`). */
+  useSHA256?: boolean;
+  /** 0 store, 8 deflate, 93 zstd. Default 93. Ignored when `precompressed` is set. */
+  method?: 0 | 8 | 93;
+  /** zlib zstd level or raw deflate level. Default 7. */
+  level?: number;
+  /**
+   * Force Zip64 Extra `0x0001` on this member even when sizes fit classic fields.
+   * Pair with `forceZip64` on `buildZipBufferSync` / `writeZipFileSync` options
+   * (or set on every member) so the EOCD is Zip64 as well.
+   */
+  forceZip64?: boolean;
+  precompressed?: ZipPrecompressedMember;
+};
+
+type FramedMember = {
+  local: Buffer;
+  payload: Buffer;
+  central: Buffer;
+};
+
+function dosFromDate(date: Date): { time: number; date: number } {
+  const year = date.getFullYear();
+  const month = date.getMonth() + 1;
+  const day = date.getDate();
+  const hours = date.getHours();
+  const minutes = date.getMinutes();
+  const seconds = Math.floor(date.getSeconds() / 2);
+  const y = Math.max(0, Math.min(127, year - 1980));
+  return {
+    time: ((hours & 0x1f) << 11) | ((minutes & 0x3f) << 5) | (seconds & 0x1f),
+    date: ((y & 0x7f) << 9) | ((month & 0x0f) << 5) | (day & 0x1f),
+  };
+}
+
+function compressPayload(
+  data: Buffer,
+  method: 0 | 8 | 93,
+  level: number,
+): { method: number; data: Buffer } {
+  if (method === 0 || data.length === 0) {
+    return { method: 0, data };
+  }
+  const compressed =
+    method === 8
+      ? deflateRawSync(data, { level: Math.max(1, Math.min(9, level)) })
+      : compressSyncAtLevel(data, Math.max(1, level));
+  if (compressed.length >= data.length) {
+    return { method: 0, data };
+  }
+  return { method, data: compressed };
+}
+
+function frameMembers(
+  members: ZipBufferMember[],
+  opts?: { forceZip64?: boolean }
+): FramedMember[] {
+  const packTime = new Date();
+  const framed: FramedMember[] = [];
+  let offset = 0;
+  const forceAll = opts?.forceZip64 === true;
+  for (const member of members) {
+    const copied = member.precompressed;
+    const uncompressed = member.data ?? Buffer.alloc(0);
+    const level = member.level ?? 7;
+    const requested = member.method ?? 93;
+    const payload = copied
+      ? {
+          method: copied.method,
+          data: copied.data,
+          uncompressedSize: copied.uncompressedSize,
+        }
+      : {
+          ...compressPayload(uncompressed, requested, level),
+          uncompressedSize: uncompressed.length,
+        };
+    const crc = copied ? copied.crc32 >>> 0 : crc32(uncompressed) >>> 0;
+    const extra = copied?.extra ?? member.additionalExtra;
+    const dos =
+      member.dosTime !== undefined && member.dosDate !== undefined
+        ? { time: member.dosTime, date: member.dosDate }
+        : dosFromDate(member.mtime ?? packTime);
+
+    const entry = new ZipEntry(member.name, null, false);
+    entry.bitFlags = 0;
+    entry.emitUnicodePath = false;
+    entry.isUpdated = true;
+    entry.filename = member.name;
+    entry.cmpMethod = payload.method;
+    entry.crc = crc;
+    entry.compressedSize = payload.data.length;
+    entry.uncompressedSize = payload.uncompressedSize;
+    entry.timeDateDOS = ((dos.date & 0xffff) << 16) | (dos.time & 0xffff);
+    entry.localHdrOffset = offset;
+    entry.forceZip64 = forceAll || member.forceZip64 === true;
+    assertBufferAllowsEntryZip64(
+      entry.uncompressedSize,
+      entry.compressedSize,
+      entry.localHdrOffset
+    );
+    if (!copied && member.useSHA256 === true) {
+      entry.sha256 = sha256(uncompressed);
+    }
+    if (extra && extra.length > 0) {
+      entry.additionalExtra = extra;
+    }
+    const local = entry.createLocalHdr();
+    const central = entry.centralDirEntry();
+    framed.push({ local, payload: payload.data, central });
+    offset += local.length + payload.data.length;
+  }
+  return framed;
+}
+
+function eocd(
+  entryCount: number,
+  centralSize: number,
+  centralOffset: number,
+  forceZip64?: boolean
+): Buffer {
+  return buildEndRecords({
+    totalEntries: entryCount,
+    centralDirSize: centralSize,
+    centralDirOffset: centralOffset,
+    zip64EocdOffset: centralOffset + centralSize,
+    allowSizeOffsetZip64: false,
+    forceZip64: forceZip64 === true,
+  });
+}
+
+export type BuildZipBufferOptions = {
+  /** Force Zip64 EOCD + per-member Extra when sizes fit classic fields. */
+  forceZip64?: boolean;
+};
+
+/** Build a ZIP in one buffer. Kept for tests and atomic in-memory rewrites. */
+export function buildZipBufferSync(
+  members: ZipBufferMember[],
+  options?: BuildZipBufferOptions
+): Buffer {
+  const forceZip64 = options?.forceZip64 === true;
+  const framed = frameMembers(members, { forceZip64 });
+  const parts: Buffer[] = [];
+  let centralSize = 0;
+  let payloadBytes = 0;
+  for (const part of framed) {
+    parts.push(part.local, part.payload);
+    payloadBytes += part.local.length + part.payload.length;
+  }
+  for (const part of framed) {
+    parts.push(part.central);
+    centralSize += part.central.length;
+  }
+  parts.push(eocd(framed.length, centralSize, payloadBytes, forceZip64));
+  return Buffer.concat(parts);
+}
+
+/** Stream local headers, payloads, the central directory, and the EOCD to `filePath`. */
+export function writeZipFileSync(
+  filePath: string,
+  members: ZipBufferMember[],
+  options?: BuildZipBufferOptions
+): void {
+  const forceZip64 = options?.forceZip64 === true;
+  const framed = frameMembers(members, { forceZip64 });
+  const fd = openSync(filePath, 'w');
+  try {
+    let payloadBytes = 0;
+    let centralSize = 0;
+    for (const part of framed) {
+      writeSync(fd, part.local);
+      writeSync(fd, part.payload);
+      payloadBytes += part.local.length + part.payload.length;
+    }
+    for (const part of framed) {
+      writeSync(fd, part.central);
+      centralSize += part.central.length;
+    }
+    writeSync(fd, eocd(framed.length, centralSize, payloadBytes, forceZip64));
+  } finally {
+    closeSync(fd);
+  }
+}

package/src/node/index.ts

@@ -34,6 +34,13 @@ export type { LegacyZstdDetectResult, LegacyZstdConfidence, DetectLegacyZstdOpti
 
 // Efficient ZIP copying using ZipEntry directly
 export { ZipCopyNode } from './ZipCopyNode';
+export { buildZipBufferSync, writeZipFileSync } from './buildZipBuffer';
+export type {
+  ZipBufferMember,
+  ZipPrecompressedMember,
+  BuildZipBufferOptions,
+} from './buildZipBuffer';
+export { inflateZipPayloadSync } from './inflateZipPayload';
 export type {
   CopyOptions,
   CopyResult,

package/src/node/inflateZipPayload.ts

@@ -0,0 +1,19 @@
+/**
+ * Synchronous inflate for Node readers (store, raw deflate, native zstd).
+ */
+
+import { inflateRawSync } from 'zlib';
+import { decompressSync } from './ZstdNode';
+
+export function inflateZipPayloadSync(method: number, compressed: Buffer): Buffer {
+  if (method === 0) {
+    return Buffer.from(compressed);
+  }
+  if (method === 8) {
+    return inflateRawSync(compressed);
+  }
+  if (method === 93) {
+    return decompressSync(compressed);
+  }
+  throw new Error(`unsupported compression method ${method}`);
+}

package/src/types/index.ts

@@ -8,8 +8,8 @@ export interface NeoZipKitInfo {
 }
 
 export const NEOZIPKIT_INFO: NeoZipKitInfo = {
-  version: '0.7.0',
-  releaseDate: '2026-04-06'
+  version: '1.5.0',
+  releaseDate: '2026-10-25'
 };
 
 // ============================================================================