@ontos-ai/knowhere-sdk 2.4.2 → 2.5.0

package/dist/sequential-parsed-document-archive-S56UEARH.mjs

@@ -0,0 +1,139 @@
+import {
+  ParsedDocumentArchiveError,
+  createDownloadIdentity,
+  isSnapshotPath,
+  normalizeResultPath
+} from "./chunk-4PWLXJGH.mjs";
+
+// src/storage/sequential-parsed-document-archive.ts
+import { Readable, Transform } from "stream";
+import { pipeline } from "stream/promises";
+import { mkdtemp, open, rm } from "fs/promises";
+import { tmpdir } from "os";
+import { join } from "path";
+import { crc32 } from "zlib";
+import { ZipReader, Reader } from "@zip.js/zip.js";
+import Parse from "unzipper/lib/parse.js";
+async function openSequentialArchive(params, identity) {
+  const response = await fetch(params.resultUrl, { signal: params.signal });
+  if (!response.ok || !response.body)
+    throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
+  if (createDownloadIdentity(params.jobId, response) !== identity) {
+    await response.body.cancel();
+    throw new Error("Parsed result archive identity changed before sequential download.");
+  }
+  let directory;
+  try {
+    directory = await mkdtemp(join(tmpdir(), "knowhere-sync-"));
+  } catch (error) {
+    await response.body.cancel();
+    throw new ParsedDocumentArchiveError(error);
+  }
+  let file;
+  try {
+    file = await open(join(directory, "archive.zip"), "w+");
+  } catch (error) {
+    await response.body.cancel();
+    await rm(directory, { recursive: true, force: true });
+    throw new ParsedDocumentArchiveError(error);
+  }
+  const input = Readable.fromWeb(response.body);
+  const parser = Parse({ forceStream: true });
+  const checksums = /* @__PURE__ */ new Map();
+  let archiveSize = 0;
+  const spool = new Transform({
+    transform(chunk, _encoding, callback) {
+      archiveSize += chunk.length;
+      void file.writeFile(chunk).then(
+        () => {
+          callback(null, chunk);
+        },
+        (error) => {
+          callback(new ParsedDocumentArchiveError(error));
+        }
+      );
+    }
+  });
+  const completed = pipeline(input, spool, parser, { signal: params.signal });
+  void completed.catch(() => void 0);
+  return {
+    identity,
+    downloadIdentity: identity,
+    canSkipEntries: false,
+    entries: (async function* () {
+      for await (const value of parser) {
+        const entry = value;
+        if (entry.type === "Directory" || !isSnapshotPath(entry.path)) {
+          await entry.autodrain().promise();
+          continue;
+        }
+        const path = normalizeResultPath(entry.path);
+        yield {
+          path,
+          read: () => {
+            let checksum = 0;
+            let size = 0;
+            return Readable.toWeb(entry).pipeThrough(
+              new TransformStream({
+                transform(chunk, controller) {
+                  checksum = crc32(chunk, checksum);
+                  size += chunk.length;
+                  controller.enqueue(chunk);
+                },
+                flush() {
+                  checksums.set(path, { checksum, size });
+                }
+              })
+            );
+          }
+        };
+      }
+      await completed;
+      const reader = new ZipReader(new DiskArchiveReader(file, archiveSize), {
+        useWebWorkers: false
+      });
+      try {
+        const files = (await reader.getEntries()).filter(
+          (entry) => !entry.directory && isSnapshotPath(entry.filename)
+        );
+        if (files.length !== checksums.size)
+          throw new Error("Parsed result ZIP directory does not match sequential entries.");
+        for (const entry of files) {
+          const actual = checksums.get(normalizeResultPath(entry.filename));
+          if (!actual || actual.checksum !== entry.signature || actual.size !== entry.uncompressedSize)
+            throw new Error(`Invalid parsed result ZIP checksum or size: ${entry.filename}`);
+        }
+      } finally {
+        await reader.close();
+      }
+    })(),
+    close: async () => {
+      input.destroy();
+      spool.destroy();
+      parser.destroy();
+      await completed.catch(() => void 0);
+      try {
+        await file.close();
+      } finally {
+        await rm(directory, { recursive: true, force: true });
+      }
+    }
+  };
+}
+var DiskArchiveReader = class extends Reader {
+  constructor(file, size) {
+    super(file);
+    this.file = file;
+    this.size = size;
+  }
+  file;
+  async readUint8Array(index, length) {
+    const body = new Uint8Array(length);
+    const { bytesRead } = await this.file.read(body, 0, length, index);
+    if (bytesRead !== length) throw new Error("Truncated parsed result ZIP.");
+    return body;
+  }
+};
+export {
+  openSequentialArchive
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ontos-ai/knowhere-sdk",
-  "version": "2.4.2",
+  "version": "2.5.0",
   "description": "Official Node.js SDK for Knowhere document parsing API",
   "keywords": [
     "knowhere",
@@ -43,14 +43,17 @@
     "pnpm": ">=9.0.0"
   },
   "dependencies": {
+    "@zip.js/zip.js": "^2.24.0",
     "axios": "^1.15.2",
-    "jszip": "^3.10.0"
+    "jszip": "^3.10.0",
+    "unzipper": "^0.12.5"
   },
   "devDependencies": {
     "@changesets/changelog-github": "^0.6.0",
     "@changesets/cli": "^2.30.0",
     "@eslint/js": "^10.0.1",
     "@types/node": "^25.2.3",
+    "@types/unzipper": "^0.10.11",
     "@typescript-eslint/eslint-plugin": "^8.55.0",
     "@typescript-eslint/parser": "^8.55.0",
     "@vitest/coverage-v8": "^4.0.18",