@ontos-ai/knowhere-sdk 2.6.1-beta.20261027150736 → 2.7.0

package/dist/index.mjs

@@ -47,7 +47,7 @@ import {
   syncParseResultToParsedDocumentStorage,
   syncRemoteResultArchive,
   validateArchiveSyncLimits
-} from "./chunk-DIONXJWK.mjs";
+} from "./chunk-OCR7IDYL.mjs";
 
 // src/client.ts
 import path2 from "path";
@@ -1068,6 +1068,12 @@ var Jobs = class extends BaseResource {
     enrichJobResult(jobResult);
     return jobResult;
   }
+  /** Renew unfinished enhancement without creating another ingestion. */
+  async retryEnhancement(jobId) {
+    return this.httpClient.post(
+      this.endpoint(`/jobs/${jobId}/enrichment/retry`)
+    );
+  }
   /**
    * Upload file for a job
    */
@@ -1764,6 +1770,21 @@ function isMissingFileError2(error) {
   return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
 }
 
+// src/knowledge/serialize-snapshot-publication.ts
+var pendingPublications = /* @__PURE__ */ new WeakMap();
+async function serializeSnapshotPublication(storage, documentId, publish) {
+  const publications = pendingPublications.get(storage) ?? /* @__PURE__ */ new Map();
+  pendingPublications.set(storage, publications);
+  const previous = publications.get(documentId) ?? Promise.resolve();
+  const current = previous.catch(() => void 0).then(publish);
+  publications.set(documentId, current);
+  try {
+    return await current;
+  } finally {
+    if (publications.get(documentId) === current) publications.delete(documentId);
+  }
+}
+
 // src/knowledge/knowledge.ts
 var DEFAULT_READ_LIMIT = 12;
 var MAX_READ_LIMIT = 40;
@@ -1881,11 +1902,37 @@ var Knowledge = class _Knowledge {
     return this.importJobResult(params);
   }
   async syncParsedDocument(params) {
+    const response = await this.performParsedDocumentSync(params);
+    if (response.completed && !response.snapshot) {
+      const storage = this.requireParsedStorageConfig().storage;
+      await serializeSnapshotPublication(storage, response.documentId, async () => {
+        if (params.documentId || params.jobId) {
+          const current = await this.client.documents.listChunks(
+            response.documentId,
+            { page: 1, pageSize: 1 }
+          );
+          if (params.jobId && current.jobId !== params.jobId) return;
+          if (!params.jobId && current.jobResultId !== response.revisionKey) return;
+        }
+        await storage.writeObject({
+          documentId: response.documentId,
+          revisionKey: "current",
+          path: ".knowhere-sdk/current-snapshot.json",
+          body: Buffer.from(
+            JSON.stringify({ snapshotId: response.revisionKey, jobResultId: response.revisionKey })
+          ),
+          contentType: "application/json; charset=utf-8"
+        });
+      });
+    }
+    return response;
+  }
+  async performParsedDocumentSync(params) {
     params.signal?.throwIfAborted();
     const normalized = normalizeDocumentReference(params);
     const parsedStorageConfig = this.requireParsedStorageConfig();
     validateArchiveSyncLimits(parsedStorageConfig.limits);
-    if (normalized.documentId) {
+    if (normalized.documentId && !normalized.jobId) {
       return this.syncRemoteDocumentToParsedStorage({
         documentId: normalized.documentId,
         revisionKey: params.revisionKey,
@@ -1897,6 +1944,25 @@ var Knowledge = class _Knowledge {
     }
     if (normalized.jobId) {
       const job = await this.client.jobs.get(normalized.jobId);
+      if (job.snapshots?.length) {
+        return this.syncJobSnapshot({
+          job,
+          documentId: normalized.documentId ?? job.documentId ?? normalized.localDocumentId ?? normalized.jobId,
+          revisionKey: params.revisionKey,
+          parsedStorageConfig,
+          signal: params.signal,
+          retainResultArchive: params.retainResultArchive
+        });
+      }
+      if (normalized.documentId) {
+        return this.syncRemoteDocumentToParsedStorage({
+          documentId: normalized.documentId,
+          revisionKey: params.revisionKey,
+          parsedStorageConfig,
+          signal: params.signal,
+          retainResultArchive: params.retainResultArchive
+        });
+      }
       if (!job.isDone || !job.resultUrl)
         throw new ValidationError("Completed job result archive is unavailable");
       return syncRemoteResultArchive({
@@ -1928,6 +1994,89 @@ var Knowledge = class _Knowledge {
       commit: synced.commit
     };
   }
+  async syncJobSnapshot(params) {
+    const snapshotId = params.revisionKey ?? params.job.currentSnapshotId;
+    const snapshot = params.job.snapshots?.find(
+      (candidate) => candidate.snapshotId === snapshotId
+    );
+    if (!params.job.isDone || !snapshot)
+      throw new ValidationError("Requested immutable parsed snapshot is unavailable");
+    if (snapshot.jobId !== params.job.jobId || snapshot.documentId !== params.documentId)
+      throw new ValidationError(
+        "Parsed snapshot document or job identity differs from the request"
+      );
+    const response = await syncRemoteResultArchive({
+      documentId: params.documentId,
+      revisionKey: snapshot.snapshotId,
+      jobId: snapshot.jobId,
+      snapshot,
+      resultUrl: snapshot.resultUrl,
+      storage: params.parsedStorageConfig.storage,
+      limits: params.parsedStorageConfig.limits,
+      signal: params.signal,
+      retainResultArchive: params.retainResultArchive
+    });
+    if (response.completed && snapshot.snapshotId === params.job.currentSnapshotId) {
+      await serializeSnapshotPublication(
+        params.parsedStorageConfig.storage,
+        params.documentId,
+        async () => {
+          const current = await this.client.documents.listChunks(
+            params.documentId,
+            { page: 1, pageSize: 1 }
+          );
+          if (current.jobResultId !== snapshot.jobResultId) return;
+          const pointer = Buffer.from(
+            JSON.stringify({
+              snapshotId: snapshot.snapshotId,
+              jobResultId: snapshot.jobResultId
+            })
+          );
+          if (snapshot.stage === "final") {
+            await params.parsedStorageConfig.storage.writeObject({
+              documentId: params.documentId,
+              revisionKey: "current",
+              path: ".knowhere-sdk/final-snapshot.json",
+              body: pointer,
+              contentType: "application/json; charset=utf-8"
+            });
+          }
+          await params.parsedStorageConfig.storage.writeObject({
+            documentId: params.documentId,
+            revisionKey: "current",
+            path: ".knowhere-sdk/current-snapshot.json",
+            body: pointer,
+            contentType: "application/json; charset=utf-8"
+          });
+        }
+      );
+    }
+    return response;
+  }
+  async resolveStoredSnapshotKey(documentId) {
+    const object = await this.parsedStorageConfig?.storage.readObject({
+      documentId,
+      revisionKey: "current",
+      path: ".knowhere-sdk/current-snapshot.json"
+    });
+    if (!object) return void 0;
+    const payload = JSON.parse(Buffer.from(object.body).toString("utf8"));
+    if (typeof payload !== "object" || payload === null || !("snapshotId" in payload))
+      throw new ValidationError("Invalid current parsed snapshot pointer");
+    if (typeof payload.snapshotId !== "string")
+      throw new ValidationError("Invalid current parsed snapshot identity");
+    const finalObject = await this.parsedStorageConfig?.storage.readObject({
+      documentId,
+      revisionKey: "current",
+      path: ".knowhere-sdk/final-snapshot.json"
+    });
+    if (finalObject && "jobResultId" in payload) {
+      const finalPayload = JSON.parse(Buffer.from(finalObject.body).toString("utf8"));
+      if (typeof finalPayload === "object" && finalPayload !== null && "snapshotId" in finalPayload && typeof finalPayload.snapshotId === "string" && "jobResultId" in finalPayload && finalPayload.jobResultId === payload.jobResultId)
+        return finalPayload.snapshotId;
+    }
+    return payload.snapshotId;
+  }
   async syncParseResultToConfiguredStorage(result) {
     const parsedStorageConfig = this.parsedStorageConfig;
     if (!parsedStorageConfig) {
@@ -2163,7 +2312,7 @@ var Knowledge = class _Knowledge {
     throw new Error(`Local Knowhere document not found: ${normalized.localDocumentId}`);
   }
   async readRemoteDocumentChunks(documentId, params) {
-    const explicitRevisionKey = params.revisionKey ?? params.jobId;
+    const explicitRevisionKey = params.revisionKey ?? params.jobId ?? await this.resolveStoredSnapshotKey(documentId);
     if (explicitRevisionKey) {
       const stored2 = await this.tryReadChunksFromParsedStorage({
         documentId,
@@ -2173,6 +2322,8 @@ var Knowledge = class _Knowledge {
       if (stored2) {
         return stored2;
       }
+      if (params.revisionKey && await this.resolveStoredSnapshotKey(documentId))
+        throw new ValidationError("Requested parsed snapshot is not committed locally");
     }
     if (hasScanReadFilters(params)) {
       return this.scanRemoteDocumentChunks(documentId, params);
@@ -2185,6 +2336,8 @@ var Knowledge = class _Knowledge {
       includeAssetUrls: true
     });
     const revisionKey = getRemoteRevisionKey(remotePage);
+    if (params.revisionKey && params.revisionKey !== revisionKey)
+      throw new ValidationError("Requested parsed snapshot revision is not committed locally");
     const stored = await this.tryReadChunksFromParsedStorage({
       documentId,
       revisionKey,
@@ -2209,7 +2362,7 @@ var Knowledge = class _Knowledge {
     };
   }
   async scanRemoteDocumentChunks(documentId, params) {
-    const explicitRevisionKey = params.revisionKey ?? params.jobId;
+    const explicitRevisionKey = params.revisionKey ?? params.jobId ?? await this.resolveStoredSnapshotKey(documentId);
     if (explicitRevisionKey) {
       const stored2 = await this.tryReadChunksFromParsedStorage({
         documentId,
@@ -2219,6 +2372,8 @@ var Knowledge = class _Knowledge {
       if (stored2) {
         return stored2;
       }
+      if (params.revisionKey && await this.resolveStoredSnapshotKey(documentId))
+        throw new ValidationError("Requested parsed snapshot is not committed locally");
     }
     const limit = clampLimit(params.limit, DEFAULT_READ_LIMIT, MAX_READ_LIMIT);
     const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
@@ -2229,6 +2384,8 @@ var Knowledge = class _Knowledge {
       includeAssetUrls: true
     });
     const revisionKey = getRemoteRevisionKey(firstPage);
+    if (params.revisionKey && params.revisionKey !== revisionKey)
+      throw new ValidationError("Requested parsed snapshot revision is not committed locally");
     const stored = await this.tryReadChunksFromParsedStorage({
       documentId,
       revisionKey,
@@ -2271,7 +2428,7 @@ var Knowledge = class _Knowledge {
     };
   }
   async grepRemoteDocumentChunks(documentId, params) {
-    const explicitRevisionKey = params.revisionKey ?? params.jobId;
+    const explicitRevisionKey = params.revisionKey ?? params.jobId ?? await this.resolveStoredSnapshotKey(documentId);
     if (explicitRevisionKey && !params.continuationCursor) {
       const stored2 = await this.tryGrepChunksFromParsedStorage({
         documentId,
@@ -2281,6 +2438,8 @@ var Knowledge = class _Knowledge {
       if (stored2) {
         return stored2;
       }
+      if (params.revisionKey && await this.resolveStoredSnapshotKey(documentId))
+        throw new ValidationError("Requested parsed snapshot is not committed locally");
     }
     const maxResults = clampLimit(params.maxResults, DEFAULT_GREP_LIMIT, MAX_GREP_LIMIT);
     const contextChars = params.contextChars ?? DEFAULT_CONTEXT_CHARS;
@@ -2295,6 +2454,8 @@ var Knowledge = class _Knowledge {
       includeAssetUrls: true
     });
     const revisionKey = getRemoteRevisionKey(firstPage);
+    if (params.revisionKey && params.revisionKey !== revisionKey)
+      throw new ValidationError("Requested parsed snapshot revision is not committed locally");
     const stored = startPage === 1 ? await this.tryGrepChunksFromParsedStorage({
       documentId,
       revisionKey,
@@ -2380,11 +2541,14 @@ var Knowledge = class _Knowledge {
     }
   }
   async getRemoteDocumentOutline(documentId, revisionKeyOverride) {
-    if (revisionKeyOverride) {
-      const stored2 = await this.tryReadOutlineFromParsedStorage(documentId, revisionKeyOverride);
+    const storedRevisionKey = revisionKeyOverride ?? await this.resolveStoredSnapshotKey(documentId);
+    if (storedRevisionKey) {
+      const stored2 = await this.tryReadOutlineFromParsedStorage(documentId, storedRevisionKey);
       if (stored2) {
         return stored2;
       }
+      if (revisionKeyOverride && await this.resolveStoredSnapshotKey(documentId))
+        throw new ValidationError("Requested parsed snapshot is not committed locally");
     }
     const limits = getParsedStorageLimits(this.parsedStorageConfig?.limits);
     const firstPage = await this.client.documents.listChunks(documentId, {
@@ -2393,6 +2557,8 @@ var Knowledge = class _Knowledge {
       includeAssetUrls: true
     });
     const revisionKey = getRemoteRevisionKey(firstPage);
+    if (revisionKeyOverride && revisionKeyOverride !== revisionKey)
+      throw new ValidationError("Requested parsed snapshot revision is not committed locally");
     const stored = await this.tryReadOutlineFromParsedStorage(documentId, revisionKey);
     if (stored) {
       return stored;
@@ -2601,11 +2767,7 @@ var Knowledge = class _Knowledge {
     const scheduler = parsedStorageConfig.scheduler ?? defaultParsedStorageScheduler;
     const task = async () => {
       try {
-        await this.syncRemoteDocumentToParsedStorage({
-          documentId,
-          revisionKey,
-          parsedStorageConfig
-        });
+        await this.syncParsedDocument({ documentId, revisionKey });
       } catch (error) {
         const previous = await parsedStorageConfig.storage.readSyncProgress({
           documentId,
@@ -2631,14 +2793,25 @@ var Knowledge = class _Knowledge {
       includeAssetUrls: true
     });
     const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
-    if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
-      throw new ValidationError("Published document revision changed before parsed snapshot sync");
-    }
     if (firstPage.jobId) {
       const job = await this.client.jobs.get(firstPage.jobId).catch((error) => {
         if (error instanceof NotFoundError) return null;
         throw error;
       });
+      if (job?.snapshots?.length) {
+        const snapshot = job.snapshots.find(
+          (candidate) => candidate.snapshotId === (params.revisionKey ?? job.currentSnapshotId)
+        );
+        if (snapshot && snapshot.jobResultId !== getRemoteRevisionKey(firstPage))
+          throw new ValidationError(
+            "Parsed snapshot upstream revision differs from the published document"
+          );
+        return this.syncJobSnapshot({ ...params, job });
+      }
+      if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey)
+        throw new ValidationError(
+          "Published document revision changed before parsed snapshot sync"
+        );
       if (job?.isDone && job.resultUrl) {
         if (job.documentId && job.documentId !== params.documentId)
           throw new ValidationError("Parsed job document identity differs from requested document");

package/dist/{sequential-parsed-document-archive-W7HPYIKA.mjs → sequential-parsed-document-archive-AIRFKO52.mjs}

@@ -3,9 +3,10 @@ import {
   createDownloadIdentity,
   isSnapshotPath,
   normalizeResultPath
-} from "./chunk-DIONXJWK.mjs";
+} from "./chunk-OCR7IDYL.mjs";
 
 // src/storage/sequential-parsed-document-archive.ts
+import { createHash } from "crypto";
 import { Readable, Transform } from "stream";
 import { pipeline } from "stream/promises";
 import { mkdtemp, open, rm } from "fs/promises";
@@ -15,7 +16,10 @@ import { crc32 } from "zlib";
 import { ZipReader, Reader } from "@zip.js/zip.js";
 import Parse from "unzipper/lib/parse.js";
 async function openSequentialArchive(params, identity, validateResponse) {
-  let response = await fetch(params.resultUrl, { signal: params.signal });
+  let response = await fetch(params.resultUrl, {
+    signal: params.signal,
+    headers: validateResponse ? { Range: "bytes=0-" } : void 0
+  });
   if (!response.ok || !response.body)
     throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
   if (validateResponse) response = validateResponse(response);
@@ -44,9 +48,11 @@ async function openSequentialArchive(params, identity, validateResponse) {
   const parser = Parse({ forceStream: true });
   const checksums = /* @__PURE__ */ new Map();
   let archiveSize = 0;
+  const archiveHash = createHash("sha256");
   const spool = new Transform({
     transform(chunk, _encoding, callback) {
       archiveSize += chunk.length;
+      archiveHash.update(chunk);
       void file.writeFile(chunk).then(
         () => {
           callback(null, chunk);
@@ -63,6 +69,23 @@ async function openSequentialArchive(params, identity, validateResponse) {
     identity,
     downloadIdentity: identity,
     canSkipEntries: false,
+    readArchive: () => {
+      let position = 0;
+      return new ReadableStream({
+        async pull(controller) {
+          params.signal?.throwIfAborted();
+          if (position >= archiveSize) {
+            controller.close();
+            return;
+          }
+          const body2 = new Uint8Array(Math.min(65536, archiveSize - position));
+          const { bytesRead } = await file.read(body2, 0, body2.length, position);
+          if (bytesRead !== body2.length) throw new Error("Truncated verified archive spool.");
+          position += bytesRead;
+          controller.enqueue(body2);
+        }
+      });
+    },
     entries: (async function* () {
       for await (const value of parser) {
         const entry = value;
@@ -93,6 +116,8 @@ async function openSequentialArchive(params, identity, validateResponse) {
         };
       }
       await completed;
+      if (params.snapshot && archiveHash.digest("hex") !== params.snapshot.checksum)
+        throw new Error("Parsed snapshot archive checksum mismatch for extracted bytes.");
       const reader = new ZipReader(new DiskArchiveReader(file, archiveSize), {
         useWebWorkers: false
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ontos-ai/knowhere-sdk",
-  "version": "2.6.1-beta.20261027150736",
+  "version": "2.7.0",
   "description": "Official Node.js SDK for Knowhere document parsing API",
   "keywords": [
     "knowhere",