@ontos-ai/knowhere-sdk 2.5.0 → 2.6.0-beta.20261026172248

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1343 @@
1
+ // src/errors/base.ts
2
+ var KnowhereError = class extends Error {
3
+ constructor(message) {
4
+ super(message);
5
+ this.name = "KnowhereError";
6
+ Object.setPrototypeOf(this, new.target.prototype);
7
+ }
8
+ };
9
+ var NetworkError = class extends KnowhereError {
10
+ constructor(message, cause) {
11
+ super(message);
12
+ this.cause = cause;
13
+ this.name = "NetworkError";
14
+ }
15
+ cause;
16
+ };
17
+ var TimeoutError = class extends NetworkError {
18
+ constructor(message = "Request timed out") {
19
+ super(message);
20
+ this.name = "TimeoutError";
21
+ }
22
+ };
23
+ var PollingTimeoutError = class extends KnowhereError {
24
+ constructor(message = "Polling timed out", elapsedMs) {
25
+ super(message);
26
+ this.elapsedMs = elapsedMs;
27
+ this.name = "PollingTimeoutError";
28
+ }
29
+ elapsedMs;
30
+ };
31
+ var ChecksumError = class extends KnowhereError {
32
+ constructor(message = "Checksum verification failed", expected, actual) {
33
+ super(message);
34
+ this.expected = expected;
35
+ this.actual = actual;
36
+ this.name = "ChecksumError";
37
+ }
38
+ expected;
39
+ actual;
40
+ };
41
+ var ValidationError = class extends KnowhereError {
42
+ constructor(message) {
43
+ super(message);
44
+ this.name = "ValidationError";
45
+ }
46
+ };
47
+ var InvalidStateError = class extends KnowhereError {
48
+ constructor(message) {
49
+ super(message);
50
+ this.name = "InvalidStateError";
51
+ }
52
+ };
53
+
54
+ // src/errors/api-errors.ts
55
+ var APIError = class extends KnowhereError {
56
+ constructor(message, statusCode, code, requestId, details, body) {
57
+ super(message);
58
+ this.statusCode = statusCode;
59
+ this.code = code;
60
+ this.requestId = requestId;
61
+ this.details = details;
62
+ this.body = body;
63
+ this.name = "APIError";
64
+ }
65
+ statusCode;
66
+ code;
67
+ requestId;
68
+ details;
69
+ body;
70
+ };
71
+ var BadRequestError = class extends APIError {
72
+ constructor(message, code, requestId, details, body) {
73
+ super(message, 400, code, requestId, details, body);
74
+ this.name = "BadRequestError";
75
+ }
76
+ };
77
+ var AuthenticationError = class extends APIError {
78
+ constructor(message = "Authentication failed", code, requestId, details, body) {
79
+ super(message, 401, code, requestId, details, body);
80
+ this.name = "AuthenticationError";
81
+ }
82
+ };
83
+ var PaymentRequiredError = class extends APIError {
84
+ constructor(message = "Payment required", code, requestId, details, body) {
85
+ super(message, 402, code, requestId, details, body);
86
+ this.name = "PaymentRequiredError";
87
+ }
88
+ };
89
+ var PermissionDeniedError = class extends APIError {
90
+ constructor(message = "Permission denied", code, requestId, details, body) {
91
+ super(message, 403, code, requestId, details, body);
92
+ this.name = "PermissionDeniedError";
93
+ }
94
+ };
95
+ var NotFoundError = class extends APIError {
96
+ constructor(message = "Resource not found", code, requestId, details, body) {
97
+ super(message, 404, code, requestId, details, body);
98
+ this.name = "NotFoundError";
99
+ }
100
+ };
101
+ var ConflictError = class extends APIError {
102
+ constructor(message = "Conflict", code, requestId, details, body) {
103
+ super(message, 409, code, requestId, details, body);
104
+ this.name = "ConflictError";
105
+ }
106
+ };
107
+ var RateLimitError = class extends APIError {
108
+ constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
109
+ super(message, 429, code, requestId, details, body);
110
+ this.retryAfter = retryAfter;
111
+ this.name = "RateLimitError";
112
+ }
113
+ retryAfter;
114
+ };
115
+ var InternalServerError = class extends APIError {
116
+ constructor(message = "Internal server error", code, requestId, details, body) {
117
+ super(message, 500, code, requestId, details, body);
118
+ this.name = "InternalServerError";
119
+ }
120
+ };
121
+ var ServiceUnavailableError = class extends APIError {
122
+ constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
123
+ super(message, statusCode, code, requestId, details, body);
124
+ this.name = "ServiceUnavailableError";
125
+ }
126
+ };
127
+ var GatewayTimeoutError = class extends APIError {
128
+ constructor(message = "Gateway timeout", code, requestId, details, body) {
129
+ super(message, 504, code, requestId, details, body);
130
+ this.name = "GatewayTimeoutError";
131
+ }
132
+ };
133
+ function createAPIError(statusCode, message, code, requestId, details, body, retryAfter) {
134
+ switch (statusCode) {
135
+ case 400:
136
+ return new BadRequestError(message, code, requestId, details, body);
137
+ case 401:
138
+ return new AuthenticationError(message, code, requestId, details, body);
139
+ case 402:
140
+ return new PaymentRequiredError(message, code, requestId, details, body);
141
+ case 403:
142
+ return new PermissionDeniedError(message, code, requestId, details, body);
143
+ case 404:
144
+ return new NotFoundError(message, code, requestId, details, body);
145
+ case 409:
146
+ return new ConflictError(message, code, requestId, details, body);
147
+ case 429:
148
+ return new RateLimitError(message, code, requestId, details, body, retryAfter);
149
+ case 500:
150
+ return new InternalServerError(message, code, requestId, details, body);
151
+ case 502:
152
+ case 503:
153
+ return new ServiceUnavailableError(message, statusCode, code, requestId, details, body);
154
+ case 504:
155
+ return new GatewayTimeoutError(message, code, requestId, details, body);
156
+ default:
157
+ return new APIError(message, statusCode, code, requestId, details, body);
158
+ }
159
+ }
160
+
161
+ // src/errors/job-errors.ts
162
+ var JobFailedError = class extends KnowhereError {
163
+ constructor(message, code, jobResult) {
164
+ super(message);
165
+ this.code = code;
166
+ this.jobResult = jobResult;
167
+ this.name = "JobFailedError";
168
+ }
169
+ code;
170
+ jobResult;
171
+ };
172
+
173
+ // src/errors/parsed-document-archive-error.ts
174
+ var ParsedDocumentArchiveError = class extends KnowhereError {
175
+ code = "ARCHIVE_SPOOL_FAILED";
176
+ cause;
177
+ constructor(cause) {
178
+ super(
179
+ "Parsed result ZIP temporary storage is unavailable. Range-enabled object storage avoids temporary disk usage."
180
+ );
181
+ this.name = "ParsedDocumentArchiveError";
182
+ this.cause = cause;
183
+ }
184
+ };
185
+
186
+ // src/storage/parsed-document-storage.ts
187
+ import JSZip from "jszip";
188
+
189
+ // src/constants.ts
190
+ var DEFAULT_BASE_URL = "https://api.knowhereto.ai";
191
+ var DEFAULT_TIMEOUT = 6e4;
192
+ var DEFAULT_MAX_RETRIES = 5;
193
+ var DEFAULT_POLL_INTERVAL = 1e4;
194
+ var MAX_POLL_INTERVAL = 3e4;
195
+ var DEFAULT_POLL_TIMEOUT = 18e5;
196
+ var POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
197
+ var POLL_INTERVAL_MULTIPLIER = 1.2;
198
+ var INITIAL_RETRY_DELAY = 500;
199
+ var MAX_RETRY_DELAY = 3e4;
200
+ var RETRY_DELAY_BASE = 2;
201
+ var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
202
+ var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
203
+ var ENV = {
204
+ API_KEY: "KNOWHERE_API_KEY",
205
+ BASE_URL: "KNOWHERE_BASE_URL",
206
+ LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
207
+ };
208
+
209
+ // src/lib/utils.ts
210
+ function sleep(ms) {
211
+ return new Promise((resolve) => setTimeout(resolve, ms));
212
+ }
213
+ function snakeToCamel(str) {
214
+ return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
215
+ }
216
+ function camelToSnake(str) {
217
+ return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
218
+ }
219
+ function keysToCamel(obj) {
220
+ if (obj === null || obj === void 0) {
221
+ return obj;
222
+ }
223
+ if (Array.isArray(obj)) {
224
+ return obj.map((item) => keysToCamel(item));
225
+ }
226
+ if (typeof obj === "object" && obj.constructor === Object) {
227
+ const result = {};
228
+ for (const [key, value] of Object.entries(obj)) {
229
+ result[snakeToCamel(key)] = keysToCamel(value);
230
+ }
231
+ return result;
232
+ }
233
+ return obj;
234
+ }
235
+ function keysToSnake(obj) {
236
+ if (obj === null || obj === void 0) {
237
+ return obj;
238
+ }
239
+ if (Array.isArray(obj)) {
240
+ return obj.map((item) => keysToSnake(item));
241
+ }
242
+ if (typeof obj === "object" && obj.constructor === Object) {
243
+ const result = {};
244
+ for (const [key, value] of Object.entries(obj)) {
245
+ result[camelToSnake(key)] = keysToSnake(value);
246
+ }
247
+ return result;
248
+ }
249
+ return obj;
250
+ }
251
+ function parseDates(obj) {
252
+ if (obj === null || obj === void 0) {
253
+ return obj;
254
+ }
255
+ if (Array.isArray(obj)) {
256
+ return obj.map((item) => parseDates(item));
257
+ }
258
+ if (typeof obj === "object" && obj.constructor === Object) {
259
+ const result = {};
260
+ for (const [key, value] of Object.entries(obj)) {
261
+ if ((key.endsWith("At") || key.endsWith("Date")) && typeof value === "string" && /^\d{4}-\d{2}-\d{2}T/.test(value)) {
262
+ result[key] = new Date(value);
263
+ } else {
264
+ result[key] = parseDates(value);
265
+ }
266
+ }
267
+ return result;
268
+ }
269
+ return obj;
270
+ }
271
+ function isTerminalStatus(status) {
272
+ return TERMINAL_JOB_STATUSES.has(status);
273
+ }
274
+ function enrichJobResult(jobResult) {
275
+ const computedProperties = {};
276
+ if (!Object.getOwnPropertyDescriptor(jobResult, "isTerminal")) {
277
+ computedProperties.isTerminal = {
278
+ get() {
279
+ return isTerminalStatus(this.status);
280
+ },
281
+ enumerable: true,
282
+ configurable: true
283
+ };
284
+ }
285
+ if (!Object.getOwnPropertyDescriptor(jobResult, "isDone")) {
286
+ computedProperties.isDone = {
287
+ get() {
288
+ return this.status === "done";
289
+ },
290
+ enumerable: true,
291
+ configurable: true
292
+ };
293
+ }
294
+ if (!Object.getOwnPropertyDescriptor(jobResult, "isFailed")) {
295
+ computedProperties.isFailed = {
296
+ get() {
297
+ return this.status === "failed";
298
+ },
299
+ enumerable: true,
300
+ configurable: true
301
+ };
302
+ }
303
+ if (Object.keys(computedProperties).length > 0) {
304
+ Object.defineProperties(jobResult, computedProperties);
305
+ }
306
+ return jobResult;
307
+ }
308
+ function enrichParseResult(parseResult, scope) {
309
+ if (scope.namespace !== void 0) {
310
+ parseResult.namespace = scope.namespace;
311
+ }
312
+ if (scope.documentId !== void 0) {
313
+ parseResult.documentId = scope.documentId;
314
+ }
315
+ return parseResult;
316
+ }
317
+ function sanitizePath(path) {
318
+ let sanitized = path.replace(/^\/+/, "");
319
+ sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
320
+ sanitized = sanitized.replace(/\\/g, "/");
321
+ return sanitized;
322
+ }
323
+ function getFileExtension(filename) {
324
+ const match = filename.match(/\.([^.]+)$/);
325
+ return match ? match[1].toLowerCase() : "";
326
+ }
327
+ function jitter(value, percent = 0.2) {
328
+ const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
329
+ return Math.round(value * randomFactor);
330
+ }
331
+
332
+ // src/storage/parsed-document-storage.ts
333
+ var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
334
+ var jsonContentType = "application/json; charset=utf-8";
335
+ var markdownContentType = "text/markdown; charset=utf-8";
336
+ var csvContentType = "text/csv; charset=utf-8";
337
+ var htmlContentType = "text/html; charset=utf-8";
338
+ var binaryContentType = "application/octet-stream";
339
+ async function syncParseResultToParsedDocumentStorage(params) {
340
+ const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
341
+ const revisionKey = params.revisionKey ?? params.result.jobId;
342
+ for await (const object of collectResultLayoutObjects(params.result)) {
343
+ params.signal?.throwIfAborted();
344
+ await params.storage.writeObject({
345
+ documentId,
346
+ revisionKey,
347
+ ...object
348
+ });
349
+ }
350
+ await params.beforeCommit?.();
351
+ params.signal?.throwIfAborted();
352
+ const commit = {
353
+ version: 1,
354
+ documentId,
355
+ revisionKey,
356
+ source: params.source ?? "resultZip",
357
+ committedAt: (/* @__PURE__ */ new Date()).toISOString()
358
+ };
359
+ await writeParsedDocumentJsonObject(params.storage, {
360
+ documentId,
361
+ revisionKey,
362
+ path: parsedDocumentCommitPath,
363
+ value: commit
364
+ });
365
+ return {
366
+ result: params.result,
367
+ documentId,
368
+ revisionKey,
369
+ commit
370
+ };
371
+ }
372
+ async function readParsedDocumentCommit(params) {
373
+ const object = await params.storage.readObject({
374
+ documentId: params.documentId,
375
+ revisionKey: params.revisionKey,
376
+ path: parsedDocumentCommitPath
377
+ });
378
+ if (!object) {
379
+ return null;
380
+ }
381
+ const commit = readJsonObject(object);
382
+ return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
383
+ }
384
+ function readJsonObject(object) {
385
+ return JSON.parse(Buffer.from(object.body).toString("utf8"));
386
+ }
387
+ async function* collectResultLayoutObjects(result) {
388
+ const storedPaths = /* @__PURE__ */ new Set();
389
+ let zip;
390
+ try {
391
+ zip = await JSZip.loadAsync(result.rawZip);
392
+ } catch {
393
+ }
394
+ if (zip) {
395
+ for (const entry of Object.values(zip.files)) {
396
+ if (entry.dir || entry.name === "result.zip") continue;
397
+ const path = normalizeResultPath(entry.name);
398
+ if (isSdkMetadataPath(path)) continue;
399
+ storedPaths.add(path);
400
+ yield {
401
+ path,
402
+ body: await entry.async("nodebuffer"),
403
+ contentType: inferContentTypeFromPath(path)
404
+ };
405
+ }
406
+ }
407
+ for (const object of generateMissingResultObjects(result, storedPaths)) yield object;
408
+ }
409
+ function* generateMissingResultObjects(result, storedPaths) {
410
+ if (!storedPaths.has("manifest.json")) yield createJsonObject("manifest.json", result.manifest);
411
+ if (!storedPaths.has("chunks.json"))
412
+ yield createJsonObject("chunks.json", keysToSnake(serializeChunks(result.chunks)));
413
+ if (!storedPaths.has("doc_nav.json") && result.docNav)
414
+ yield createJsonObject("doc_nav.json", result.docNav);
415
+ if (!storedPaths.has("full.md") && result.fullMarkdown)
416
+ yield createTextObject("full.md", result.fullMarkdown, markdownContentType);
417
+ if (!storedPaths.has("chunks_slim.json") && result.chunksSlim)
418
+ yield createJsonObject("chunks_slim.json", { chunks: result.chunksSlim });
419
+ if (!storedPaths.has("hierarchy.json") && result.hierarchy)
420
+ yield createJsonObject("hierarchy.json", result.hierarchy);
421
+ if (!storedPaths.has("toc_hierarchies.json") && result.tocHierarchies)
422
+ yield createJsonObject("toc_hierarchies.json", result.tocHierarchies);
423
+ if (!storedPaths.has("kb.csv") && result.kbCsv)
424
+ yield createTextObject("kb.csv", result.kbCsv, csvContentType);
425
+ if (!storedPaths.has("hierarchy_view.html") && result.hierarchyViewHtml)
426
+ yield createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType);
427
+ for (const chunk of result.imageChunks) {
428
+ const path = normalizeResultPath(chunk.filePath);
429
+ if (storedPaths.has(path)) continue;
430
+ yield {
431
+ path,
432
+ body: chunk.data,
433
+ contentType: inferContentTypeFromPath(path),
434
+ metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
435
+ };
436
+ }
437
+ for (const chunk of result.tableChunks) {
438
+ const path = normalizeResultPath(chunk.filePath);
439
+ if (storedPaths.has(path)) continue;
440
+ yield {
441
+ path,
442
+ body: Buffer.from(chunk.html, "utf8"),
443
+ contentType: htmlContentType,
444
+ metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
445
+ };
446
+ }
447
+ }
448
+ function serializeChunks(chunks) {
449
+ return {
450
+ chunks: chunks.map((chunk) => {
451
+ const rawChunk = {
452
+ chunkId: chunk.chunkId,
453
+ type: chunk.type,
454
+ contentSource: chunk.contentSource,
455
+ content: chunk.content,
456
+ path: chunk.path,
457
+ metadata: chunk.metadata
458
+ };
459
+ if (chunk.type === "image" || chunk.type === "table") {
460
+ return {
461
+ ...rawChunk,
462
+ filePath: chunk.filePath,
463
+ assetUrl: chunk.assetUrl
464
+ };
465
+ }
466
+ return rawChunk;
467
+ })
468
+ };
469
+ }
470
+ async function writeParsedDocumentJsonObject(storage, params) {
471
+ await storage.writeObject({
472
+ documentId: params.documentId,
473
+ revisionKey: params.revisionKey,
474
+ path: params.path,
475
+ body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
476
+ contentType: jsonContentType
477
+ });
478
+ }
479
+ function createJsonObject(path, value) {
480
+ return {
481
+ path,
482
+ body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
483
+ contentType: jsonContentType
484
+ };
485
+ }
486
+ function createTextObject(path, value, contentType) {
487
+ return {
488
+ path,
489
+ body: Buffer.from(value, "utf8"),
490
+ contentType
491
+ };
492
+ }
493
+ function inferContentTypeFromPath(filePath) {
494
+ const lower = filePath.toLowerCase();
495
+ if (lower.endsWith(".json")) return jsonContentType;
496
+ if (lower.endsWith(".md")) return markdownContentType;
497
+ if (lower.endsWith(".csv")) return csvContentType;
498
+ if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
499
+ if (lower.endsWith(".png")) return "image/png";
500
+ if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
501
+ if (lower.endsWith(".gif")) return "image/gif";
502
+ if (lower.endsWith(".webp")) return "image/webp";
503
+ if (lower.endsWith(".svg")) return "image/svg+xml";
504
+ return binaryContentType;
505
+ }
506
+ function normalizeResultPath(value) {
507
+ const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
508
+ if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
509
+ throw new Error(`Invalid parsed result path: ${value}`);
510
+ }
511
+ return normalized;
512
+ }
513
+ function isSdkMetadataPath(path) {
514
+ return path === ".knowhere-sdk" || path.startsWith(".knowhere-sdk/");
515
+ }
516
+
517
+ // src/storage/streamed-parsed-document-sync.ts
518
+ import { createHash as createHash2 } from "crypto";
519
+
520
+ // src/storage/archive-cache-transport-error.ts
521
+ var ArchiveCacheTransportError = class extends Error {
522
+ constructor(message) {
523
+ super(message);
524
+ this.name = "ArchiveCacheTransportError";
525
+ }
526
+ };
527
+
528
+ // src/storage/parsed-document-archive-cache.ts
529
+ import { createHash } from "crypto";
530
+ var ARCHIVE_CACHE_TIMEOUT_MS = 3e4;
531
+ var ArchiveOriginIntegrityError = class extends Error {
532
+ };
533
+ async function prepareCachedArchive(params) {
534
+ params.signal?.throwIfAborted();
535
+ if (params.archiveCacheBypassed) return createFallback(params);
536
+ if (!params.storage.writeObjectStream || !params.storage.headObject || !params.storage.getObjectUrl) {
537
+ await params.onBypass?.();
538
+ return createFallback(params);
539
+ }
540
+ const controller = new AbortController();
541
+ const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
542
+ const mutations = /* @__PURE__ */ new Set();
543
+ let rejectCancellation;
544
+ const cancellation = new Promise((_resolve, reject) => {
545
+ rejectCancellation = () => {
546
+ const reason = signal.reason;
547
+ reject(
548
+ reason instanceof Error ? reason : new Error("Parsed result archive cache was cancelled.", { cause: reason })
549
+ );
550
+ };
551
+ signal.addEventListener("abort", rejectCancellation, { once: true });
552
+ });
553
+ const timer = setTimeout(() => {
554
+ controller.abort(new Error("Parsed result archive cache exceeded its thirty-second budget."));
555
+ }, ARCHIVE_CACHE_TIMEOUT_MS);
556
+ let origin;
557
+ const preparation = (async () => {
558
+ origin = await probeArchiveOrigin({ ...params, signal });
559
+ signal.throwIfAborted();
560
+ if (!hasStrongEntityTag(origin.entityTag) || origin.contentLength === void 0)
561
+ throw new Error("Parsed result archive cache requires a strong origin version and length.");
562
+ return prepareArchiveCache(params, origin, signal, mutations);
563
+ })();
564
+ try {
565
+ return await Promise.race([preparation, cancellation]);
566
+ } catch (error) {
567
+ controller.abort(error);
568
+ await Promise.allSettled([...mutations]);
569
+ params.signal?.throwIfAborted();
570
+ if (error instanceof ArchiveOriginIntegrityError) throw error;
571
+ await params.onBypass?.();
572
+ return createFallback(params, origin);
573
+ } finally {
574
+ clearTimeout(timer);
575
+ if (rejectCancellation) signal.removeEventListener("abort", rejectCancellation);
576
+ }
577
+ }
578
+ function createFallback(params, origin) {
579
+ return {
580
+ resultUrl: params.resultUrl,
581
+ downloadIdentity: origin?.identity,
582
+ archiveCacheBypassed: true,
583
+ isCached: false,
584
+ validateResponse(response) {
585
+ if (origin && createOriginIdentity(params.jobId, response) !== origin.identity)
586
+ rejectResponse(
587
+ response,
588
+ new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.")
589
+ );
590
+ return response;
591
+ }
592
+ };
593
+ }
594
+ async function probeArchiveOrigin(params) {
595
+ const response = await fetch(params.resultUrl, {
596
+ headers: { Range: "bytes=0-0" },
597
+ signal: params.signal
598
+ });
599
+ try {
600
+ if (!response.ok) throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
601
+ return {
602
+ identity: createOriginIdentity(params.jobId, response),
603
+ contentLength: readOriginLength(response),
604
+ entityTag: response.headers.get("etag")
605
+ };
606
+ } finally {
607
+ await response.body?.cancel();
608
+ }
609
+ }
610
+ async function prepareArchiveCache(params, origin, signal, mutations) {
611
+ const object = { ...params, path: "result.zip" };
612
+ const initialHead = await params.storage.headObject(object);
613
+ signal.throwIfAborted();
614
+ let marker = null;
615
+ if (initialHead) {
616
+ const version = readObjectVersion(initialHead);
617
+ const stored = await params.storage.readObject({
618
+ ...object,
619
+ path: createMarkerPath(origin.identity, version.entityTag)
620
+ });
621
+ signal.throwIfAborted();
622
+ if (stored) marker = readMarker(stored, params, origin, version);
623
+ }
624
+ if (!marker) marker = await storeArchive(params, origin, signal, mutations);
625
+ signal.throwIfAborted();
626
+ const resultUrl = await params.storage.getObjectUrl(object);
627
+ signal.throwIfAborted();
628
+ if (!resultUrl) throw new Error("Parsed result archive cache has no readable object URL.");
629
+ const validateResponse = (response2) => validateCachedResponse(response2, marker, params.signal);
630
+ const response = await fetch(resultUrl, {
631
+ headers: { Range: "bytes=0-0", "If-Match": marker.storageEntityTag },
632
+ signal
633
+ });
634
+ try {
635
+ if (response.headers.get("content-range") !== `bytes 0-0/${marker.contentLength}` || response.headers.get("content-length") !== "1")
636
+ rejectResponse(
637
+ response,
638
+ new ArchiveCacheTransportError("Parsed result archive cache probe exceeded one byte.")
639
+ );
640
+ const validated = validateResponse(response);
641
+ await validated.arrayBuffer();
642
+ } catch (error) {
643
+ await response.body?.cancel().catch(() => void 0);
644
+ throw error;
645
+ }
646
+ signal.throwIfAborted();
647
+ return {
648
+ resultUrl,
649
+ downloadIdentity: origin.identity,
650
+ archiveCacheBypassed: false,
651
+ isCached: true,
652
+ validateResponse
653
+ };
654
+ }
655
+ async function storeArchive(params, origin, signal, mutations) {
656
+ const response = await fetch(params.resultUrl, {
657
+ headers: { "If-Match": origin.entityTag },
658
+ signal
659
+ });
660
+ if (signal.aborted) {
661
+ await response.body?.cancel();
662
+ signal.throwIfAborted();
663
+ }
664
+ if (!response.ok || !response.body) {
665
+ await response.body?.cancel();
666
+ if (response.status === 412)
667
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
668
+ throw new Error(`Parsed result archive cache download failed (HTTP ${response.status}).`);
669
+ }
670
+ if (createOriginIdentity(params.jobId, response) !== origin.identity) {
671
+ await response.body.cancel();
672
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
673
+ }
674
+ const hash = createHash("sha256");
675
+ let contentLength = 0;
676
+ let hasCompleted = false;
677
+ const body = response.body.pipeThrough(
678
+ new TransformStream({
679
+ transform(chunk, controller) {
680
+ signal.throwIfAborted();
681
+ contentLength += chunk.length;
682
+ if (contentLength > origin.contentLength)
683
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
684
+ hash.update(chunk);
685
+ controller.enqueue(chunk);
686
+ },
687
+ flush() {
688
+ if (contentLength !== origin.contentLength)
689
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
690
+ hasCompleted = true;
691
+ }
692
+ })
693
+ );
694
+ const object = { ...params, path: "result.zip" };
695
+ let write;
696
+ try {
697
+ signal.throwIfAborted();
698
+ write = await settleMutation(
699
+ params.storage.writeObjectStream({
700
+ ...object,
701
+ body,
702
+ contentType: "application/zip",
703
+ contentLength: origin.contentLength,
704
+ signal
705
+ }),
706
+ mutations
707
+ );
708
+ } catch (error) {
709
+ await body.cancel(error).catch(() => void 0);
710
+ throw error;
711
+ }
712
+ signal.throwIfAborted();
713
+ if (!hasCompleted)
714
+ throw new Error("Parsed result archive cache sink did not consume the stream.");
715
+ if (!hasStrongEntityTag(write.entityTag))
716
+ throw new Error("Parsed result archive cache requires a strong object version.");
717
+ const head = await params.storage.headObject(object);
718
+ signal.throwIfAborted();
719
+ if (!head) throw new Error("Parsed result archive cache was not persisted.");
720
+ const actualVersion = readObjectVersion(head);
721
+ if (write.entityTag !== actualVersion.entityTag || actualVersion.contentLength !== contentLength)
722
+ throw new Error("Parsed result archive cache object version or length changed during upload.");
723
+ const marker = {
724
+ version: 1,
725
+ jobId: params.jobId,
726
+ revisionKey: params.revisionKey,
727
+ originIdentity: origin.identity,
728
+ contentLength,
729
+ sha256: hash.digest("hex"),
730
+ storageEntityTag: actualVersion.entityTag
731
+ };
732
+ signal.throwIfAborted();
733
+ await settleMutation(
734
+ params.storage.writeObject({
735
+ ...object,
736
+ path: createMarkerPath(origin.identity, actualVersion.entityTag),
737
+ body: Buffer.from(JSON.stringify(marker)),
738
+ contentType: "application/json; charset=utf-8"
739
+ }),
740
+ mutations
741
+ );
742
+ signal.throwIfAborted();
743
+ return marker;
744
+ }
745
+ async function settleMutation(mutation, mutations) {
746
+ mutations.add(mutation);
747
+ try {
748
+ return await mutation;
749
+ } finally {
750
+ mutations.delete(mutation);
751
+ }
752
+ }
753
+ function readMarker(object, params, origin, version) {
754
+ const value = JSON.parse(Buffer.from(object.body).toString("utf8"));
755
+ if (typeof value !== "object" || value === null || !("version" in value) || value.version !== 1 || !("jobId" in value) || value.jobId !== params.jobId || !("revisionKey" in value) || value.revisionKey !== params.revisionKey || !("originIdentity" in value) || value.originIdentity !== origin.identity || !("contentLength" in value) || value.contentLength !== origin.contentLength || value.contentLength !== version.contentLength || !("storageEntityTag" in value) || value.storageEntityTag !== version.entityTag || !("sha256" in value) || typeof value.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(value.sha256))
756
+ throw new Error("Parsed result archive cache completion marker is invalid.");
757
+ return {
758
+ version: 1,
759
+ jobId: params.jobId,
760
+ revisionKey: params.revisionKey,
761
+ originIdentity: origin.identity,
762
+ contentLength: version.contentLength,
763
+ storageEntityTag: version.entityTag,
764
+ sha256: value.sha256
765
+ };
766
+ }
767
+ function readObjectVersion(object) {
768
+ if (!hasStrongEntityTag(object.entityTag) || !isPositiveInteger(object.contentLength))
769
+ throw new Error("Parsed result archive cache requires a strong object version and length.");
770
+ return { entityTag: object.entityTag, contentLength: object.contentLength };
771
+ }
772
+ function validateCachedResponse(response, marker, signal) {
773
+ if (response.headers.get("etag") !== marker.storageEntityTag)
774
+ rejectResponse(
775
+ response,
776
+ new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
777
+ );
778
+ if (response.status !== 206 || !response.body)
779
+ rejectResponse(
780
+ response,
781
+ new ArchiveCacheTransportError("Parsed result archive cache requires a ranged response.")
782
+ );
783
+ const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
784
+ response.headers.get("content-range") ?? ""
785
+ );
786
+ const contentLength = Number(response.headers.get("content-length"));
787
+ if (!range || Number(range[3]) !== marker.contentLength || Number(range[1]) > Number(range[2]) || Number(range[2]) >= marker.contentLength || contentLength !== Number(range[2]) - Number(range[1]) + 1)
788
+ rejectResponse(
789
+ response,
790
+ new ArchiveCacheTransportError("Parsed result archive cache response length changed.")
791
+ );
792
+ let actualLength = 0;
793
+ return new Response(
794
+ response.body.pipeThrough(
795
+ new TransformStream({
796
+ transform(chunk, controller) {
797
+ signal?.throwIfAborted();
798
+ actualLength += chunk.length;
799
+ if (actualLength > contentLength)
800
+ throw new ArchiveCacheTransportError(
801
+ "Parsed result archive cache response length changed."
802
+ );
803
+ controller.enqueue(chunk);
804
+ },
805
+ flush() {
806
+ if (actualLength !== contentLength)
807
+ throw new ArchiveCacheTransportError(
808
+ "Parsed result archive cache response length changed."
809
+ );
810
+ }
811
+ })
812
+ ),
813
+ { status: response.status, statusText: response.statusText, headers: response.headers }
814
+ );
815
+ }
816
+ function rejectResponse(response, error) {
817
+ void response.body?.cancel(error).catch(() => void 0);
818
+ throw error;
819
+ }
820
+ function createMarkerPath(originIdentity, entityTag) {
821
+ const identity = createHash("sha256").update(JSON.stringify([originIdentity, entityTag])).digest("hex");
822
+ return `.knowhere-sdk/archive-cache/${identity}.json`;
823
+ }
824
+ function createOriginIdentity(jobId, response) {
825
+ return createHash("sha256").update(
826
+ JSON.stringify([
827
+ jobId,
828
+ response.headers.get("etag"),
829
+ response.headers.get("last-modified"),
830
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
831
+ ])
832
+ ).digest("hex");
833
+ }
834
+ function readOriginLength(response) {
835
+ const value = Number(
836
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
837
+ );
838
+ return isPositiveInteger(value) ? value : void 0;
839
+ }
840
+ function hasStrongEntityTag(value) {
841
+ return typeof value === "string" && value.length > 0 && !value.startsWith("W/");
842
+ }
843
+ function isPositiveInteger(value) {
844
+ return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
845
+ }
846
+
847
+ // src/storage/streamed-parsed-document-sync.ts
848
+ import { HttpRangeReader, ZipReader } from "@zip.js/zip.js";
849
+ async function syncRemoteResultArchive(params) {
850
+ params.signal?.throwIfAborted();
851
+ validateArchiveSyncLimits(params.limits);
852
+ const startedAt = Date.now();
853
+ const maxEntries = params.limits?.maxEntriesPerSync ?? 100;
854
+ const deadlineMs = params.limits?.archiveSyncDeadlineMs ?? 12e4;
855
+ const checkpointFileInterval = params.limits?.checkpointFileInterval ?? 1;
856
+ const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
857
+ const controller = new AbortController();
858
+ const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
859
+ const transferParams = { ...params, signal };
860
+ const previous = await params.storage.readSyncProgress(params);
861
+ const source = await openArchive(transferParams, previous);
862
+ const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
863
+ const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
864
+ const entryHashes = [];
865
+ const successfulHashes = /* @__PURE__ */ new Map();
866
+ const running = /* @__PURE__ */ new Set();
867
+ const paths = /* @__PURE__ */ new Set();
868
+ let entryIndex = 0;
869
+ let startedEntries = 0;
870
+ let successfulEntries = 0;
871
+ let checkpointEntry = nextEntry;
872
+ let checkpointAt = Date.now();
873
+ let failure;
874
+ let archiveCacheBypassed = source.archiveCacheBypassed;
875
+ async function flushProgress(isSegmentBoundary, isRecovering = false) {
876
+ let contiguousEntry = checkpointEntry;
877
+ while (successfulHashes.has(contiguousEntry)) {
878
+ entryHashes[contiguousEntry] = successfulHashes.get(contiguousEntry);
879
+ contiguousEntry += 1;
880
+ }
881
+ if (contiguousEntry === checkpointEntry) return;
882
+ await params.storage.writeSyncProgress(
883
+ {
884
+ documentId: params.documentId,
885
+ revisionKey: params.revisionKey,
886
+ pinnedRevisionKey: params.revisionKey,
887
+ archiveIdentity: source.identity,
888
+ archiveCacheBypassed,
889
+ archiveEntryHashes: entryHashes.slice(0, contiguousEntry),
890
+ nextArchiveEntry: contiguousEntry,
891
+ status: "running",
892
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
893
+ },
894
+ { signal: isRecovering ? void 0 : signal, isSegmentBoundary }
895
+ );
896
+ for (let index = checkpointEntry; index < contiguousEntry; index += 1)
897
+ successfulHashes.delete(index);
898
+ checkpointEntry = contiguousEntry;
899
+ successfulEntries = successfulHashes.size;
900
+ checkpointAt = Date.now();
901
+ }
902
+ function startEntry(entry, index) {
903
+ const task = copyEntry(transferParams, entry).then(
904
+ (hash) => {
905
+ successfulHashes.set(index, hash);
906
+ successfulEntries += 1;
907
+ },
908
+ (error) => {
909
+ if (failure === void 0)
910
+ failure = error instanceof Error ? error : new Error(String(error));
911
+ controller.abort(error);
912
+ }
913
+ ).finally(() => {
914
+ running.delete(task);
915
+ });
916
+ running.add(task);
917
+ }
918
+ async function drainEntries() {
919
+ await Promise.all(running);
920
+ if (failure !== void 0) throw failure;
921
+ signal.throwIfAborted();
922
+ }
923
+ try {
924
+ if (previous?.archiveIdentity && previous.archiveIdentity !== source.identity)
925
+ throw new Error("Parsed result archive identity changed; start a new revision.");
926
+ if (previous?.pinnedRevisionKey && previous.pinnedRevisionKey !== params.revisionKey)
927
+ throw new Error("Parsed result revision changed while resuming archive sync.");
928
+ for await (const entry of source.entries) {
929
+ if (failure !== void 0) throw failure;
930
+ signal.throwIfAborted();
931
+ if (paths.has(entry.path)) throw new Error(`Duplicate parsed result path: ${entry.path}`);
932
+ paths.add(entry.path);
933
+ if (entryIndex < nextEntry) {
934
+ const previousHash = previous?.archiveEntryHashes?.[entryIndex];
935
+ if (!previousHash) throw new Error("Parsed archive checkpoint is missing an entry hash.");
936
+ if (!source.canSkipEntries && await copyEntry(transferParams, entry, true) !== previousHash)
937
+ throw new Error("Parsed result archive entry changed while resuming.");
938
+ entryHashes.push(previousHash);
939
+ entryIndex += 1;
940
+ continue;
941
+ }
942
+ if (startedEntries > 0 && (startedEntries >= maxEntries || Date.now() - startedAt >= deadlineMs)) {
943
+ await drainEntries();
944
+ await flushProgress(true);
945
+ return { documentId: params.documentId, revisionKey: params.revisionKey, completed: false };
946
+ }
947
+ if (successfulEntries >= checkpointFileInterval || Date.now() - checkpointAt >= checkpointIntervalMs) {
948
+ await drainEntries();
949
+ await flushProgress(false);
950
+ }
951
+ startEntry(entry, entryIndex);
952
+ entryIndex += 1;
953
+ startedEntries += 1;
954
+ if (running.size >= concurrency) await Promise.race(running);
955
+ }
956
+ await drainEntries();
957
+ await flushProgress(true);
958
+ await validateRequiredObjects(params, paths);
959
+ await params.verifyRevision?.();
960
+ signal.throwIfAborted();
961
+ const commit = {
962
+ version: 1,
963
+ documentId: params.documentId,
964
+ revisionKey: params.revisionKey,
965
+ source: "resultZip",
966
+ committedAt: (/* @__PURE__ */ new Date()).toISOString()
967
+ };
968
+ await params.storage.writeObject({
969
+ documentId: params.documentId,
970
+ revisionKey: params.revisionKey,
971
+ path: parsedDocumentCommitPath,
972
+ body: Buffer.from(JSON.stringify(commit, null, 2)),
973
+ contentType: "application/json; charset=utf-8"
974
+ });
975
+ await params.storage.writeSyncProgress(
976
+ {
977
+ documentId: params.documentId,
978
+ revisionKey: params.revisionKey,
979
+ pinnedRevisionKey: params.revisionKey,
980
+ archiveIdentity: source.identity,
981
+ archiveCacheBypassed,
982
+ archiveEntryHashes: entryHashes,
983
+ nextArchiveEntry: entryIndex,
984
+ status: "completed",
985
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
986
+ },
987
+ { signal, isSegmentBoundary: true }
988
+ );
989
+ if (params.retainResultArchive && !source.isCached) {
990
+ try {
991
+ await retainArchive(transferParams, source.downloadIdentity);
992
+ } catch {
993
+ }
994
+ }
995
+ return {
996
+ documentId: params.documentId,
997
+ revisionKey: params.revisionKey,
998
+ completed: true,
999
+ commit
1000
+ };
1001
+ } catch (error) {
1002
+ controller.abort(error);
1003
+ await Promise.all(running);
1004
+ const hasCacheTransportFailure = error instanceof ArchiveCacheTransportError || failure instanceof ArchiveCacheTransportError;
1005
+ if (hasCacheTransportFailure) archiveCacheBypassed = true;
1006
+ await flushProgress(true, true).catch(() => void 0);
1007
+ if (hasCacheTransportFailure) {
1008
+ await persistArchiveCacheBypass(
1009
+ { ...params, signal: void 0 },
1010
+ {
1011
+ documentId: params.documentId,
1012
+ revisionKey: params.revisionKey,
1013
+ pinnedRevisionKey: params.revisionKey,
1014
+ archiveIdentity: source.identity,
1015
+ archiveEntryHashes: entryHashes.slice(0, checkpointEntry),
1016
+ nextArchiveEntry: checkpointEntry,
1017
+ status: "running",
1018
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1019
+ }
1020
+ ).catch(() => void 0);
1021
+ }
1022
+ throw failure ?? error;
1023
+ } finally {
1024
+ controller.abort();
1025
+ await Promise.all(running);
1026
+ await source.close();
1027
+ }
1028
+ }
1029
+ function validateArchiveSyncLimits(limits) {
1030
+ const names = [
1031
+ "maxEntriesPerSync",
1032
+ "archiveSyncDeadlineMs",
1033
+ "maxConcurrentFiles",
1034
+ "checkpointFileInterval",
1035
+ "checkpointIntervalMs"
1036
+ ];
1037
+ for (const name of names) {
1038
+ const value = limits?.[name];
1039
+ if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0))
1040
+ throw new ValidationError(`${name} must be a positive safe integer.`);
1041
+ }
1042
+ }
1043
+ async function copyEntry(params, entry, isVerification = false) {
1044
+ const hash = createHash2("sha256");
1045
+ const body = entry.read().pipeThrough(
1046
+ new TransformStream({
1047
+ transform(chunk, controller) {
1048
+ params.signal?.throwIfAborted();
1049
+ hash.update(chunk);
1050
+ controller.enqueue(chunk);
1051
+ }
1052
+ })
1053
+ );
1054
+ if (isVerification)
1055
+ await body.pipeTo(new WritableStream(), { signal: params.signal });
1056
+ else
1057
+ await writeStream(params.storage, {
1058
+ documentId: params.documentId,
1059
+ revisionKey: params.revisionKey,
1060
+ path: entry.path,
1061
+ body,
1062
+ contentType: inferContentTypeFromPath(entry.path),
1063
+ contentLength: entry.contentLength,
1064
+ signal: params.signal
1065
+ });
1066
+ return createHash2("sha256").update(entry.path).update(hash.digest()).digest("hex");
1067
+ }
1068
+ async function openArchive(params, previous) {
1069
+ const prepared = params.cacheResultArchive ? await prepareCachedArchive({
1070
+ ...params,
1071
+ archiveCacheBypassed: previous?.archiveCacheBypassed,
1072
+ onBypass: async () => {
1073
+ await persistArchiveCacheBypass(params, previous);
1074
+ }
1075
+ }) : void 0;
1076
+ try {
1077
+ return await openArchiveTransport(params, previous, prepared);
1078
+ } catch (error) {
1079
+ if (!prepared?.isCached) throw error;
1080
+ params.signal?.throwIfAborted();
1081
+ const bypass = await persistArchiveCacheBypass(params, previous);
1082
+ return openArchive({ ...params, cacheResultArchive: false }, bypass);
1083
+ }
1084
+ }
1085
+ async function persistArchiveCacheBypass(params, previous) {
1086
+ const progress = {
1087
+ ...previous?.status === "completed" ? {} : previous,
1088
+ documentId: params.documentId,
1089
+ revisionKey: params.revisionKey,
1090
+ pinnedRevisionKey: params.revisionKey,
1091
+ archiveCacheBypassed: true,
1092
+ status: "running",
1093
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1094
+ };
1095
+ await params.storage.writeSyncProgress(progress, { signal: params.signal });
1096
+ return progress;
1097
+ }
1098
+ async function openArchiveTransport(params, previous, prepared) {
1099
+ const resultUrl = prepared?.resultUrl ?? params.resultUrl;
1100
+ const archiveCacheBypassed = prepared?.archiveCacheBypassed ?? previous?.archiveCacheBypassed;
1101
+ let probe = await fetch(resultUrl, {
1102
+ headers: { Range: "bytes=0-0" },
1103
+ signal: params.signal
1104
+ });
1105
+ if (!probe.ok) {
1106
+ await probe.body?.cancel();
1107
+ throw new Error(`Parsed result download failed (HTTP ${probe.status}).`);
1108
+ }
1109
+ if (prepared) probe = prepared.validateResponse(probe);
1110
+ const identity = prepared?.downloadIdentity ?? createDownloadIdentity(params.jobId, probe);
1111
+ const supportsRange = probe.status === 206 && probe.headers.has("content-range");
1112
+ await probe.body?.cancel();
1113
+ if (!supportsRange) {
1114
+ const { openSequentialArchive } = await import("./sequential-parsed-document-archive-YQ5U3VWM.mjs");
1115
+ const source = await openSequentialArchive(
1116
+ { ...params, resultUrl },
1117
+ identity,
1118
+ prepared?.isCached ? prepared.validateResponse : void 0
1119
+ );
1120
+ return { ...source, archiveCacheBypassed };
1121
+ }
1122
+ const options = {
1123
+ forceRangeRequests: true,
1124
+ preventHeadRequest: true,
1125
+ checkResourceChanges: true,
1126
+ fetch: async (input, init) => {
1127
+ let response;
1128
+ try {
1129
+ response = await fetch(input, { ...init, signal: params.signal });
1130
+ } catch (error) {
1131
+ params.signal?.throwIfAborted();
1132
+ if (prepared?.isCached)
1133
+ throw new ArchiveCacheTransportError(
1134
+ "Parsed result archive cache transport unavailable."
1135
+ );
1136
+ throw error;
1137
+ }
1138
+ if (prepared?.isCached) return prepared.validateResponse(response);
1139
+ if (response.status !== 206) {
1140
+ await response.body?.cancel();
1141
+ throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
1142
+ }
1143
+ return prepared ? prepared.validateResponse(response) : response;
1144
+ }
1145
+ };
1146
+ const reader = new ZipReader(new HttpRangeReader(resultUrl, options), {
1147
+ useWebWorkers: false,
1148
+ checkCrc32: true
1149
+ });
1150
+ const producers = /* @__PURE__ */ new Set();
1151
+ try {
1152
+ const entries = await reader.getEntries();
1153
+ const files = entries.filter(
1154
+ (entry) => !entry.directory && isSnapshotPath(entry.filename)
1155
+ );
1156
+ const archiveIdentity = createHash2("sha256").update(identity).update(
1157
+ JSON.stringify(
1158
+ files.map((entry) => [entry.filename, entry.uncompressedSize, entry.signature])
1159
+ )
1160
+ ).digest("hex");
1161
+ return {
1162
+ identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
1163
+ downloadIdentity: identity,
1164
+ archiveCacheBypassed,
1165
+ isCached: prepared?.isCached,
1166
+ canSkipEntries: previous?.archiveIdentity !== identity,
1167
+ entries: (function* () {
1168
+ for (const entry of files)
1169
+ yield {
1170
+ path: normalizeResultPath(entry.filename),
1171
+ contentLength: entry.uncompressedSize,
1172
+ read: () => readRangeEntry(entry, params.signal, producers)
1173
+ };
1174
+ })(),
1175
+ close: async () => {
1176
+ await Promise.all(producers);
1177
+ await reader.close();
1178
+ }
1179
+ };
1180
+ } catch (error) {
1181
+ await reader.close();
1182
+ throw error;
1183
+ }
1184
+ }
1185
+ function readRangeEntry(entry, signal, producers) {
1186
+ const stream = new TransformStream();
1187
+ const producer = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(
1188
+ () => void 0,
1189
+ async (error) => {
1190
+ try {
1191
+ await stream.writable.abort(error);
1192
+ } catch {
1193
+ }
1194
+ }
1195
+ ).finally(() => {
1196
+ producers.delete(producer);
1197
+ });
1198
+ producers.add(producer);
1199
+ return stream.readable;
1200
+ }
1201
+ async function writeStream(storage, params) {
1202
+ if (storage.writeObjectStream) {
1203
+ try {
1204
+ await storage.writeObjectStream(params);
1205
+ } catch (error) {
1206
+ await params.body.cancel(error).catch(() => void 0);
1207
+ throw error;
1208
+ }
1209
+ return;
1210
+ }
1211
+ const parts = [];
1212
+ for await (const part of params.body) {
1213
+ params.signal?.throwIfAborted();
1214
+ parts.push(part);
1215
+ }
1216
+ await storage.writeObject({ ...params, body: Buffer.concat(parts) });
1217
+ }
1218
+ async function validateRequiredObjects(params, paths) {
1219
+ const required = /* @__PURE__ */ new Set(["manifest.json", "chunks.json"]);
1220
+ for (const path of required)
1221
+ if (!paths.has(path)) throw new Error(`Required parsed result file missing: ${path}`);
1222
+ const manifestObject = await params.storage.readObject({ ...params, path: "manifest.json" });
1223
+ const chunksObject = await params.storage.readObject({ ...params, path: "chunks.json" });
1224
+ if (!manifestObject || !chunksObject)
1225
+ throw new Error("Required parsed result manifest.json or chunks.json was not persisted.");
1226
+ const manifest = JSON.parse(Buffer.from(manifestObject.body).toString("utf8"));
1227
+ const payload = JSON.parse(Buffer.from(chunksObject.body).toString("utf8"));
1228
+ if (!isRecord(manifest)) throw new Error("Invalid parsed result manifest.json.");
1229
+ const manifestJobId = manifest.job_id ?? manifest.jobId;
1230
+ if (manifestJobId !== params.jobId)
1231
+ throw new Error("Parsed result manifest job identity differs from the pinned job.");
1232
+ const chunks = Array.isArray(payload) ? payload : isRecord(payload) ? payload.chunks : void 0;
1233
+ if (!Array.isArray(chunks)) throw new Error("Invalid parsed result chunks.json.");
1234
+ if (isRecord(manifest.files)) {
1235
+ for (const path of Object.values(manifest.files))
1236
+ if (typeof path === "string") required.add(normalizeResultPath(path));
1237
+ }
1238
+ for (const chunk of chunks) {
1239
+ if (!isRecord(chunk)) throw new Error("Invalid parsed result chunk.");
1240
+ const metadata = chunk.metadata;
1241
+ const chunkType = chunk.type ?? chunk.chunk_type ?? chunk.chunkType;
1242
+ const filePath = chunk.file_path ?? chunk.filePath ?? (isRecord(metadata) ? metadata.file_path ?? metadata.filePath : void 0) ?? (chunkType === "image" || chunkType === "table" ? chunk.path : void 0);
1243
+ if (typeof filePath === "string") required.add(normalizeResultPath(filePath));
1244
+ if (!isRecord(metadata)) continue;
1245
+ const pageAssets = metadata.page_assets ?? metadata.pageAssets;
1246
+ if (!Array.isArray(pageAssets)) continue;
1247
+ for (const asset of pageAssets)
1248
+ if (isRecord(asset)) {
1249
+ const ref = asset.artifact_ref ?? asset.artifactRef;
1250
+ if (typeof ref === "string") required.add(normalizeResultPath(ref));
1251
+ }
1252
+ }
1253
+ for (const path of required)
1254
+ if (!paths.has(path)) throw new Error(`Required parsed result file missing: ${path}`);
1255
+ }
1256
+ async function retainArchive(params, expectedIdentity) {
1257
+ const response = await fetch(params.resultUrl, { signal: params.signal });
1258
+ if (!response.ok || !response.body)
1259
+ throw new Error(`Optional archive download failed (HTTP ${response.status}).`);
1260
+ if (createDownloadIdentity(params.jobId, response) !== expectedIdentity) {
1261
+ await response.body.cancel();
1262
+ throw new Error("Optional result archive identity changed.");
1263
+ }
1264
+ await writeStream(params.storage, {
1265
+ documentId: params.documentId,
1266
+ revisionKey: params.revisionKey,
1267
+ path: "result.zip",
1268
+ body: response.body,
1269
+ contentType: "application/zip",
1270
+ signal: params.signal
1271
+ });
1272
+ }
1273
+ function isSnapshotPath(path) {
1274
+ const normalized = normalizeResultPath(path.replace(/\/$/, ""));
1275
+ return normalized !== "result.zip" && normalized !== ".knowhere-sdk" && !normalized.startsWith(".knowhere-sdk/");
1276
+ }
1277
+ function isRecord(value) {
1278
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1279
+ }
1280
+ function createDownloadIdentity(jobId, response) {
1281
+ return createHash2("sha256").update(
1282
+ JSON.stringify([
1283
+ jobId,
1284
+ response.headers.get("etag"),
1285
+ response.headers.get("last-modified"),
1286
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
1287
+ ])
1288
+ ).digest("hex");
1289
+ }
1290
+
1291
+ export {
1292
+ DEFAULT_BASE_URL,
1293
+ DEFAULT_TIMEOUT,
1294
+ DEFAULT_MAX_RETRIES,
1295
+ DEFAULT_POLL_INTERVAL,
1296
+ MAX_POLL_INTERVAL,
1297
+ DEFAULT_POLL_TIMEOUT,
1298
+ POLL_INTERVAL_INCREASE_THRESHOLD,
1299
+ POLL_INTERVAL_MULTIPLIER,
1300
+ INITIAL_RETRY_DELAY,
1301
+ MAX_RETRY_DELAY,
1302
+ RETRY_DELAY_BASE,
1303
+ RETRYABLE_STATUS_CODES,
1304
+ ENV,
1305
+ KnowhereError,
1306
+ NetworkError,
1307
+ TimeoutError,
1308
+ PollingTimeoutError,
1309
+ ChecksumError,
1310
+ ValidationError,
1311
+ InvalidStateError,
1312
+ APIError,
1313
+ BadRequestError,
1314
+ AuthenticationError,
1315
+ PaymentRequiredError,
1316
+ PermissionDeniedError,
1317
+ NotFoundError,
1318
+ ConflictError,
1319
+ RateLimitError,
1320
+ InternalServerError,
1321
+ ServiceUnavailableError,
1322
+ GatewayTimeoutError,
1323
+ createAPIError,
1324
+ JobFailedError,
1325
+ ParsedDocumentArchiveError,
1326
+ sleep,
1327
+ keysToCamel,
1328
+ keysToSnake,
1329
+ parseDates,
1330
+ enrichJobResult,
1331
+ enrichParseResult,
1332
+ sanitizePath,
1333
+ getFileExtension,
1334
+ jitter,
1335
+ syncParseResultToParsedDocumentStorage,
1336
+ readParsedDocumentCommit,
1337
+ readJsonObject,
1338
+ normalizeResultPath,
1339
+ syncRemoteResultArchive,
1340
+ validateArchiveSyncLimits,
1341
+ isSnapshotPath,
1342
+ createDownloadIdentity
1343
+ };