@ontos-ai/knowhere-sdk 2.4.2 → 2.6.0-beta.20261026172248

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -5,6 +5,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
5
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
6
  var __getProtoOf = Object.getPrototypeOf;
7
7
  var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __esm = (fn, res) => function __init() {
9
+ return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
10
+ };
8
11
  var __export = (target, all) => {
9
12
  for (var name in all)
10
13
  __defProp(target, name, { get: all[name], enumerable: true });
@@ -27,205 +30,92 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
27
30
  ));
28
31
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
32
 
30
- // src/index.ts
31
- var index_exports = {};
32
- __export(index_exports, {
33
- APIError: () => APIError,
34
- AuthenticationError: () => AuthenticationError,
35
- BadRequestError: () => BadRequestError,
36
- ChecksumError: () => ChecksumError,
37
- ConflictError: () => ConflictError,
38
- DiskParsedDocumentStorage: () => DiskParsedDocumentStorage,
39
- Documents: () => Documents,
40
- GatewayTimeoutError: () => GatewayTimeoutError,
41
- InternalServerError: () => InternalServerError,
42
- InvalidStateError: () => InvalidStateError,
43
- JobFailedError: () => JobFailedError,
44
- Jobs: () => Jobs,
45
- Knowhere: () => Knowhere,
46
- KnowhereError: () => KnowhereError,
47
- Knowledge: () => Knowledge,
48
- LocalKnowledgeStore: () => LocalKnowledgeStore,
49
- NODE_SDK_DOCUMENT_METADATA_DEFAULTS: () => NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
50
- NetworkError: () => NetworkError,
51
- NotFoundError: () => NotFoundError,
52
- PaymentRequiredError: () => PaymentRequiredError,
53
- PermissionDeniedError: () => PermissionDeniedError,
54
- PollingTimeoutError: () => PollingTimeoutError,
55
- RateLimitError: () => RateLimitError,
56
- Retrieval: () => Retrieval,
57
- ServiceUnavailableError: () => ServiceUnavailableError,
58
- TimeoutError: () => TimeoutError,
59
- VERSION: () => VERSION,
60
- ValidationError: () => ValidationError,
61
- default: () => Knowhere,
62
- mergeDocumentMetadataDefaults: () => mergeDocumentMetadataDefaults,
63
- storeParseResultAssets: () => storeParseResultAssets,
64
- syncParseResultToParsedDocumentStorage: () => syncParseResultToParsedDocumentStorage
65
- });
66
- module.exports = __toCommonJS(index_exports);
67
-
68
- // src/client.ts
69
- var import_path3 = __toESM(require("path"));
70
-
71
- // src/lib/http-client.ts
72
- var import_axios = __toESM(require("axios"));
73
-
74
- // src/version.ts
75
- var VERSION = "2.1.3";
76
-
77
33
  // src/constants.ts
78
- var DEFAULT_BASE_URL = "https://api.knowhereto.ai";
79
- var DEFAULT_TIMEOUT = 6e4;
80
- var DEFAULT_MAX_RETRIES = 5;
81
- var DEFAULT_POLL_INTERVAL = 1e4;
82
- var MAX_POLL_INTERVAL = 3e4;
83
- var DEFAULT_POLL_TIMEOUT = 18e5;
84
- var POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
85
- var POLL_INTERVAL_MULTIPLIER = 1.2;
86
- var INITIAL_RETRY_DELAY = 500;
87
- var MAX_RETRY_DELAY = 3e4;
88
- var RETRY_DELAY_BASE = 2;
89
- var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
90
- var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
91
- var ENV = {
92
- API_KEY: "KNOWHERE_API_KEY",
93
- BASE_URL: "KNOWHERE_BASE_URL",
94
- LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
95
- };
34
+ var DEFAULT_BASE_URL, DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES, DEFAULT_POLL_INTERVAL, MAX_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT, POLL_INTERVAL_INCREASE_THRESHOLD, POLL_INTERVAL_MULTIPLIER, INITIAL_RETRY_DELAY, MAX_RETRY_DELAY, RETRY_DELAY_BASE, RETRYABLE_STATUS_CODES, TERMINAL_JOB_STATUSES, ENV;
35
+ var init_constants = __esm({
36
+ "src/constants.ts"() {
37
+ "use strict";
38
+ DEFAULT_BASE_URL = "https://api.knowhereto.ai";
39
+ DEFAULT_TIMEOUT = 6e4;
40
+ DEFAULT_MAX_RETRIES = 5;
41
+ DEFAULT_POLL_INTERVAL = 1e4;
42
+ MAX_POLL_INTERVAL = 3e4;
43
+ DEFAULT_POLL_TIMEOUT = 18e5;
44
+ POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
45
+ POLL_INTERVAL_MULTIPLIER = 1.2;
46
+ INITIAL_RETRY_DELAY = 500;
47
+ MAX_RETRY_DELAY = 3e4;
48
+ RETRY_DELAY_BASE = 2;
49
+ RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
50
+ TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
51
+ ENV = {
52
+ API_KEY: "KNOWHERE_API_KEY",
53
+ BASE_URL: "KNOWHERE_BASE_URL",
54
+ LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
55
+ };
56
+ }
57
+ });
96
58
 
97
59
  // src/errors/base.ts
98
- var KnowhereError = class extends Error {
99
- constructor(message) {
100
- super(message);
101
- this.name = "KnowhereError";
102
- Object.setPrototypeOf(this, new.target.prototype);
103
- }
104
- };
105
- var NetworkError = class extends KnowhereError {
106
- constructor(message, cause) {
107
- super(message);
108
- this.cause = cause;
109
- this.name = "NetworkError";
110
- }
111
- cause;
112
- };
113
- var TimeoutError = class extends NetworkError {
114
- constructor(message = "Request timed out") {
115
- super(message);
116
- this.name = "TimeoutError";
117
- }
118
- };
119
- var PollingTimeoutError = class extends KnowhereError {
120
- constructor(message = "Polling timed out", elapsedMs) {
121
- super(message);
122
- this.elapsedMs = elapsedMs;
123
- this.name = "PollingTimeoutError";
124
- }
125
- elapsedMs;
126
- };
127
- var ChecksumError = class extends KnowhereError {
128
- constructor(message = "Checksum verification failed", expected, actual) {
129
- super(message);
130
- this.expected = expected;
131
- this.actual = actual;
132
- this.name = "ChecksumError";
133
- }
134
- expected;
135
- actual;
136
- };
137
- var ValidationError = class extends KnowhereError {
138
- constructor(message) {
139
- super(message);
140
- this.name = "ValidationError";
141
- }
142
- };
143
- var InvalidStateError = class extends KnowhereError {
144
- constructor(message) {
145
- super(message);
146
- this.name = "InvalidStateError";
60
+ var KnowhereError, NetworkError, TimeoutError, PollingTimeoutError, ChecksumError, ValidationError, InvalidStateError;
61
+ var init_base = __esm({
62
+ "src/errors/base.ts"() {
63
+ "use strict";
64
+ KnowhereError = class extends Error {
65
+ constructor(message) {
66
+ super(message);
67
+ this.name = "KnowhereError";
68
+ Object.setPrototypeOf(this, new.target.prototype);
69
+ }
70
+ };
71
+ NetworkError = class extends KnowhereError {
72
+ constructor(message, cause) {
73
+ super(message);
74
+ this.cause = cause;
75
+ this.name = "NetworkError";
76
+ }
77
+ cause;
78
+ };
79
+ TimeoutError = class extends NetworkError {
80
+ constructor(message = "Request timed out") {
81
+ super(message);
82
+ this.name = "TimeoutError";
83
+ }
84
+ };
85
+ PollingTimeoutError = class extends KnowhereError {
86
+ constructor(message = "Polling timed out", elapsedMs) {
87
+ super(message);
88
+ this.elapsedMs = elapsedMs;
89
+ this.name = "PollingTimeoutError";
90
+ }
91
+ elapsedMs;
92
+ };
93
+ ChecksumError = class extends KnowhereError {
94
+ constructor(message = "Checksum verification failed", expected, actual) {
95
+ super(message);
96
+ this.expected = expected;
97
+ this.actual = actual;
98
+ this.name = "ChecksumError";
99
+ }
100
+ expected;
101
+ actual;
102
+ };
103
+ ValidationError = class extends KnowhereError {
104
+ constructor(message) {
105
+ super(message);
106
+ this.name = "ValidationError";
107
+ }
108
+ };
109
+ InvalidStateError = class extends KnowhereError {
110
+ constructor(message) {
111
+ super(message);
112
+ this.name = "InvalidStateError";
113
+ }
114
+ };
147
115
  }
148
- };
116
+ });
149
117
 
150
118
  // src/errors/api-errors.ts
151
- var APIError = class extends KnowhereError {
152
- constructor(message, statusCode, code, requestId, details, body) {
153
- super(message);
154
- this.statusCode = statusCode;
155
- this.code = code;
156
- this.requestId = requestId;
157
- this.details = details;
158
- this.body = body;
159
- this.name = "APIError";
160
- }
161
- statusCode;
162
- code;
163
- requestId;
164
- details;
165
- body;
166
- };
167
- var BadRequestError = class extends APIError {
168
- constructor(message, code, requestId, details, body) {
169
- super(message, 400, code, requestId, details, body);
170
- this.name = "BadRequestError";
171
- }
172
- };
173
- var AuthenticationError = class extends APIError {
174
- constructor(message = "Authentication failed", code, requestId, details, body) {
175
- super(message, 401, code, requestId, details, body);
176
- this.name = "AuthenticationError";
177
- }
178
- };
179
- var PaymentRequiredError = class extends APIError {
180
- constructor(message = "Payment required", code, requestId, details, body) {
181
- super(message, 402, code, requestId, details, body);
182
- this.name = "PaymentRequiredError";
183
- }
184
- };
185
- var PermissionDeniedError = class extends APIError {
186
- constructor(message = "Permission denied", code, requestId, details, body) {
187
- super(message, 403, code, requestId, details, body);
188
- this.name = "PermissionDeniedError";
189
- }
190
- };
191
- var NotFoundError = class extends APIError {
192
- constructor(message = "Resource not found", code, requestId, details, body) {
193
- super(message, 404, code, requestId, details, body);
194
- this.name = "NotFoundError";
195
- }
196
- };
197
- var ConflictError = class extends APIError {
198
- constructor(message = "Conflict", code, requestId, details, body) {
199
- super(message, 409, code, requestId, details, body);
200
- this.name = "ConflictError";
201
- }
202
- };
203
- var RateLimitError = class extends APIError {
204
- constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
205
- super(message, 429, code, requestId, details, body);
206
- this.retryAfter = retryAfter;
207
- this.name = "RateLimitError";
208
- }
209
- retryAfter;
210
- };
211
- var InternalServerError = class extends APIError {
212
- constructor(message = "Internal server error", code, requestId, details, body) {
213
- super(message, 500, code, requestId, details, body);
214
- this.name = "InternalServerError";
215
- }
216
- };
217
- var ServiceUnavailableError = class extends APIError {
218
- constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
219
- super(message, statusCode, code, requestId, details, body);
220
- this.name = "ServiceUnavailableError";
221
- }
222
- };
223
- var GatewayTimeoutError = class extends APIError {
224
- constructor(message = "Gateway timeout", code, requestId, details, body) {
225
- super(message, 504, code, requestId, details, body);
226
- this.name = "GatewayTimeoutError";
227
- }
228
- };
229
119
  function createAPIError(statusCode, message, code, requestId, details, body, retryAfter) {
230
120
  switch (statusCode) {
231
121
  case 400:
@@ -253,18 +143,141 @@ function createAPIError(statusCode, message, code, requestId, details, body, ret
253
143
  return new APIError(message, statusCode, code, requestId, details, body);
254
144
  }
255
145
  }
146
+ var APIError, BadRequestError, AuthenticationError, PaymentRequiredError, PermissionDeniedError, NotFoundError, ConflictError, RateLimitError, InternalServerError, ServiceUnavailableError, GatewayTimeoutError;
147
+ var init_api_errors = __esm({
148
+ "src/errors/api-errors.ts"() {
149
+ "use strict";
150
+ init_base();
151
+ APIError = class extends KnowhereError {
152
+ constructor(message, statusCode, code, requestId, details, body) {
153
+ super(message);
154
+ this.statusCode = statusCode;
155
+ this.code = code;
156
+ this.requestId = requestId;
157
+ this.details = details;
158
+ this.body = body;
159
+ this.name = "APIError";
160
+ }
161
+ statusCode;
162
+ code;
163
+ requestId;
164
+ details;
165
+ body;
166
+ };
167
+ BadRequestError = class extends APIError {
168
+ constructor(message, code, requestId, details, body) {
169
+ super(message, 400, code, requestId, details, body);
170
+ this.name = "BadRequestError";
171
+ }
172
+ };
173
+ AuthenticationError = class extends APIError {
174
+ constructor(message = "Authentication failed", code, requestId, details, body) {
175
+ super(message, 401, code, requestId, details, body);
176
+ this.name = "AuthenticationError";
177
+ }
178
+ };
179
+ PaymentRequiredError = class extends APIError {
180
+ constructor(message = "Payment required", code, requestId, details, body) {
181
+ super(message, 402, code, requestId, details, body);
182
+ this.name = "PaymentRequiredError";
183
+ }
184
+ };
185
+ PermissionDeniedError = class extends APIError {
186
+ constructor(message = "Permission denied", code, requestId, details, body) {
187
+ super(message, 403, code, requestId, details, body);
188
+ this.name = "PermissionDeniedError";
189
+ }
190
+ };
191
+ NotFoundError = class extends APIError {
192
+ constructor(message = "Resource not found", code, requestId, details, body) {
193
+ super(message, 404, code, requestId, details, body);
194
+ this.name = "NotFoundError";
195
+ }
196
+ };
197
+ ConflictError = class extends APIError {
198
+ constructor(message = "Conflict", code, requestId, details, body) {
199
+ super(message, 409, code, requestId, details, body);
200
+ this.name = "ConflictError";
201
+ }
202
+ };
203
+ RateLimitError = class extends APIError {
204
+ constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
205
+ super(message, 429, code, requestId, details, body);
206
+ this.retryAfter = retryAfter;
207
+ this.name = "RateLimitError";
208
+ }
209
+ retryAfter;
210
+ };
211
+ InternalServerError = class extends APIError {
212
+ constructor(message = "Internal server error", code, requestId, details, body) {
213
+ super(message, 500, code, requestId, details, body);
214
+ this.name = "InternalServerError";
215
+ }
216
+ };
217
+ ServiceUnavailableError = class extends APIError {
218
+ constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
219
+ super(message, statusCode, code, requestId, details, body);
220
+ this.name = "ServiceUnavailableError";
221
+ }
222
+ };
223
+ GatewayTimeoutError = class extends APIError {
224
+ constructor(message = "Gateway timeout", code, requestId, details, body) {
225
+ super(message, 504, code, requestId, details, body);
226
+ this.name = "GatewayTimeoutError";
227
+ }
228
+ };
229
+ }
230
+ });
256
231
 
257
232
  // src/errors/job-errors.ts
258
- var JobFailedError = class extends KnowhereError {
259
- constructor(message, code, jobResult) {
260
- super(message);
261
- this.code = code;
262
- this.jobResult = jobResult;
263
- this.name = "JobFailedError";
264
- }
265
- code;
266
- jobResult;
267
- };
233
+ var JobFailedError;
234
+ var init_job_errors = __esm({
235
+ "src/errors/job-errors.ts"() {
236
+ "use strict";
237
+ init_base();
238
+ JobFailedError = class extends KnowhereError {
239
+ constructor(message, code, jobResult) {
240
+ super(message);
241
+ this.code = code;
242
+ this.jobResult = jobResult;
243
+ this.name = "JobFailedError";
244
+ }
245
+ code;
246
+ jobResult;
247
+ };
248
+ }
249
+ });
250
+
251
+ // src/errors/parsed-document-archive-error.ts
252
+ var ParsedDocumentArchiveError;
253
+ var init_parsed_document_archive_error = __esm({
254
+ "src/errors/parsed-document-archive-error.ts"() {
255
+ "use strict";
256
+ init_base();
257
+ ParsedDocumentArchiveError = class extends KnowhereError {
258
+ code = "ARCHIVE_SPOOL_FAILED";
259
+ cause;
260
+ constructor(cause) {
261
+ super(
262
+ "Parsed result ZIP temporary storage is unavailable. Range-enabled object storage avoids temporary disk usage."
263
+ );
264
+ this.name = "ParsedDocumentArchiveError";
265
+ this.cause = cause;
266
+ }
267
+ };
268
+ }
269
+ });
270
+
271
+ // src/errors/index.ts
272
+ var init_errors = __esm({
273
+ "src/errors/index.ts"() {
274
+ "use strict";
275
+ init_base();
276
+ init_api_errors();
277
+ init_job_errors();
278
+ init_parsed_document_archive_error();
279
+ }
280
+ });
268
281
 
269
282
  // src/lib/utils.ts
270
283
  function sleep(ms) {
@@ -388,8 +401,1204 @@ function jitter(value, percent = 0.2) {
388
401
  const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
389
402
  return Math.round(value * randomFactor);
390
403
  }
404
+ var init_utils = __esm({
405
+ "src/lib/utils.ts"() {
406
+ "use strict";
407
+ init_constants();
408
+ }
409
+ });
410
+
411
+ // src/storage/parsed-document-storage.ts
412
+ async function syncParseResultToParsedDocumentStorage(params) {
413
+ const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
414
+ const revisionKey = params.revisionKey ?? params.result.jobId;
415
+ for await (const object of collectResultLayoutObjects(params.result)) {
416
+ params.signal?.throwIfAborted();
417
+ await params.storage.writeObject({
418
+ documentId,
419
+ revisionKey,
420
+ ...object
421
+ });
422
+ }
423
+ await params.beforeCommit?.();
424
+ params.signal?.throwIfAborted();
425
+ const commit = {
426
+ version: 1,
427
+ documentId,
428
+ revisionKey,
429
+ source: params.source ?? "resultZip",
430
+ committedAt: (/* @__PURE__ */ new Date()).toISOString()
431
+ };
432
+ await writeParsedDocumentJsonObject(params.storage, {
433
+ documentId,
434
+ revisionKey,
435
+ path: parsedDocumentCommitPath,
436
+ value: commit
437
+ });
438
+ return {
439
+ result: params.result,
440
+ documentId,
441
+ revisionKey,
442
+ commit
443
+ };
444
+ }
445
+ async function readParsedDocumentCommit(params) {
446
+ const object = await params.storage.readObject({
447
+ documentId: params.documentId,
448
+ revisionKey: params.revisionKey,
449
+ path: parsedDocumentCommitPath
450
+ });
451
+ if (!object) {
452
+ return null;
453
+ }
454
+ const commit = readJsonObject(object);
455
+ return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
456
+ }
457
+ function readJsonObject(object) {
458
+ return JSON.parse(Buffer.from(object.body).toString("utf8"));
459
+ }
460
+ async function* collectResultLayoutObjects(result) {
461
+ const storedPaths = /* @__PURE__ */ new Set();
462
+ let zip;
463
+ try {
464
+ zip = await import_jszip3.default.loadAsync(result.rawZip);
465
+ } catch {
466
+ }
467
+ if (zip) {
468
+ for (const entry of Object.values(zip.files)) {
469
+ if (entry.dir || entry.name === "result.zip") continue;
470
+ const path4 = normalizeResultPath(entry.name);
471
+ if (isSdkMetadataPath(path4)) continue;
472
+ storedPaths.add(path4);
473
+ yield {
474
+ path: path4,
475
+ body: await entry.async("nodebuffer"),
476
+ contentType: inferContentTypeFromPath(path4)
477
+ };
478
+ }
479
+ }
480
+ for (const object of generateMissingResultObjects(result, storedPaths)) yield object;
481
+ }
482
+ function* generateMissingResultObjects(result, storedPaths) {
483
+ if (!storedPaths.has("manifest.json")) yield createJsonObject("manifest.json", result.manifest);
484
+ if (!storedPaths.has("chunks.json"))
485
+ yield createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)));
486
+ if (!storedPaths.has("doc_nav.json") && result.docNav)
487
+ yield createJsonObject("doc_nav.json", result.docNav);
488
+ if (!storedPaths.has("full.md") && result.fullMarkdown)
489
+ yield createTextObject("full.md", result.fullMarkdown, markdownContentType);
490
+ if (!storedPaths.has("chunks_slim.json") && result.chunksSlim)
491
+ yield createJsonObject("chunks_slim.json", { chunks: result.chunksSlim });
492
+ if (!storedPaths.has("hierarchy.json") && result.hierarchy)
493
+ yield createJsonObject("hierarchy.json", result.hierarchy);
494
+ if (!storedPaths.has("toc_hierarchies.json") && result.tocHierarchies)
495
+ yield createJsonObject("toc_hierarchies.json", result.tocHierarchies);
496
+ if (!storedPaths.has("kb.csv") && result.kbCsv)
497
+ yield createTextObject("kb.csv", result.kbCsv, csvContentType);
498
+ if (!storedPaths.has("hierarchy_view.html") && result.hierarchyViewHtml)
499
+ yield createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType);
500
+ for (const chunk of result.imageChunks) {
501
+ const path4 = normalizeResultPath(chunk.filePath);
502
+ if (storedPaths.has(path4)) continue;
503
+ yield {
504
+ path: path4,
505
+ body: chunk.data,
506
+ contentType: inferContentTypeFromPath(path4),
507
+ metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
508
+ };
509
+ }
510
+ for (const chunk of result.tableChunks) {
511
+ const path4 = normalizeResultPath(chunk.filePath);
512
+ if (storedPaths.has(path4)) continue;
513
+ yield {
514
+ path: path4,
515
+ body: Buffer.from(chunk.html, "utf8"),
516
+ contentType: htmlContentType,
517
+ metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
518
+ };
519
+ }
520
+ }
521
+ function serializeChunks2(chunks) {
522
+ return {
523
+ chunks: chunks.map((chunk) => {
524
+ const rawChunk = {
525
+ chunkId: chunk.chunkId,
526
+ type: chunk.type,
527
+ contentSource: chunk.contentSource,
528
+ content: chunk.content,
529
+ path: chunk.path,
530
+ metadata: chunk.metadata
531
+ };
532
+ if (chunk.type === "image" || chunk.type === "table") {
533
+ return {
534
+ ...rawChunk,
535
+ filePath: chunk.filePath,
536
+ assetUrl: chunk.assetUrl
537
+ };
538
+ }
539
+ return rawChunk;
540
+ })
541
+ };
542
+ }
543
+ async function writeParsedDocumentJsonObject(storage, params) {
544
+ await storage.writeObject({
545
+ documentId: params.documentId,
546
+ revisionKey: params.revisionKey,
547
+ path: params.path,
548
+ body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
549
+ contentType: jsonContentType
550
+ });
551
+ }
552
+ function createJsonObject(path4, value) {
553
+ return {
554
+ path: path4,
555
+ body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
556
+ contentType: jsonContentType
557
+ };
558
+ }
559
+ function createTextObject(path4, value, contentType) {
560
+ return {
561
+ path: path4,
562
+ body: Buffer.from(value, "utf8"),
563
+ contentType
564
+ };
565
+ }
566
+ function inferContentTypeFromPath(filePath) {
567
+ const lower = filePath.toLowerCase();
568
+ if (lower.endsWith(".json")) return jsonContentType;
569
+ if (lower.endsWith(".md")) return markdownContentType;
570
+ if (lower.endsWith(".csv")) return csvContentType;
571
+ if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
572
+ if (lower.endsWith(".png")) return "image/png";
573
+ if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
574
+ if (lower.endsWith(".gif")) return "image/gif";
575
+ if (lower.endsWith(".webp")) return "image/webp";
576
+ if (lower.endsWith(".svg")) return "image/svg+xml";
577
+ return binaryContentType;
578
+ }
579
+ function normalizeResultPath(value) {
580
+ const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
581
+ if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
582
+ throw new Error(`Invalid parsed result path: ${value}`);
583
+ }
584
+ return normalized;
585
+ }
586
+ function isSdkMetadataPath(path4) {
587
+ return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
588
+ }
589
+ var import_jszip3, parsedDocumentCommitPath, jsonContentType, markdownContentType, csvContentType, htmlContentType, binaryContentType;
590
+ var init_parsed_document_storage = __esm({
591
+ "src/storage/parsed-document-storage.ts"() {
592
+ "use strict";
593
+ import_jszip3 = __toESM(require("jszip"));
594
+ init_utils();
595
+ parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
596
+ jsonContentType = "application/json; charset=utf-8";
597
+ markdownContentType = "text/markdown; charset=utf-8";
598
+ csvContentType = "text/csv; charset=utf-8";
599
+ htmlContentType = "text/html; charset=utf-8";
600
+ binaryContentType = "application/octet-stream";
601
+ }
602
+ });
603
+
604
+ // src/storage/archive-cache-transport-error.ts
605
+ var ArchiveCacheTransportError;
606
+ var init_archive_cache_transport_error = __esm({
607
+ "src/storage/archive-cache-transport-error.ts"() {
608
+ "use strict";
609
+ ArchiveCacheTransportError = class extends Error {
610
+ constructor(message) {
611
+ super(message);
612
+ this.name = "ArchiveCacheTransportError";
613
+ }
614
+ };
615
+ }
616
+ });
617
+
618
+ // src/storage/parsed-document-archive-cache.ts
619
+ async function prepareCachedArchive(params) {
620
+ params.signal?.throwIfAborted();
621
+ if (params.archiveCacheBypassed) return createFallback(params);
622
+ if (!params.storage.writeObjectStream || !params.storage.headObject || !params.storage.getObjectUrl) {
623
+ await params.onBypass?.();
624
+ return createFallback(params);
625
+ }
626
+ const controller = new AbortController();
627
+ const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
628
+ const mutations = /* @__PURE__ */ new Set();
629
+ let rejectCancellation;
630
+ const cancellation = new Promise((_resolve, reject) => {
631
+ rejectCancellation = () => {
632
+ const reason = signal.reason;
633
+ reject(
634
+ reason instanceof Error ? reason : new Error("Parsed result archive cache was cancelled.", { cause: reason })
635
+ );
636
+ };
637
+ signal.addEventListener("abort", rejectCancellation, { once: true });
638
+ });
639
+ const timer = setTimeout(() => {
640
+ controller.abort(new Error("Parsed result archive cache exceeded its thirty-second budget."));
641
+ }, ARCHIVE_CACHE_TIMEOUT_MS);
642
+ let origin;
643
+ const preparation = (async () => {
644
+ origin = await probeArchiveOrigin({ ...params, signal });
645
+ signal.throwIfAborted();
646
+ if (!hasStrongEntityTag(origin.entityTag) || origin.contentLength === void 0)
647
+ throw new Error("Parsed result archive cache requires a strong origin version and length.");
648
+ return prepareArchiveCache(params, origin, signal, mutations);
649
+ })();
650
+ try {
651
+ return await Promise.race([preparation, cancellation]);
652
+ } catch (error) {
653
+ controller.abort(error);
654
+ await Promise.allSettled([...mutations]);
655
+ params.signal?.throwIfAborted();
656
+ if (error instanceof ArchiveOriginIntegrityError) throw error;
657
+ await params.onBypass?.();
658
+ return createFallback(params, origin);
659
+ } finally {
660
+ clearTimeout(timer);
661
+ if (rejectCancellation) signal.removeEventListener("abort", rejectCancellation);
662
+ }
663
+ }
664
+ function createFallback(params, origin) {
665
+ return {
666
+ resultUrl: params.resultUrl,
667
+ downloadIdentity: origin?.identity,
668
+ archiveCacheBypassed: true,
669
+ isCached: false,
670
+ validateResponse(response) {
671
+ if (origin && createOriginIdentity(params.jobId, response) !== origin.identity)
672
+ rejectResponse(
673
+ response,
674
+ new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.")
675
+ );
676
+ return response;
677
+ }
678
+ };
679
+ }
680
+ async function probeArchiveOrigin(params) {
681
+ const response = await fetch(params.resultUrl, {
682
+ headers: { Range: "bytes=0-0" },
683
+ signal: params.signal
684
+ });
685
+ try {
686
+ if (!response.ok) throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
687
+ return {
688
+ identity: createOriginIdentity(params.jobId, response),
689
+ contentLength: readOriginLength(response),
690
+ entityTag: response.headers.get("etag")
691
+ };
692
+ } finally {
693
+ await response.body?.cancel();
694
+ }
695
+ }
696
+ async function prepareArchiveCache(params, origin, signal, mutations) {
697
+ const object = { ...params, path: "result.zip" };
698
+ const initialHead = await params.storage.headObject(object);
699
+ signal.throwIfAborted();
700
+ let marker = null;
701
+ if (initialHead) {
702
+ const version = readObjectVersion(initialHead);
703
+ const stored = await params.storage.readObject({
704
+ ...object,
705
+ path: createMarkerPath(origin.identity, version.entityTag)
706
+ });
707
+ signal.throwIfAborted();
708
+ if (stored) marker = readMarker(stored, params, origin, version);
709
+ }
710
+ if (!marker) marker = await storeArchive(params, origin, signal, mutations);
711
+ signal.throwIfAborted();
712
+ const resultUrl = await params.storage.getObjectUrl(object);
713
+ signal.throwIfAborted();
714
+ if (!resultUrl) throw new Error("Parsed result archive cache has no readable object URL.");
715
+ const validateResponse = (response2) => validateCachedResponse(response2, marker, params.signal);
716
+ const response = await fetch(resultUrl, {
717
+ headers: { Range: "bytes=0-0", "If-Match": marker.storageEntityTag },
718
+ signal
719
+ });
720
+ try {
721
+ if (response.headers.get("content-range") !== `bytes 0-0/${marker.contentLength}` || response.headers.get("content-length") !== "1")
722
+ rejectResponse(
723
+ response,
724
+ new ArchiveCacheTransportError("Parsed result archive cache probe exceeded one byte.")
725
+ );
726
+ const validated = validateResponse(response);
727
+ await validated.arrayBuffer();
728
+ } catch (error) {
729
+ await response.body?.cancel().catch(() => void 0);
730
+ throw error;
731
+ }
732
+ signal.throwIfAborted();
733
+ return {
734
+ resultUrl,
735
+ downloadIdentity: origin.identity,
736
+ archiveCacheBypassed: false,
737
+ isCached: true,
738
+ validateResponse
739
+ };
740
+ }
741
+ async function storeArchive(params, origin, signal, mutations) {
742
+ const response = await fetch(params.resultUrl, {
743
+ headers: { "If-Match": origin.entityTag },
744
+ signal
745
+ });
746
+ if (signal.aborted) {
747
+ await response.body?.cancel();
748
+ signal.throwIfAborted();
749
+ }
750
+ if (!response.ok || !response.body) {
751
+ await response.body?.cancel();
752
+ if (response.status === 412)
753
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
754
+ throw new Error(`Parsed result archive cache download failed (HTTP ${response.status}).`);
755
+ }
756
+ if (createOriginIdentity(params.jobId, response) !== origin.identity) {
757
+ await response.body.cancel();
758
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
759
+ }
760
+ const hash = (0, import_crypto.createHash)("sha256");
761
+ let contentLength = 0;
762
+ let hasCompleted = false;
763
+ const body = response.body.pipeThrough(
764
+ new TransformStream({
765
+ transform(chunk, controller) {
766
+ signal.throwIfAborted();
767
+ contentLength += chunk.length;
768
+ if (contentLength > origin.contentLength)
769
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
770
+ hash.update(chunk);
771
+ controller.enqueue(chunk);
772
+ },
773
+ flush() {
774
+ if (contentLength !== origin.contentLength)
775
+ throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
776
+ hasCompleted = true;
777
+ }
778
+ })
779
+ );
780
+ const object = { ...params, path: "result.zip" };
781
+ let write;
782
+ try {
783
+ signal.throwIfAborted();
784
+ write = await settleMutation(
785
+ params.storage.writeObjectStream({
786
+ ...object,
787
+ body,
788
+ contentType: "application/zip",
789
+ contentLength: origin.contentLength,
790
+ signal
791
+ }),
792
+ mutations
793
+ );
794
+ } catch (error) {
795
+ await body.cancel(error).catch(() => void 0);
796
+ throw error;
797
+ }
798
+ signal.throwIfAborted();
799
+ if (!hasCompleted)
800
+ throw new Error("Parsed result archive cache sink did not consume the stream.");
801
+ if (!hasStrongEntityTag(write.entityTag))
802
+ throw new Error("Parsed result archive cache requires a strong object version.");
803
+ const head = await params.storage.headObject(object);
804
+ signal.throwIfAborted();
805
+ if (!head) throw new Error("Parsed result archive cache was not persisted.");
806
+ const actualVersion = readObjectVersion(head);
807
+ if (write.entityTag !== actualVersion.entityTag || actualVersion.contentLength !== contentLength)
808
+ throw new Error("Parsed result archive cache object version or length changed during upload.");
809
+ const marker = {
810
+ version: 1,
811
+ jobId: params.jobId,
812
+ revisionKey: params.revisionKey,
813
+ originIdentity: origin.identity,
814
+ contentLength,
815
+ sha256: hash.digest("hex"),
816
+ storageEntityTag: actualVersion.entityTag
817
+ };
818
+ signal.throwIfAborted();
819
+ await settleMutation(
820
+ params.storage.writeObject({
821
+ ...object,
822
+ path: createMarkerPath(origin.identity, actualVersion.entityTag),
823
+ body: Buffer.from(JSON.stringify(marker)),
824
+ contentType: "application/json; charset=utf-8"
825
+ }),
826
+ mutations
827
+ );
828
+ signal.throwIfAborted();
829
+ return marker;
830
+ }
831
+ async function settleMutation(mutation, mutations) {
832
+ mutations.add(mutation);
833
+ try {
834
+ return await mutation;
835
+ } finally {
836
+ mutations.delete(mutation);
837
+ }
838
+ }
839
+ function readMarker(object, params, origin, version) {
840
+ const value = JSON.parse(Buffer.from(object.body).toString("utf8"));
841
+ if (typeof value !== "object" || value === null || !("version" in value) || value.version !== 1 || !("jobId" in value) || value.jobId !== params.jobId || !("revisionKey" in value) || value.revisionKey !== params.revisionKey || !("originIdentity" in value) || value.originIdentity !== origin.identity || !("contentLength" in value) || value.contentLength !== origin.contentLength || value.contentLength !== version.contentLength || !("storageEntityTag" in value) || value.storageEntityTag !== version.entityTag || !("sha256" in value) || typeof value.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(value.sha256))
842
+ throw new Error("Parsed result archive cache completion marker is invalid.");
843
+ return {
844
+ version: 1,
845
+ jobId: params.jobId,
846
+ revisionKey: params.revisionKey,
847
+ originIdentity: origin.identity,
848
+ contentLength: version.contentLength,
849
+ storageEntityTag: version.entityTag,
850
+ sha256: value.sha256
851
+ };
852
+ }
853
+ function readObjectVersion(object) {
854
+ if (!hasStrongEntityTag(object.entityTag) || !isPositiveInteger(object.contentLength))
855
+ throw new Error("Parsed result archive cache requires a strong object version and length.");
856
+ return { entityTag: object.entityTag, contentLength: object.contentLength };
857
+ }
858
+ function validateCachedResponse(response, marker, signal) {
859
+ if (response.headers.get("etag") !== marker.storageEntityTag)
860
+ rejectResponse(
861
+ response,
862
+ new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
863
+ );
864
+ if (response.status !== 206 || !response.body)
865
+ rejectResponse(
866
+ response,
867
+ new ArchiveCacheTransportError("Parsed result archive cache requires a ranged response.")
868
+ );
869
+ const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
870
+ response.headers.get("content-range") ?? ""
871
+ );
872
+ const contentLength = Number(response.headers.get("content-length"));
873
+ if (!range || Number(range[3]) !== marker.contentLength || Number(range[1]) > Number(range[2]) || Number(range[2]) >= marker.contentLength || contentLength !== Number(range[2]) - Number(range[1]) + 1)
874
+ rejectResponse(
875
+ response,
876
+ new ArchiveCacheTransportError("Parsed result archive cache response length changed.")
877
+ );
878
+ let actualLength = 0;
879
+ return new Response(
880
+ response.body.pipeThrough(
881
+ new TransformStream({
882
+ transform(chunk, controller) {
883
+ signal?.throwIfAborted();
884
+ actualLength += chunk.length;
885
+ if (actualLength > contentLength)
886
+ throw new ArchiveCacheTransportError(
887
+ "Parsed result archive cache response length changed."
888
+ );
889
+ controller.enqueue(chunk);
890
+ },
891
+ flush() {
892
+ if (actualLength !== contentLength)
893
+ throw new ArchiveCacheTransportError(
894
+ "Parsed result archive cache response length changed."
895
+ );
896
+ }
897
+ })
898
+ ),
899
+ { status: response.status, statusText: response.statusText, headers: response.headers }
900
+ );
901
+ }
902
+ function rejectResponse(response, error) {
903
+ void response.body?.cancel(error).catch(() => void 0);
904
+ throw error;
905
+ }
906
+ function createMarkerPath(originIdentity, entityTag) {
907
+ const identity = (0, import_crypto.createHash)("sha256").update(JSON.stringify([originIdentity, entityTag])).digest("hex");
908
+ return `.knowhere-sdk/archive-cache/${identity}.json`;
909
+ }
910
+ function createOriginIdentity(jobId, response) {
911
+ return (0, import_crypto.createHash)("sha256").update(
912
+ JSON.stringify([
913
+ jobId,
914
+ response.headers.get("etag"),
915
+ response.headers.get("last-modified"),
916
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
917
+ ])
918
+ ).digest("hex");
919
+ }
920
+ function readOriginLength(response) {
921
+ const value = Number(
922
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
923
+ );
924
+ return isPositiveInteger(value) ? value : void 0;
925
+ }
926
+ function hasStrongEntityTag(value) {
927
+ return typeof value === "string" && value.length > 0 && !value.startsWith("W/");
928
+ }
929
+ function isPositiveInteger(value) {
930
+ return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
931
+ }
932
+ var import_crypto, ARCHIVE_CACHE_TIMEOUT_MS, ArchiveOriginIntegrityError;
933
+ var init_parsed_document_archive_cache = __esm({
934
+ "src/storage/parsed-document-archive-cache.ts"() {
935
+ "use strict";
936
+ import_crypto = require("crypto");
937
+ init_archive_cache_transport_error();
938
+ ARCHIVE_CACHE_TIMEOUT_MS = 3e4;
939
+ ArchiveOriginIntegrityError = class extends Error {
940
+ };
941
+ }
942
+ });
943
+
944
+ // src/storage/sequential-parsed-document-archive.ts
945
+ var sequential_parsed_document_archive_exports = {};
946
+ __export(sequential_parsed_document_archive_exports, {
947
+ openSequentialArchive: () => openSequentialArchive
948
+ });
949
+ async function openSequentialArchive(params, identity, validateResponse) {
950
+ let response = await fetch(params.resultUrl, { signal: params.signal });
951
+ if (!response.ok || !response.body)
952
+ throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
953
+ if (validateResponse) response = validateResponse(response);
954
+ else if (createDownloadIdentity(params.jobId, response) !== identity) {
955
+ await response.body.cancel();
956
+ throw new Error("Parsed result archive identity changed before sequential download.");
957
+ }
958
+ const body = response.body;
959
+ if (!body) throw new Error("Parsed result download returned an empty body.");
960
+ let directory;
961
+ try {
962
+ directory = await (0, import_promises2.mkdtemp)((0, import_path2.join)((0, import_os.tmpdir)(), "knowhere-sync-"));
963
+ } catch (error) {
964
+ await body.cancel();
965
+ throw new ParsedDocumentArchiveError(error);
966
+ }
967
+ let file;
968
+ try {
969
+ file = await (0, import_promises2.open)((0, import_path2.join)(directory, "archive.zip"), "w+");
970
+ } catch (error) {
971
+ await body.cancel();
972
+ await (0, import_promises2.rm)(directory, { recursive: true, force: true });
973
+ throw new ParsedDocumentArchiveError(error);
974
+ }
975
+ const input = import_stream.Readable.fromWeb(body);
976
+ const parser = (0, import_parse.default)({ forceStream: true });
977
+ const checksums = /* @__PURE__ */ new Map();
978
+ let archiveSize = 0;
979
+ const spool = new import_stream.Transform({
980
+ transform(chunk, _encoding, callback) {
981
+ archiveSize += chunk.length;
982
+ void file.writeFile(chunk).then(
983
+ () => {
984
+ callback(null, chunk);
985
+ },
986
+ (error) => {
987
+ callback(new ParsedDocumentArchiveError(error));
988
+ }
989
+ );
990
+ }
991
+ });
992
+ const completed = (0, import_promises.pipeline)(input, spool, parser, { signal: params.signal });
993
+ void completed.catch(() => void 0);
994
+ return {
995
+ identity,
996
+ downloadIdentity: identity,
997
+ canSkipEntries: false,
998
+ entries: (async function* () {
999
+ for await (const value of parser) {
1000
+ const entry = value;
1001
+ if (entry.type === "Directory" || !isSnapshotPath(entry.path)) {
1002
+ await entry.autodrain().promise();
1003
+ continue;
1004
+ }
1005
+ const path4 = normalizeResultPath(entry.path);
1006
+ yield {
1007
+ path: path4,
1008
+ contentLength: !(entry.vars.flags & 8) && "uncompressedSize" in entry.vars && typeof entry.vars.uncompressedSize === "number" ? entry.vars.uncompressedSize : void 0,
1009
+ read: () => {
1010
+ let checksum = 0;
1011
+ let size = 0;
1012
+ return import_stream.Readable.toWeb(entry).pipeThrough(
1013
+ new TransformStream({
1014
+ transform(chunk, controller) {
1015
+ checksum = (0, import_zlib.crc32)(chunk, checksum);
1016
+ size += chunk.length;
1017
+ controller.enqueue(chunk);
1018
+ },
1019
+ flush() {
1020
+ checksums.set(path4, { checksum, size });
1021
+ }
1022
+ })
1023
+ );
1024
+ }
1025
+ };
1026
+ }
1027
+ await completed;
1028
+ const reader = new import_zip.ZipReader(new DiskArchiveReader(file, archiveSize), {
1029
+ useWebWorkers: false
1030
+ });
1031
+ try {
1032
+ const files = (await reader.getEntries()).filter(
1033
+ (entry) => !entry.directory && isSnapshotPath(entry.filename)
1034
+ );
1035
+ if (files.length !== checksums.size)
1036
+ throw new Error("Parsed result ZIP directory does not match sequential entries.");
1037
+ for (const entry of files) {
1038
+ const actual = checksums.get(normalizeResultPath(entry.filename));
1039
+ if (!actual || actual.checksum !== entry.signature || actual.size !== entry.uncompressedSize)
1040
+ throw new Error(`Invalid parsed result ZIP checksum or size: ${entry.filename}`);
1041
+ }
1042
+ } finally {
1043
+ await reader.close();
1044
+ }
1045
+ })(),
1046
+ close: async () => {
1047
+ input.destroy();
1048
+ spool.destroy();
1049
+ parser.destroy();
1050
+ await completed.catch(() => void 0);
1051
+ try {
1052
+ await file.close();
1053
+ } finally {
1054
+ await (0, import_promises2.rm)(directory, { recursive: true, force: true });
1055
+ }
1056
+ }
1057
+ };
1058
+ }
1059
+ var import_stream, import_promises, import_promises2, import_os, import_path2, import_zlib, import_zip, import_parse, DiskArchiveReader;
1060
+ var init_sequential_parsed_document_archive = __esm({
1061
+ "src/storage/sequential-parsed-document-archive.ts"() {
1062
+ "use strict";
1063
+ import_stream = require("stream");
1064
+ import_promises = require("stream/promises");
1065
+ import_promises2 = require("fs/promises");
1066
+ import_os = require("os");
1067
+ import_path2 = require("path");
1068
+ import_zlib = require("zlib");
1069
+ import_zip = require("@zip.js/zip.js");
1070
+ import_parse = __toESM(require("unzipper/lib/parse.js"));
1071
+ init_parsed_document_archive_error();
1072
+ init_parsed_document_storage();
1073
+ init_streamed_parsed_document_sync();
1074
+ DiskArchiveReader = class extends import_zip.Reader {
1075
+ constructor(file, size) {
1076
+ super(file);
1077
+ this.file = file;
1078
+ this.size = size;
1079
+ }
1080
+ file;
1081
+ async readUint8Array(index, length) {
1082
+ const body = new Uint8Array(length);
1083
+ const { bytesRead } = await this.file.read(body, 0, length, index);
1084
+ if (bytesRead !== length) throw new Error("Truncated parsed result ZIP.");
1085
+ return body;
1086
+ }
1087
+ };
1088
+ }
1089
+ });
1090
+
1091
+ // src/storage/streamed-parsed-document-sync.ts
1092
+ async function syncRemoteResultArchive(params) {
1093
+ params.signal?.throwIfAborted();
1094
+ validateArchiveSyncLimits(params.limits);
1095
+ const startedAt = Date.now();
1096
+ const maxEntries = params.limits?.maxEntriesPerSync ?? 100;
1097
+ const deadlineMs = params.limits?.archiveSyncDeadlineMs ?? 12e4;
1098
+ const checkpointFileInterval = params.limits?.checkpointFileInterval ?? 1;
1099
+ const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
1100
+ const controller = new AbortController();
1101
+ const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
1102
+ const transferParams = { ...params, signal };
1103
+ const previous = await params.storage.readSyncProgress(params);
1104
+ const source = await openArchive(transferParams, previous);
1105
+ const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
1106
+ const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
1107
+ const entryHashes = [];
1108
+ const successfulHashes = /* @__PURE__ */ new Map();
1109
+ const running = /* @__PURE__ */ new Set();
1110
+ const paths = /* @__PURE__ */ new Set();
1111
+ let entryIndex = 0;
1112
+ let startedEntries = 0;
1113
+ let successfulEntries = 0;
1114
+ let checkpointEntry = nextEntry;
1115
+ let checkpointAt = Date.now();
1116
+ let failure;
1117
+ let archiveCacheBypassed = source.archiveCacheBypassed;
1118
+ async function flushProgress(isSegmentBoundary, isRecovering = false) {
1119
+ let contiguousEntry = checkpointEntry;
1120
+ while (successfulHashes.has(contiguousEntry)) {
1121
+ entryHashes[contiguousEntry] = successfulHashes.get(contiguousEntry);
1122
+ contiguousEntry += 1;
1123
+ }
1124
+ if (contiguousEntry === checkpointEntry) return;
1125
+ await params.storage.writeSyncProgress(
1126
+ {
1127
+ documentId: params.documentId,
1128
+ revisionKey: params.revisionKey,
1129
+ pinnedRevisionKey: params.revisionKey,
1130
+ archiveIdentity: source.identity,
1131
+ archiveCacheBypassed,
1132
+ archiveEntryHashes: entryHashes.slice(0, contiguousEntry),
1133
+ nextArchiveEntry: contiguousEntry,
1134
+ status: "running",
1135
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1136
+ },
1137
+ { signal: isRecovering ? void 0 : signal, isSegmentBoundary }
1138
+ );
1139
+ for (let index = checkpointEntry; index < contiguousEntry; index += 1)
1140
+ successfulHashes.delete(index);
1141
+ checkpointEntry = contiguousEntry;
1142
+ successfulEntries = successfulHashes.size;
1143
+ checkpointAt = Date.now();
1144
+ }
1145
+ function startEntry(entry, index) {
1146
+ const task = copyEntry(transferParams, entry).then(
1147
+ (hash) => {
1148
+ successfulHashes.set(index, hash);
1149
+ successfulEntries += 1;
1150
+ },
1151
+ (error) => {
1152
+ if (failure === void 0)
1153
+ failure = error instanceof Error ? error : new Error(String(error));
1154
+ controller.abort(error);
1155
+ }
1156
+ ).finally(() => {
1157
+ running.delete(task);
1158
+ });
1159
+ running.add(task);
1160
+ }
1161
+ async function drainEntries() {
1162
+ await Promise.all(running);
1163
+ if (failure !== void 0) throw failure;
1164
+ signal.throwIfAborted();
1165
+ }
1166
+ try {
1167
+ if (previous?.archiveIdentity && previous.archiveIdentity !== source.identity)
1168
+ throw new Error("Parsed result archive identity changed; start a new revision.");
1169
+ if (previous?.pinnedRevisionKey && previous.pinnedRevisionKey !== params.revisionKey)
1170
+ throw new Error("Parsed result revision changed while resuming archive sync.");
1171
+ for await (const entry of source.entries) {
1172
+ if (failure !== void 0) throw failure;
1173
+ signal.throwIfAborted();
1174
+ if (paths.has(entry.path)) throw new Error(`Duplicate parsed result path: ${entry.path}`);
1175
+ paths.add(entry.path);
1176
+ if (entryIndex < nextEntry) {
1177
+ const previousHash = previous?.archiveEntryHashes?.[entryIndex];
1178
+ if (!previousHash) throw new Error("Parsed archive checkpoint is missing an entry hash.");
1179
+ if (!source.canSkipEntries && await copyEntry(transferParams, entry, true) !== previousHash)
1180
+ throw new Error("Parsed result archive entry changed while resuming.");
1181
+ entryHashes.push(previousHash);
1182
+ entryIndex += 1;
1183
+ continue;
1184
+ }
1185
+ if (startedEntries > 0 && (startedEntries >= maxEntries || Date.now() - startedAt >= deadlineMs)) {
1186
+ await drainEntries();
1187
+ await flushProgress(true);
1188
+ return { documentId: params.documentId, revisionKey: params.revisionKey, completed: false };
1189
+ }
1190
+ if (successfulEntries >= checkpointFileInterval || Date.now() - checkpointAt >= checkpointIntervalMs) {
1191
+ await drainEntries();
1192
+ await flushProgress(false);
1193
+ }
1194
+ startEntry(entry, entryIndex);
1195
+ entryIndex += 1;
1196
+ startedEntries += 1;
1197
+ if (running.size >= concurrency) await Promise.race(running);
1198
+ }
1199
+ await drainEntries();
1200
+ await flushProgress(true);
1201
+ await validateRequiredObjects(params, paths);
1202
+ await params.verifyRevision?.();
1203
+ signal.throwIfAborted();
1204
+ const commit = {
1205
+ version: 1,
1206
+ documentId: params.documentId,
1207
+ revisionKey: params.revisionKey,
1208
+ source: "resultZip",
1209
+ committedAt: (/* @__PURE__ */ new Date()).toISOString()
1210
+ };
1211
+ await params.storage.writeObject({
1212
+ documentId: params.documentId,
1213
+ revisionKey: params.revisionKey,
1214
+ path: parsedDocumentCommitPath,
1215
+ body: Buffer.from(JSON.stringify(commit, null, 2)),
1216
+ contentType: "application/json; charset=utf-8"
1217
+ });
1218
+ await params.storage.writeSyncProgress(
1219
+ {
1220
+ documentId: params.documentId,
1221
+ revisionKey: params.revisionKey,
1222
+ pinnedRevisionKey: params.revisionKey,
1223
+ archiveIdentity: source.identity,
1224
+ archiveCacheBypassed,
1225
+ archiveEntryHashes: entryHashes,
1226
+ nextArchiveEntry: entryIndex,
1227
+ status: "completed",
1228
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1229
+ },
1230
+ { signal, isSegmentBoundary: true }
1231
+ );
1232
+ if (params.retainResultArchive && !source.isCached) {
1233
+ try {
1234
+ await retainArchive(transferParams, source.downloadIdentity);
1235
+ } catch {
1236
+ }
1237
+ }
1238
+ return {
1239
+ documentId: params.documentId,
1240
+ revisionKey: params.revisionKey,
1241
+ completed: true,
1242
+ commit
1243
+ };
1244
+ } catch (error) {
1245
+ controller.abort(error);
1246
+ await Promise.all(running);
1247
+ const hasCacheTransportFailure = error instanceof ArchiveCacheTransportError || failure instanceof ArchiveCacheTransportError;
1248
+ if (hasCacheTransportFailure) archiveCacheBypassed = true;
1249
+ await flushProgress(true, true).catch(() => void 0);
1250
+ if (hasCacheTransportFailure) {
1251
+ await persistArchiveCacheBypass(
1252
+ { ...params, signal: void 0 },
1253
+ {
1254
+ documentId: params.documentId,
1255
+ revisionKey: params.revisionKey,
1256
+ pinnedRevisionKey: params.revisionKey,
1257
+ archiveIdentity: source.identity,
1258
+ archiveEntryHashes: entryHashes.slice(0, checkpointEntry),
1259
+ nextArchiveEntry: checkpointEntry,
1260
+ status: "running",
1261
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1262
+ }
1263
+ ).catch(() => void 0);
1264
+ }
1265
+ throw failure ?? error;
1266
+ } finally {
1267
+ controller.abort();
1268
+ await Promise.all(running);
1269
+ await source.close();
1270
+ }
1271
+ }
1272
+ function validateArchiveSyncLimits(limits) {
1273
+ const names = [
1274
+ "maxEntriesPerSync",
1275
+ "archiveSyncDeadlineMs",
1276
+ "maxConcurrentFiles",
1277
+ "checkpointFileInterval",
1278
+ "checkpointIntervalMs"
1279
+ ];
1280
+ for (const name of names) {
1281
+ const value = limits?.[name];
1282
+ if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0))
1283
+ throw new ValidationError(`${name} must be a positive safe integer.`);
1284
+ }
1285
+ }
1286
+ async function copyEntry(params, entry, isVerification = false) {
1287
+ const hash = (0, import_crypto2.createHash)("sha256");
1288
+ const body = entry.read().pipeThrough(
1289
+ new TransformStream({
1290
+ transform(chunk, controller) {
1291
+ params.signal?.throwIfAborted();
1292
+ hash.update(chunk);
1293
+ controller.enqueue(chunk);
1294
+ }
1295
+ })
1296
+ );
1297
+ if (isVerification)
1298
+ await body.pipeTo(new WritableStream(), { signal: params.signal });
1299
+ else
1300
+ await writeStream(params.storage, {
1301
+ documentId: params.documentId,
1302
+ revisionKey: params.revisionKey,
1303
+ path: entry.path,
1304
+ body,
1305
+ contentType: inferContentTypeFromPath(entry.path),
1306
+ contentLength: entry.contentLength,
1307
+ signal: params.signal
1308
+ });
1309
+ return (0, import_crypto2.createHash)("sha256").update(entry.path).update(hash.digest()).digest("hex");
1310
+ }
1311
+ async function openArchive(params, previous) {
1312
+ const prepared = params.cacheResultArchive ? await prepareCachedArchive({
1313
+ ...params,
1314
+ archiveCacheBypassed: previous?.archiveCacheBypassed,
1315
+ onBypass: async () => {
1316
+ await persistArchiveCacheBypass(params, previous);
1317
+ }
1318
+ }) : void 0;
1319
+ try {
1320
+ return await openArchiveTransport(params, previous, prepared);
1321
+ } catch (error) {
1322
+ if (!prepared?.isCached) throw error;
1323
+ params.signal?.throwIfAborted();
1324
+ const bypass = await persistArchiveCacheBypass(params, previous);
1325
+ return openArchive({ ...params, cacheResultArchive: false }, bypass);
1326
+ }
1327
+ }
1328
+ async function persistArchiveCacheBypass(params, previous) {
1329
+ const progress = {
1330
+ ...previous?.status === "completed" ? {} : previous,
1331
+ documentId: params.documentId,
1332
+ revisionKey: params.revisionKey,
1333
+ pinnedRevisionKey: params.revisionKey,
1334
+ archiveCacheBypassed: true,
1335
+ status: "running",
1336
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1337
+ };
1338
+ await params.storage.writeSyncProgress(progress, { signal: params.signal });
1339
+ return progress;
1340
+ }
1341
+ async function openArchiveTransport(params, previous, prepared) {
1342
+ const resultUrl = prepared?.resultUrl ?? params.resultUrl;
1343
+ const archiveCacheBypassed = prepared?.archiveCacheBypassed ?? previous?.archiveCacheBypassed;
1344
+ let probe = await fetch(resultUrl, {
1345
+ headers: { Range: "bytes=0-0" },
1346
+ signal: params.signal
1347
+ });
1348
+ if (!probe.ok) {
1349
+ await probe.body?.cancel();
1350
+ throw new Error(`Parsed result download failed (HTTP ${probe.status}).`);
1351
+ }
1352
+ if (prepared) probe = prepared.validateResponse(probe);
1353
+ const identity = prepared?.downloadIdentity ?? createDownloadIdentity(params.jobId, probe);
1354
+ const supportsRange = probe.status === 206 && probe.headers.has("content-range");
1355
+ await probe.body?.cancel();
1356
+ if (!supportsRange) {
1357
+ const { openSequentialArchive: openSequentialArchive2 } = await Promise.resolve().then(() => (init_sequential_parsed_document_archive(), sequential_parsed_document_archive_exports));
1358
+ const source = await openSequentialArchive2(
1359
+ { ...params, resultUrl },
1360
+ identity,
1361
+ prepared?.isCached ? prepared.validateResponse : void 0
1362
+ );
1363
+ return { ...source, archiveCacheBypassed };
1364
+ }
1365
+ const options = {
1366
+ forceRangeRequests: true,
1367
+ preventHeadRequest: true,
1368
+ checkResourceChanges: true,
1369
+ fetch: async (input, init) => {
1370
+ let response;
1371
+ try {
1372
+ response = await fetch(input, { ...init, signal: params.signal });
1373
+ } catch (error) {
1374
+ params.signal?.throwIfAborted();
1375
+ if (prepared?.isCached)
1376
+ throw new ArchiveCacheTransportError(
1377
+ "Parsed result archive cache transport unavailable."
1378
+ );
1379
+ throw error;
1380
+ }
1381
+ if (prepared?.isCached) return prepared.validateResponse(response);
1382
+ if (response.status !== 206) {
1383
+ await response.body?.cancel();
1384
+ throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
1385
+ }
1386
+ return prepared ? prepared.validateResponse(response) : response;
1387
+ }
1388
+ };
1389
+ const reader = new import_zip2.ZipReader(new import_zip2.HttpRangeReader(resultUrl, options), {
1390
+ useWebWorkers: false,
1391
+ checkCrc32: true
1392
+ });
1393
+ const producers = /* @__PURE__ */ new Set();
1394
+ try {
1395
+ const entries = await reader.getEntries();
1396
+ const files = entries.filter(
1397
+ (entry) => !entry.directory && isSnapshotPath(entry.filename)
1398
+ );
1399
+ const archiveIdentity = (0, import_crypto2.createHash)("sha256").update(identity).update(
1400
+ JSON.stringify(
1401
+ files.map((entry) => [entry.filename, entry.uncompressedSize, entry.signature])
1402
+ )
1403
+ ).digest("hex");
1404
+ return {
1405
+ identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
1406
+ downloadIdentity: identity,
1407
+ archiveCacheBypassed,
1408
+ isCached: prepared?.isCached,
1409
+ canSkipEntries: previous?.archiveIdentity !== identity,
1410
+ entries: (function* () {
1411
+ for (const entry of files)
1412
+ yield {
1413
+ path: normalizeResultPath(entry.filename),
1414
+ contentLength: entry.uncompressedSize,
1415
+ read: () => readRangeEntry(entry, params.signal, producers)
1416
+ };
1417
+ })(),
1418
+ close: async () => {
1419
+ await Promise.all(producers);
1420
+ await reader.close();
1421
+ }
1422
+ };
1423
+ } catch (error) {
1424
+ await reader.close();
1425
+ throw error;
1426
+ }
1427
+ }
1428
+ function readRangeEntry(entry, signal, producers) {
1429
+ const stream = new TransformStream();
1430
+ const producer = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(
1431
+ () => void 0,
1432
+ async (error) => {
1433
+ try {
1434
+ await stream.writable.abort(error);
1435
+ } catch {
1436
+ }
1437
+ }
1438
+ ).finally(() => {
1439
+ producers.delete(producer);
1440
+ });
1441
+ producers.add(producer);
1442
+ return stream.readable;
1443
+ }
1444
+ async function writeStream(storage, params) {
1445
+ if (storage.writeObjectStream) {
1446
+ try {
1447
+ await storage.writeObjectStream(params);
1448
+ } catch (error) {
1449
+ await params.body.cancel(error).catch(() => void 0);
1450
+ throw error;
1451
+ }
1452
+ return;
1453
+ }
1454
+ const parts = [];
1455
+ for await (const part of params.body) {
1456
+ params.signal?.throwIfAborted();
1457
+ parts.push(part);
1458
+ }
1459
+ await storage.writeObject({ ...params, body: Buffer.concat(parts) });
1460
+ }
1461
+ async function validateRequiredObjects(params, paths) {
1462
+ const required = /* @__PURE__ */ new Set(["manifest.json", "chunks.json"]);
1463
+ for (const path4 of required)
1464
+ if (!paths.has(path4)) throw new Error(`Required parsed result file missing: ${path4}`);
1465
+ const manifestObject = await params.storage.readObject({ ...params, path: "manifest.json" });
1466
+ const chunksObject = await params.storage.readObject({ ...params, path: "chunks.json" });
1467
+ if (!manifestObject || !chunksObject)
1468
+ throw new Error("Required parsed result manifest.json or chunks.json was not persisted.");
1469
+ const manifest = JSON.parse(Buffer.from(manifestObject.body).toString("utf8"));
1470
+ const payload = JSON.parse(Buffer.from(chunksObject.body).toString("utf8"));
1471
+ if (!isRecord2(manifest)) throw new Error("Invalid parsed result manifest.json.");
1472
+ const manifestJobId = manifest.job_id ?? manifest.jobId;
1473
+ if (manifestJobId !== params.jobId)
1474
+ throw new Error("Parsed result manifest job identity differs from the pinned job.");
1475
+ const chunks = Array.isArray(payload) ? payload : isRecord2(payload) ? payload.chunks : void 0;
1476
+ if (!Array.isArray(chunks)) throw new Error("Invalid parsed result chunks.json.");
1477
+ if (isRecord2(manifest.files)) {
1478
+ for (const path4 of Object.values(manifest.files))
1479
+ if (typeof path4 === "string") required.add(normalizeResultPath(path4));
1480
+ }
1481
+ for (const chunk of chunks) {
1482
+ if (!isRecord2(chunk)) throw new Error("Invalid parsed result chunk.");
1483
+ const metadata = chunk.metadata;
1484
+ const chunkType = chunk.type ?? chunk.chunk_type ?? chunk.chunkType;
1485
+ const filePath = chunk.file_path ?? chunk.filePath ?? (isRecord2(metadata) ? metadata.file_path ?? metadata.filePath : void 0) ?? (chunkType === "image" || chunkType === "table" ? chunk.path : void 0);
1486
+ if (typeof filePath === "string") required.add(normalizeResultPath(filePath));
1487
+ if (!isRecord2(metadata)) continue;
1488
+ const pageAssets = metadata.page_assets ?? metadata.pageAssets;
1489
+ if (!Array.isArray(pageAssets)) continue;
1490
+ for (const asset of pageAssets)
1491
+ if (isRecord2(asset)) {
1492
+ const ref = asset.artifact_ref ?? asset.artifactRef;
1493
+ if (typeof ref === "string") required.add(normalizeResultPath(ref));
1494
+ }
1495
+ }
1496
+ for (const path4 of required)
1497
+ if (!paths.has(path4)) throw new Error(`Required parsed result file missing: ${path4}`);
1498
+ }
1499
+ async function retainArchive(params, expectedIdentity) {
1500
+ const response = await fetch(params.resultUrl, { signal: params.signal });
1501
+ if (!response.ok || !response.body)
1502
+ throw new Error(`Optional archive download failed (HTTP ${response.status}).`);
1503
+ if (createDownloadIdentity(params.jobId, response) !== expectedIdentity) {
1504
+ await response.body.cancel();
1505
+ throw new Error("Optional result archive identity changed.");
1506
+ }
1507
+ await writeStream(params.storage, {
1508
+ documentId: params.documentId,
1509
+ revisionKey: params.revisionKey,
1510
+ path: "result.zip",
1511
+ body: response.body,
1512
+ contentType: "application/zip",
1513
+ signal: params.signal
1514
+ });
1515
+ }
1516
+ function isSnapshotPath(path4) {
1517
+ const normalized = normalizeResultPath(path4.replace(/\/$/, ""));
1518
+ return normalized !== "result.zip" && normalized !== ".knowhere-sdk" && !normalized.startsWith(".knowhere-sdk/");
1519
+ }
1520
+ function isRecord2(value) {
1521
+ return typeof value === "object" && value !== null && !Array.isArray(value);
1522
+ }
1523
+ function createDownloadIdentity(jobId, response) {
1524
+ return (0, import_crypto2.createHash)("sha256").update(
1525
+ JSON.stringify([
1526
+ jobId,
1527
+ response.headers.get("etag"),
1528
+ response.headers.get("last-modified"),
1529
+ response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
1530
+ ])
1531
+ ).digest("hex");
1532
+ }
1533
+ var import_crypto2, import_zip2;
1534
+ var init_streamed_parsed_document_sync = __esm({
1535
+ "src/storage/streamed-parsed-document-sync.ts"() {
1536
+ "use strict";
1537
+ import_crypto2 = require("crypto");
1538
+ init_errors();
1539
+ init_archive_cache_transport_error();
1540
+ init_parsed_document_archive_cache();
1541
+ import_zip2 = require("@zip.js/zip.js");
1542
+ init_parsed_document_storage();
1543
+ }
1544
+ });
1545
+
1546
+ // src/index.ts
1547
+ var index_exports = {};
1548
+ __export(index_exports, {
1549
+ APIError: () => APIError,
1550
+ AuthenticationError: () => AuthenticationError,
1551
+ BadRequestError: () => BadRequestError,
1552
+ ChecksumError: () => ChecksumError,
1553
+ ConflictError: () => ConflictError,
1554
+ DiskParsedDocumentStorage: () => DiskParsedDocumentStorage,
1555
+ Documents: () => Documents,
1556
+ GatewayTimeoutError: () => GatewayTimeoutError,
1557
+ InternalServerError: () => InternalServerError,
1558
+ InvalidStateError: () => InvalidStateError,
1559
+ JobFailedError: () => JobFailedError,
1560
+ Jobs: () => Jobs,
1561
+ Knowhere: () => Knowhere,
1562
+ KnowhereError: () => KnowhereError,
1563
+ Knowledge: () => Knowledge,
1564
+ LocalKnowledgeStore: () => LocalKnowledgeStore,
1565
+ NODE_SDK_DOCUMENT_METADATA_DEFAULTS: () => NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
1566
+ NetworkError: () => NetworkError,
1567
+ NotFoundError: () => NotFoundError,
1568
+ ParsedDocumentArchiveError: () => ParsedDocumentArchiveError,
1569
+ PaymentRequiredError: () => PaymentRequiredError,
1570
+ PermissionDeniedError: () => PermissionDeniedError,
1571
+ PollingTimeoutError: () => PollingTimeoutError,
1572
+ RateLimitError: () => RateLimitError,
1573
+ Retrieval: () => Retrieval,
1574
+ ServiceUnavailableError: () => ServiceUnavailableError,
1575
+ TimeoutError: () => TimeoutError,
1576
+ VERSION: () => VERSION,
1577
+ ValidationError: () => ValidationError,
1578
+ default: () => Knowhere,
1579
+ mergeDocumentMetadataDefaults: () => mergeDocumentMetadataDefaults,
1580
+ storeParseResultAssets: () => storeParseResultAssets,
1581
+ syncParseResultToParsedDocumentStorage: () => syncParseResultToParsedDocumentStorage
1582
+ });
1583
+ module.exports = __toCommonJS(index_exports);
1584
+
1585
+ // src/client.ts
1586
+ var import_path4 = __toESM(require("path"));
1587
+
1588
+ // src/lib/http-client.ts
1589
+ var import_axios = __toESM(require("axios"));
1590
+
1591
+ // src/version.ts
1592
+ var VERSION = "2.1.3";
1593
+
1594
+ // src/lib/http-client.ts
1595
+ init_constants();
1596
+ init_errors();
1597
+ init_utils();
391
1598
 
392
1599
  // src/lib/retry.ts
1600
+ init_constants();
1601
+ init_utils();
393
1602
  function getErrorCode(error) {
394
1603
  const errorWithResponse = error;
395
1604
  return errorWithResponse?.response?.data?.error?.code ?? errorWithResponse?.response?.data?.code ?? errorWithResponse?.code;
@@ -780,6 +1989,7 @@ function mergeDocumentMetadataDefaults(defaults, provided) {
780
1989
 
781
1990
  // src/lib/upload.ts
782
1991
  var import_fs = require("fs");
1992
+ init_errors();
783
1993
  async function uploadFile(httpClient, uploadUrl, file, options) {
784
1994
  let data;
785
1995
  let contentLength;
@@ -815,6 +2025,9 @@ function isReadStream(obj) {
815
2025
  }
816
2026
 
817
2027
  // src/lib/polling.ts
2028
+ init_constants();
2029
+ init_errors();
2030
+ init_utils();
818
2031
  async function pollJobStatus(httpClient, jobId, options) {
819
2032
  const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
820
2033
  const pollTimeout = options?.pollTimeout ?? DEFAULT_POLL_TIMEOUT;
@@ -865,6 +2078,8 @@ async function pollJobStatus(httpClient, jobId, options) {
865
2078
  var import_jszip = __toESM(require("jszip"));
866
2079
  var import_fs2 = require("fs");
867
2080
  var import_path = require("path");
2081
+ init_errors();
2082
+ init_utils();
868
2083
  async function parseResult(httpClient, resultUrl, options) {
869
2084
  const zipBuffer = await httpClient.download(resultUrl);
870
2085
  if (options?.verifyChecksum !== false) {
@@ -1367,6 +2582,8 @@ function isMissingFileError(error) {
1367
2582
  }
1368
2583
 
1369
2584
  // src/resources/jobs.ts
2585
+ init_utils();
2586
+ init_errors();
1370
2587
  var Jobs = class extends BaseResource {
1371
2588
  pendingUploadJobs = /* @__PURE__ */ new Map();
1372
2589
  /**
@@ -1607,7 +2824,8 @@ var Documents = class extends BaseResource {
1607
2824
  };
1608
2825
 
1609
2826
  // src/knowledge/knowledge.ts
1610
- var import_crypto2 = require("crypto");
2827
+ var import_crypto4 = require("crypto");
2828
+ init_errors();
1611
2829
 
1612
2830
  // src/storage/asset-storage.ts
1613
2831
  var import_jszip2 = __toESM(require("jszip"));
@@ -1871,220 +3089,16 @@ function isRecord(value) {
1871
3089
  return typeof value === "object" && value !== null;
1872
3090
  }
1873
3091
 
1874
- // src/storage/parsed-document-storage.ts
1875
- var import_jszip3 = __toESM(require("jszip"));
1876
- var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
1877
- var jsonContentType = "application/json; charset=utf-8";
1878
- var markdownContentType = "text/markdown; charset=utf-8";
1879
- var csvContentType = "text/csv; charset=utf-8";
1880
- var htmlContentType = "text/html; charset=utf-8";
1881
- var binaryContentType = "application/octet-stream";
1882
- async function syncParseResultToParsedDocumentStorage(params) {
1883
- const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
1884
- const revisionKey = params.revisionKey ?? params.result.jobId;
1885
- const objects = await collectResultLayoutObjects(params.result);
1886
- for (const object of objects) {
1887
- await params.storage.writeObject({
1888
- documentId,
1889
- revisionKey,
1890
- ...object
1891
- });
1892
- }
1893
- const commit = {
1894
- version: 1,
1895
- documentId,
1896
- revisionKey,
1897
- source: params.source ?? "resultZip",
1898
- committedAt: (/* @__PURE__ */ new Date()).toISOString()
1899
- };
1900
- await writeParsedDocumentJsonObject(params.storage, {
1901
- documentId,
1902
- revisionKey,
1903
- path: parsedDocumentCommitPath,
1904
- value: commit
1905
- });
1906
- return {
1907
- result: params.result,
1908
- documentId,
1909
- revisionKey,
1910
- commit
1911
- };
1912
- }
1913
- async function readParsedDocumentCommit(params) {
1914
- const object = await params.storage.readObject({
1915
- documentId: params.documentId,
1916
- revisionKey: params.revisionKey,
1917
- path: parsedDocumentCommitPath
1918
- });
1919
- if (!object) {
1920
- return null;
1921
- }
1922
- const commit = readJsonObject(object);
1923
- return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
1924
- }
1925
- function readJsonObject(object) {
1926
- return JSON.parse(Buffer.from(object.body).toString("utf8"));
1927
- }
1928
- async function collectResultLayoutObjects(result) {
1929
- const objectsByPath = /* @__PURE__ */ new Map();
1930
- const rawZipObjects = await collectRawZipObjects(result.rawZip);
1931
- for (const object of rawZipObjects) {
1932
- objectsByPath.set(object.path, object);
1933
- }
1934
- for (const object of collectGeneratedResultObjects(result)) {
1935
- if (!objectsByPath.has(object.path)) {
1936
- objectsByPath.set(object.path, object);
1937
- }
1938
- }
1939
- return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
1940
- }
1941
- async function collectRawZipObjects(rawZip) {
1942
- try {
1943
- const zip = await import_jszip3.default.loadAsync(rawZip);
1944
- const objects = [];
1945
- for (const entry of Object.values(zip.files)) {
1946
- if (entry.dir || entry.name === "result.zip") {
1947
- continue;
1948
- }
1949
- objects.push({
1950
- path: normalizeResultPath(entry.name),
1951
- body: new Uint8Array(await entry.async("nodebuffer")),
1952
- contentType: inferContentTypeFromPath(entry.name)
1953
- });
1954
- }
1955
- return objects;
1956
- } catch {
1957
- return [];
1958
- }
1959
- }
1960
- function collectGeneratedResultObjects(result) {
1961
- const objects = [
1962
- createJsonObject("manifest.json", result.manifest),
1963
- createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
1964
- ];
1965
- if (result.docNav) {
1966
- objects.push(createJsonObject("doc_nav.json", result.docNav));
1967
- }
1968
- if (result.fullMarkdown) {
1969
- objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
1970
- }
1971
- if (result.chunksSlim) {
1972
- objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
1973
- }
1974
- if (result.hierarchy) {
1975
- objects.push(createJsonObject("hierarchy.json", result.hierarchy));
1976
- }
1977
- if (result.tocHierarchies) {
1978
- objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
1979
- }
1980
- if (result.kbCsv) {
1981
- objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
1982
- }
1983
- if (result.hierarchyViewHtml) {
1984
- objects.push(
1985
- createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
1986
- );
1987
- }
1988
- for (const chunk of result.imageChunks) {
1989
- objects.push({
1990
- path: normalizeResultPath(chunk.filePath),
1991
- body: new Uint8Array(chunk.data),
1992
- contentType: inferContentTypeFromPath(chunk.filePath),
1993
- metadata: {
1994
- chunkId: chunk.chunkId,
1995
- chunkType: chunk.type,
1996
- sourcePath: chunk.filePath
1997
- }
1998
- });
1999
- }
2000
- for (const chunk of result.tableChunks) {
2001
- objects.push({
2002
- path: normalizeResultPath(chunk.filePath),
2003
- body: Buffer.from(chunk.html, "utf8"),
2004
- contentType: htmlContentType,
2005
- metadata: {
2006
- chunkId: chunk.chunkId,
2007
- chunkType: chunk.type,
2008
- sourcePath: chunk.filePath
2009
- }
2010
- });
2011
- }
2012
- return objects;
2013
- }
2014
- function serializeChunks2(chunks) {
2015
- return {
2016
- chunks: chunks.map((chunk) => {
2017
- const rawChunk = {
2018
- chunkId: chunk.chunkId,
2019
- type: chunk.type,
2020
- contentSource: chunk.contentSource,
2021
- content: chunk.content,
2022
- path: chunk.path,
2023
- metadata: chunk.metadata
2024
- };
2025
- if (chunk.type === "image" || chunk.type === "table") {
2026
- return {
2027
- ...rawChunk,
2028
- filePath: chunk.filePath,
2029
- assetUrl: chunk.assetUrl
2030
- };
2031
- }
2032
- return rawChunk;
2033
- })
2034
- };
2035
- }
2036
- async function writeParsedDocumentJsonObject(storage, params) {
2037
- await storage.writeObject({
2038
- documentId: params.documentId,
2039
- revisionKey: params.revisionKey,
2040
- path: params.path,
2041
- body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
2042
- contentType: jsonContentType
2043
- });
2044
- }
2045
- function createJsonObject(path4, value) {
2046
- return {
2047
- path: path4,
2048
- body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
2049
- contentType: jsonContentType
2050
- };
2051
- }
2052
- function createTextObject(path4, value, contentType) {
2053
- return {
2054
- path: path4,
2055
- body: Buffer.from(value, "utf8"),
2056
- contentType
2057
- };
2058
- }
2059
- function inferContentTypeFromPath(filePath) {
2060
- const lower = filePath.toLowerCase();
2061
- if (lower.endsWith(".json")) return jsonContentType;
2062
- if (lower.endsWith(".md")) return markdownContentType;
2063
- if (lower.endsWith(".csv")) return csvContentType;
2064
- if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
2065
- if (lower.endsWith(".png")) return "image/png";
2066
- if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
2067
- if (lower.endsWith(".gif")) return "image/gif";
2068
- if (lower.endsWith(".webp")) return "image/webp";
2069
- if (lower.endsWith(".svg")) return "image/svg+xml";
2070
- return binaryContentType;
2071
- }
2072
- function normalizeResultPath(value) {
2073
- const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
2074
- if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
2075
- throw new Error(`Invalid parsed result path: ${value}`);
2076
- }
2077
- return normalized;
2078
- }
2079
- function isSdkMetadataPath(path4) {
2080
- return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
2081
- }
3092
+ // src/knowledge/knowledge.ts
3093
+ init_parsed_document_storage();
3094
+ init_utils();
3095
+ init_streamed_parsed_document_sync();
2082
3096
 
2083
3097
  // src/knowledge/local-store.ts
2084
- var import_crypto = require("crypto");
2085
- var import_os = __toESM(require("os"));
3098
+ var import_crypto3 = require("crypto");
3099
+ var import_os2 = __toESM(require("os"));
2086
3100
  var import_fs3 = require("fs");
2087
- var import_path2 = __toESM(require("path"));
3101
+ var import_path3 = __toESM(require("path"));
2088
3102
  var STORE_VERSION = 1;
2089
3103
  var LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
2090
3104
  var LocalKnowledgeStore = class {
@@ -2092,8 +3106,8 @@ var LocalKnowledgeStore = class {
2092
3106
  indexPath;
2093
3107
  resultCache = /* @__PURE__ */ new Map();
2094
3108
  constructor(cacheDirectory) {
2095
- this.cacheDirectory = cacheDirectory ?? import_path2.default.join(import_os.default.homedir(), ".knowhere-node-sdk", "knowledge");
2096
- this.indexPath = import_path2.default.join(this.cacheDirectory, "index.json");
3109
+ this.cacheDirectory = cacheDirectory ?? import_path3.default.join(import_os2.default.homedir(), ".knowhere-node-sdk", "knowledge");
3110
+ this.indexPath = import_path3.default.join(this.cacheDirectory, "index.json");
2097
3111
  }
2098
3112
  async saveResult(result, options) {
2099
3113
  await import_fs3.promises.mkdir(this.cacheDirectory, { recursive: true });
@@ -2218,8 +3232,8 @@ var LocalKnowledgeStore = class {
2218
3232
  return { document, result };
2219
3233
  }
2220
3234
  getResultDirectoryPath(localDocumentId) {
2221
- const documentsDirectory = import_path2.default.resolve(this.cacheDirectory, "documents");
2222
- const resultDirectoryPath = import_path2.default.resolve(documentsDirectory, localDocumentId);
3235
+ const documentsDirectory = import_path3.default.resolve(this.cacheDirectory, "documents");
3236
+ const resultDirectoryPath = import_path3.default.resolve(documentsDirectory, localDocumentId);
2223
3237
  if (!isPathInsideDirectory(resultDirectoryPath, documentsDirectory)) {
2224
3238
  throw new Error(`Local Knowhere document ID resolves outside the cache: ${localDocumentId}`);
2225
3239
  }
@@ -2253,7 +3267,7 @@ var LocalKnowledgeStore = class {
2253
3267
  }
2254
3268
  };
2255
3269
  function validateLocalDocumentId(localDocumentId) {
2256
- if (!LOCAL_DOCUMENT_ID_PATTERN.test(localDocumentId) || localDocumentId.includes("..") || import_path2.default.basename(localDocumentId) !== localDocumentId) {
3270
+ if (!LOCAL_DOCUMENT_ID_PATTERN.test(localDocumentId) || localDocumentId.includes("..") || import_path3.default.basename(localDocumentId) !== localDocumentId) {
2257
3271
  throw new Error(
2258
3272
  "Local Knowhere document ID must be a safe slug containing only letters, numbers, dots, underscores, or hyphens"
2259
3273
  );
@@ -2261,11 +3275,11 @@ function validateLocalDocumentId(localDocumentId) {
2261
3275
  return localDocumentId;
2262
3276
  }
2263
3277
  function isPathInsideDirectory(targetPath, parentDirectory) {
2264
- const relativePath = import_path2.default.relative(parentDirectory, targetPath);
2265
- return relativePath.length === 0 || !relativePath.startsWith("..") && !import_path2.default.isAbsolute(relativePath);
3278
+ const relativePath = import_path3.default.relative(parentDirectory, targetPath);
3279
+ return relativePath.length === 0 || !relativePath.startsWith("..") && !import_path3.default.isAbsolute(relativePath);
2266
3280
  }
2267
3281
  function createLocalDocumentId(result) {
2268
- const hash = (0, import_crypto.createHash)("sha256").update(result.jobId).update("\0").update(result.manifest.sourceFileName).digest("hex").slice(0, 16);
3282
+ const hash = (0, import_crypto3.createHash)("sha256").update(result.jobId).update("\0").update(result.manifest.sourceFileName).digest("hex").slice(0, 16);
2269
3283
  return `local_${hash}`;
2270
3284
  }
2271
3285
  function countChunkTypes(result) {
@@ -2419,29 +3433,35 @@ var Knowledge = class _Knowledge {
2419
3433
  return this.importJobResult(params);
2420
3434
  }
2421
3435
  async syncParsedDocument(params) {
3436
+ params.signal?.throwIfAborted();
2422
3437
  const normalized = normalizeDocumentReference(params);
2423
3438
  const parsedStorageConfig = this.requireParsedStorageConfig();
3439
+ validateArchiveSyncLimits(parsedStorageConfig.limits);
2424
3440
  if (normalized.documentId) {
2425
3441
  return this.syncRemoteDocumentToParsedStorage({
2426
3442
  documentId: normalized.documentId,
2427
3443
  revisionKey: params.revisionKey,
2428
- parsedStorageConfig
3444
+ parsedStorageConfig,
3445
+ signal: params.signal,
3446
+ retainResultArchive: params.retainResultArchive,
3447
+ cacheResultArchive: params.cacheResultArchive
2429
3448
  });
2430
3449
  }
2431
3450
  if (normalized.jobId) {
2432
- const loadedResult = await this.client.jobs.load(normalized.jobId);
2433
- const synced2 = await syncParseResultToParsedDocumentStorage({
2434
- result: loadedResult,
3451
+ const job = await this.client.jobs.get(normalized.jobId);
3452
+ if (!job.isDone || !job.resultUrl)
3453
+ throw new ValidationError("Completed job result archive is unavailable");
3454
+ return syncRemoteResultArchive({
3455
+ documentId: job.documentId ?? normalized.localDocumentId ?? normalized.jobId,
3456
+ revisionKey: params.revisionKey ?? normalized.jobId,
3457
+ jobId: normalized.jobId,
3458
+ resultUrl: job.resultUrl,
2435
3459
  storage: parsedStorageConfig.storage,
2436
- documentId: loadedResult.documentId ?? normalized.localDocumentId ?? normalized.jobId,
2437
- revisionKey: params.revisionKey ?? normalized.jobId
3460
+ limits: parsedStorageConfig.limits,
3461
+ signal: params.signal,
3462
+ retainResultArchive: params.retainResultArchive,
3463
+ cacheResultArchive: params.cacheResultArchive
2438
3464
  });
2439
- return {
2440
- documentId: synced2.documentId,
2441
- revisionKey: synced2.revisionKey,
2442
- completed: true,
2443
- commit: synced2.commit
2444
- };
2445
3465
  }
2446
3466
  if (!normalized.localDocumentId) {
2447
3467
  throw new ValidationError("localDocumentId, documentId, or jobId is required");
@@ -3139,10 +4159,14 @@ var Knowledge = class _Knowledge {
3139
4159
  parsedStorageConfig
3140
4160
  });
3141
4161
  } catch (error) {
4162
+ const previous = await parsedStorageConfig.storage.readSyncProgress({
4163
+ documentId,
4164
+ revisionKey
4165
+ });
3142
4166
  await parsedStorageConfig.storage.writeSyncProgress({
4167
+ ...previous,
3143
4168
  documentId,
3144
4169
  revisionKey,
3145
- nextChunkPage: 1,
3146
4170
  status: "failed",
3147
4171
  updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
3148
4172
  error: error instanceof Error ? error.message : String(error)
@@ -3152,6 +4176,7 @@ var Knowledge = class _Knowledge {
3152
4176
  void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
3153
4177
  }
3154
4178
  async syncRemoteDocumentToParsedStorage(params) {
4179
+ params.signal?.throwIfAborted();
3155
4180
  const firstPage = await this.client.documents.listChunks(params.documentId, {
3156
4181
  page: 1,
3157
4182
  pageSize: 1,
@@ -3159,55 +4184,68 @@ var Knowledge = class _Knowledge {
3159
4184
  });
3160
4185
  const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
3161
4186
  if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
3162
- return this.syncRemoteDocumentToParsedStorage({
3163
- documentId: params.documentId,
3164
- parsedStorageConfig: params.parsedStorageConfig
3165
- });
4187
+ throw new ValidationError("Published document revision changed before parsed snapshot sync");
3166
4188
  }
3167
4189
  if (firstPage.jobId) {
3168
- try {
3169
- const loadedResult = await this.client.jobs.load(firstPage.jobId);
3170
- const synced = await syncParseResultToParsedDocumentStorage({
3171
- result: loadedResult,
3172
- storage: params.parsedStorageConfig.storage,
3173
- documentId: params.documentId,
3174
- revisionKey,
3175
- source: "resultZip"
3176
- });
3177
- await params.parsedStorageConfig.storage.writeSyncProgress({
4190
+ const job = await this.client.jobs.get(firstPage.jobId).catch((error) => {
4191
+ if (error instanceof NotFoundError) return null;
4192
+ throw error;
4193
+ });
4194
+ if (job?.isDone && job.resultUrl) {
4195
+ if (job.documentId && job.documentId !== params.documentId)
4196
+ throw new ValidationError("Parsed job document identity differs from requested document");
4197
+ return syncRemoteResultArchive({
3178
4198
  documentId: params.documentId,
3179
4199
  revisionKey,
3180
- status: "completed",
3181
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
4200
+ jobId: firstPage.jobId,
4201
+ resultUrl: job.resultUrl,
4202
+ storage: params.parsedStorageConfig.storage,
4203
+ limits: params.parsedStorageConfig.limits,
4204
+ signal: params.signal,
4205
+ retainResultArchive: params.retainResultArchive,
4206
+ cacheResultArchive: params.cacheResultArchive,
4207
+ verifyRevision: async () => {
4208
+ const current = await this.client.documents.listChunks(params.documentId, {
4209
+ page: 1,
4210
+ pageSize: 1,
4211
+ includeAssetUrls: true
4212
+ });
4213
+ if (getRemoteRevisionKey(current) !== revisionKey || current.jobId !== firstPage.jobId)
4214
+ throw new ValidationError(
4215
+ "Published document revision changed during parsed snapshot sync"
4216
+ );
4217
+ }
3182
4218
  });
3183
- return {
3184
- documentId: synced.documentId,
3185
- revisionKey: synced.revisionKey,
3186
- completed: true,
3187
- commit: synced.commit
3188
- };
3189
- } catch {
3190
4219
  }
3191
4220
  }
3192
4221
  return this.syncRemoteDocumentChunksToParsedStorage({
3193
4222
  documentId: params.documentId,
3194
4223
  revisionKey,
3195
4224
  firstPage,
3196
- parsedStorageConfig: params.parsedStorageConfig
4225
+ parsedStorageConfig: params.parsedStorageConfig,
4226
+ signal: params.signal
3197
4227
  });
3198
4228
  }
3199
4229
  async syncRemoteDocumentChunksToParsedStorage(params) {
4230
+ params.signal?.throwIfAborted();
3200
4231
  const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
3201
4232
  const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
3202
4233
  documentId: params.documentId,
3203
4234
  revisionKey: params.revisionKey
3204
4235
  });
3205
- let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
4236
+ let nextPage = existingProgress?.status === "running" || existingProgress?.status === "failed" ? existingProgress.nextChunkPage ?? 1 : 1;
3206
4237
  let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
3207
4238
  page: nextPage,
3208
4239
  pageSize: limits.remotePageSize,
3209
4240
  includeAssetUrls: true
3210
4241
  });
4242
+ validateRemoteReconstructionRevision(
4243
+ params.documentId,
4244
+ params.revisionKey,
4245
+ params.firstPage.jobId,
4246
+ response
4247
+ );
4248
+ params.signal?.throwIfAborted();
3211
4249
  const totalPages = response.pagination.totalPages;
3212
4250
  let syncedPages = 0;
3213
4251
  const startedAt = Date.now();
@@ -3219,6 +4257,13 @@ var Knowledge = class _Knowledge {
3219
4257
  includeAssetUrls: true
3220
4258
  });
3221
4259
  }
4260
+ params.signal?.throwIfAborted();
4261
+ validateRemoteReconstructionRevision(
4262
+ params.documentId,
4263
+ params.revisionKey,
4264
+ params.firstPage.jobId,
4265
+ response
4266
+ );
3222
4267
  await params.parsedStorageConfig.storage.writeObject({
3223
4268
  documentId: params.documentId,
3224
4269
  revisionKey: params.revisionKey,
@@ -3266,7 +4311,23 @@ var Knowledge = class _Knowledge {
3266
4311
  storage: params.parsedStorageConfig.storage,
3267
4312
  documentId: params.documentId,
3268
4313
  revisionKey: params.revisionKey,
3269
- source: "remoteReconstruction"
4314
+ source: "remoteReconstruction",
4315
+ signal: params.signal,
4316
+ beforeCommit: async () => {
4317
+ params.signal?.throwIfAborted();
4318
+ const current = await this.client.documents.listChunks(params.documentId, {
4319
+ page: 1,
4320
+ pageSize: 1,
4321
+ includeAssetUrls: true
4322
+ });
4323
+ params.signal?.throwIfAborted();
4324
+ validateRemoteReconstructionRevision(
4325
+ params.documentId,
4326
+ params.revisionKey,
4327
+ params.firstPage.jobId,
4328
+ current
4329
+ );
4330
+ }
3270
4331
  });
3271
4332
  await params.parsedStorageConfig.storage.writeSyncProgress({
3272
4333
  documentId: params.documentId,
@@ -3338,7 +4399,7 @@ function createLocalDocumentIdForRemote(documentId) {
3338
4399
  if (SAFE_LOCAL_DOCUMENT_ID_PATTERN.test(documentId) && !documentId.includes("..") && !documentId.includes("/") && !documentId.includes("\\")) {
3339
4400
  return documentId;
3340
4401
  }
3341
- const hash = (0, import_crypto2.createHash)("sha256").update(documentId).digest("hex").slice(0, 16);
4402
+ const hash = (0, import_crypto4.createHash)("sha256").update(documentId).digest("hex").slice(0, 16);
3342
4403
  return `remote_${hash}`;
3343
4404
  }
3344
4405
  var defaultParsedStorageScheduler = {
@@ -3558,10 +4619,10 @@ function parseStoredResultChunks(params) {
3558
4619
  }
3559
4620
  function extractStoredChunkRecords(payload) {
3560
4621
  if (Array.isArray(payload)) {
3561
- return payload.filter(isRecord2);
4622
+ return payload.filter(isRecord3);
3562
4623
  }
3563
- if (isRecord2(payload) && Array.isArray(payload.chunks)) {
3564
- return payload.chunks.filter(isRecord2);
4624
+ if (isRecord3(payload) && Array.isArray(payload.chunks)) {
4625
+ return payload.chunks.filter(isRecord3);
3565
4626
  }
3566
4627
  return [];
3567
4628
  }
@@ -3581,7 +4642,7 @@ function readRecordNumber(record, ...keys) {
3581
4642
  }
3582
4643
  function readRecordMetadata(record) {
3583
4644
  const metadata = record.metadata;
3584
- return isRecord2(metadata) ? keysToCamel(metadata) : {};
4645
+ return isRecord3(metadata) ? keysToCamel(metadata) : {};
3585
4646
  }
3586
4647
  function isKnowledgeChunkType(value) {
3587
4648
  return value === "text" || value === "image" || value === "table" || value === "page";
@@ -3680,7 +4741,7 @@ async function applyStoredPageAssetUrls(params) {
3680
4741
  }
3681
4742
  return Promise.all(
3682
4743
  pageAssets.map(async (pageAsset) => {
3683
- if (!isRecord2(pageAsset)) {
4744
+ if (!isRecord3(pageAsset)) {
3684
4745
  return pageAsset;
3685
4746
  }
3686
4747
  const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
@@ -3709,7 +4770,7 @@ function getStoredPageAssets(metadata) {
3709
4770
  return pageAssets?.map(normalizeStoredPageAsset);
3710
4771
  }
3711
4772
  function normalizeStoredPageAsset(pageAsset) {
3712
- if (!isRecord2(pageAsset)) {
4773
+ if (!isRecord3(pageAsset)) {
3713
4774
  return pageAsset;
3714
4775
  }
3715
4776
  const normalizedPageAsset = { ...pageAsset };
@@ -3787,7 +4848,7 @@ function parseContinuationCursor(value) {
3787
4848
  }
3788
4849
  try {
3789
4850
  const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
3790
- if (!isRecord2(parsed)) {
4851
+ if (!isRecord3(parsed)) {
3791
4852
  return null;
3792
4853
  }
3793
4854
  const documentId = parsed.documentId;
@@ -3825,11 +4886,11 @@ function inferContentTypeFromPath2(filePath) {
3825
4886
  if (extension === "html" || extension === "htm") return "text/html; charset=utf-8";
3826
4887
  return "application/octet-stream";
3827
4888
  }
3828
- function isRecord2(value) {
4889
+ function isRecord3(value) {
3829
4890
  return typeof value === "object" && value !== null && !Array.isArray(value);
3830
4891
  }
3831
4892
  function isDocumentChunkArray(value) {
3832
- return Array.isArray(value) && value.every(isRecord2);
4893
+ return Array.isArray(value) && value.every(isRecord3);
3833
4894
  }
3834
4895
  function indexChunks(result) {
3835
4896
  return result.chunks.map((chunk, index) => {
@@ -4132,17 +5193,25 @@ function toRemoteSearchResult(result, documentByServerId) {
4132
5193
  sourceFileName: result.source.sourceFileName ?? void 0
4133
5194
  };
4134
5195
  }
5196
+ function validateRemoteReconstructionRevision(documentId, revisionKey, jobId, response) {
5197
+ if (response.documentId !== documentId || getRemoteRevisionKey(response) !== revisionKey || (response.jobId ?? void 0) !== (jobId ?? void 0)) {
5198
+ throw new ValidationError("Published document revision changed during remote reconstruction");
5199
+ }
5200
+ }
4135
5201
 
4136
5202
  // src/client.ts
5203
+ init_constants();
5204
+ init_errors();
5205
+ init_utils();
4137
5206
  function inferFileName(file, explicitFileName) {
4138
5207
  if (explicitFileName) {
4139
5208
  return explicitFileName;
4140
5209
  }
4141
5210
  if (typeof file === "string") {
4142
- return import_path3.default.basename(file);
5211
+ return import_path4.default.basename(file);
4143
5212
  }
4144
5213
  if (isReadStream2(file) && typeof file.path === "string") {
4145
- return import_path3.default.basename(file.path);
5214
+ return import_path4.default.basename(file.path);
4146
5215
  }
4147
5216
  return void 0;
4148
5217
  }
@@ -4288,9 +5357,9 @@ var Knowhere = class {
4288
5357
  };
4289
5358
 
4290
5359
  // src/storage/disk-parsed-document-storage.ts
4291
- var import_crypto3 = require("crypto");
5360
+ var import_crypto5 = require("crypto");
4292
5361
  var import_fs4 = require("fs");
4293
- var import_path4 = __toESM(require("path"));
5362
+ var import_path5 = __toESM(require("path"));
4294
5363
  var syncProgressPath = ".knowhere-sdk/sync-progress.json";
4295
5364
  var DiskParsedDocumentStorage = class {
4296
5365
  rootDirectory;
@@ -4319,7 +5388,7 @@ var DiskParsedDocumentStorage = class {
4319
5388
  }
4320
5389
  async writeObject(params) {
4321
5390
  const objectPath = this.getObjectPath(params);
4322
- await import_fs4.promises.mkdir(import_path4.default.dirname(objectPath), { recursive: true });
5391
+ await import_fs4.promises.mkdir(import_path5.default.dirname(objectPath), { recursive: true });
4323
5392
  await import_fs4.promises.writeFile(objectPath, params.body);
4324
5393
  await this.writeObjectMetadata(objectPath, {
4325
5394
  contentType: params.contentType,
@@ -4378,14 +5447,14 @@ var DiskParsedDocumentStorage = class {
4378
5447
  }
4379
5448
  getObjectPath(params) {
4380
5449
  const revisionDirectory = this.getRevisionDirectory(params.documentId, params.revisionKey);
4381
- const objectPath = import_path4.default.resolve(revisionDirectory, normalizeRelativeStoragePath(params.path));
5450
+ const objectPath = import_path5.default.resolve(revisionDirectory, normalizeRelativeStoragePath(params.path));
4382
5451
  if (!isPathInsideDirectory2(objectPath, revisionDirectory)) {
4383
5452
  throw new Error(`Parsed document object path resolves outside storage: ${params.path}`);
4384
5453
  }
4385
5454
  return objectPath;
4386
5455
  }
4387
5456
  getRevisionDirectory(documentId, revisionKey) {
4388
- return import_path4.default.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey));
5457
+ return import_path5.default.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey));
4389
5458
  }
4390
5459
  async readOptionalJson(params) {
4391
5460
  const object = await this.readObject(params);
@@ -4421,7 +5490,7 @@ function hashPathPart(value) {
4421
5490
  if (normalized.length === 0) {
4422
5491
  throw new Error("Parsed document storage path part must be a non-empty value.");
4423
5492
  }
4424
- return (0, import_crypto3.createHash)("sha256").update(normalized).digest("hex").slice(0, 24);
5493
+ return (0, import_crypto5.createHash)("sha256").update(normalized).digest("hex").slice(0, 24);
4425
5494
  }
4426
5495
  function normalizeRelativeStoragePath(value) {
4427
5496
  const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
@@ -4431,8 +5500,8 @@ function normalizeRelativeStoragePath(value) {
4431
5500
  return normalized;
4432
5501
  }
4433
5502
  function isPathInsideDirectory2(filePath, directory) {
4434
- const relative = import_path4.default.relative(directory, filePath);
4435
- return relative === "" || !relative.startsWith("..") && !import_path4.default.isAbsolute(relative);
5503
+ const relative = import_path5.default.relative(directory, filePath);
5504
+ return relative === "" || !relative.startsWith("..") && !import_path5.default.isAbsolute(relative);
4436
5505
  }
4437
5506
  function toFileUrl(filePath) {
4438
5507
  return `file://${filePath}`;
@@ -4440,6 +5509,10 @@ function toFileUrl(filePath) {
4440
5509
  function isMissingFileError3(error) {
4441
5510
  return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
4442
5511
  }
5512
+
5513
+ // src/index.ts
5514
+ init_parsed_document_storage();
5515
+ init_errors();
4443
5516
  // Annotate the CommonJS export names for ESM import in node:
4444
5517
  0 && (module.exports = {
4445
5518
  APIError,
@@ -4461,6 +5534,7 @@ function isMissingFileError3(error) {
4461
5534
  NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
4462
5535
  NetworkError,
4463
5536
  NotFoundError,
5537
+ ParsedDocumentArchiveError,
4464
5538
  PaymentRequiredError,
4465
5539
  PermissionDeniedError,
4466
5540
  PollingTimeoutError,