@ontos-ai/knowhere-sdk 2.4.1 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/chunk-4PWLXJGH.mjs +719 -0
- package/dist/index.d.mts +45 -8
- package/dist/index.d.ts +45 -8
- package/dist/index.js +1051 -513
- package/dist/index.mjs +127 -446
- package/dist/sequential-parsed-document-archive-S56UEARH.mjs +139 -0
- package/package.json +5 -2
|
@@ -0,0 +1,719 @@
|
|
|
1
|
+
// src/errors/base.ts
|
|
2
|
+
var KnowhereError = class extends Error {
|
|
3
|
+
constructor(message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = "KnowhereError";
|
|
6
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
7
|
+
}
|
|
8
|
+
};
|
|
9
|
+
var NetworkError = class extends KnowhereError {
|
|
10
|
+
constructor(message, cause) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.cause = cause;
|
|
13
|
+
this.name = "NetworkError";
|
|
14
|
+
}
|
|
15
|
+
cause;
|
|
16
|
+
};
|
|
17
|
+
var TimeoutError = class extends NetworkError {
|
|
18
|
+
constructor(message = "Request timed out") {
|
|
19
|
+
super(message);
|
|
20
|
+
this.name = "TimeoutError";
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var PollingTimeoutError = class extends KnowhereError {
|
|
24
|
+
constructor(message = "Polling timed out", elapsedMs) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.elapsedMs = elapsedMs;
|
|
27
|
+
this.name = "PollingTimeoutError";
|
|
28
|
+
}
|
|
29
|
+
elapsedMs;
|
|
30
|
+
};
|
|
31
|
+
var ChecksumError = class extends KnowhereError {
|
|
32
|
+
constructor(message = "Checksum verification failed", expected, actual) {
|
|
33
|
+
super(message);
|
|
34
|
+
this.expected = expected;
|
|
35
|
+
this.actual = actual;
|
|
36
|
+
this.name = "ChecksumError";
|
|
37
|
+
}
|
|
38
|
+
expected;
|
|
39
|
+
actual;
|
|
40
|
+
};
|
|
41
|
+
var ValidationError = class extends KnowhereError {
|
|
42
|
+
constructor(message) {
|
|
43
|
+
super(message);
|
|
44
|
+
this.name = "ValidationError";
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
var InvalidStateError = class extends KnowhereError {
|
|
48
|
+
constructor(message) {
|
|
49
|
+
super(message);
|
|
50
|
+
this.name = "InvalidStateError";
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// src/errors/parsed-document-archive-error.ts
|
|
55
|
+
var ParsedDocumentArchiveError = class extends KnowhereError {
|
|
56
|
+
code = "ARCHIVE_SPOOL_FAILED";
|
|
57
|
+
cause;
|
|
58
|
+
constructor(cause) {
|
|
59
|
+
super(
|
|
60
|
+
"Parsed result ZIP temporary storage is unavailable. Range-enabled object storage avoids temporary disk usage."
|
|
61
|
+
);
|
|
62
|
+
this.name = "ParsedDocumentArchiveError";
|
|
63
|
+
this.cause = cause;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
// src/storage/parsed-document-storage.ts
|
|
68
|
+
import JSZip from "jszip";
|
|
69
|
+
|
|
70
|
+
// src/constants.ts
|
|
71
|
+
var DEFAULT_BASE_URL = "https://api.knowhereto.ai";
|
|
72
|
+
var DEFAULT_TIMEOUT = 6e4;
|
|
73
|
+
var DEFAULT_MAX_RETRIES = 5;
|
|
74
|
+
var DEFAULT_POLL_INTERVAL = 1e4;
|
|
75
|
+
var MAX_POLL_INTERVAL = 3e4;
|
|
76
|
+
var DEFAULT_POLL_TIMEOUT = 18e5;
|
|
77
|
+
var POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
|
|
78
|
+
var POLL_INTERVAL_MULTIPLIER = 1.2;
|
|
79
|
+
var INITIAL_RETRY_DELAY = 500;
|
|
80
|
+
var MAX_RETRY_DELAY = 3e4;
|
|
81
|
+
var RETRY_DELAY_BASE = 2;
|
|
82
|
+
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
|
|
83
|
+
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
|
|
84
|
+
var ENV = {
|
|
85
|
+
API_KEY: "KNOWHERE_API_KEY",
|
|
86
|
+
BASE_URL: "KNOWHERE_BASE_URL",
|
|
87
|
+
LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
// src/lib/utils.ts
|
|
91
|
+
function sleep(ms) {
|
|
92
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
93
|
+
}
|
|
94
|
+
function snakeToCamel(str) {
|
|
95
|
+
return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
96
|
+
}
|
|
97
|
+
function camelToSnake(str) {
|
|
98
|
+
return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
99
|
+
}
|
|
100
|
+
function keysToCamel(obj) {
|
|
101
|
+
if (obj === null || obj === void 0) {
|
|
102
|
+
return obj;
|
|
103
|
+
}
|
|
104
|
+
if (Array.isArray(obj)) {
|
|
105
|
+
return obj.map((item) => keysToCamel(item));
|
|
106
|
+
}
|
|
107
|
+
if (typeof obj === "object" && obj.constructor === Object) {
|
|
108
|
+
const result = {};
|
|
109
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
110
|
+
result[snakeToCamel(key)] = keysToCamel(value);
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
114
|
+
return obj;
|
|
115
|
+
}
|
|
116
|
+
function keysToSnake(obj) {
|
|
117
|
+
if (obj === null || obj === void 0) {
|
|
118
|
+
return obj;
|
|
119
|
+
}
|
|
120
|
+
if (Array.isArray(obj)) {
|
|
121
|
+
return obj.map((item) => keysToSnake(item));
|
|
122
|
+
}
|
|
123
|
+
if (typeof obj === "object" && obj.constructor === Object) {
|
|
124
|
+
const result = {};
|
|
125
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
126
|
+
result[camelToSnake(key)] = keysToSnake(value);
|
|
127
|
+
}
|
|
128
|
+
return result;
|
|
129
|
+
}
|
|
130
|
+
return obj;
|
|
131
|
+
}
|
|
132
|
+
function parseDates(obj) {
|
|
133
|
+
if (obj === null || obj === void 0) {
|
|
134
|
+
return obj;
|
|
135
|
+
}
|
|
136
|
+
if (Array.isArray(obj)) {
|
|
137
|
+
return obj.map((item) => parseDates(item));
|
|
138
|
+
}
|
|
139
|
+
if (typeof obj === "object" && obj.constructor === Object) {
|
|
140
|
+
const result = {};
|
|
141
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
142
|
+
if ((key.endsWith("At") || key.endsWith("Date")) && typeof value === "string" && /^\d{4}-\d{2}-\d{2}T/.test(value)) {
|
|
143
|
+
result[key] = new Date(value);
|
|
144
|
+
} else {
|
|
145
|
+
result[key] = parseDates(value);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return result;
|
|
149
|
+
}
|
|
150
|
+
return obj;
|
|
151
|
+
}
|
|
152
|
+
function isTerminalStatus(status) {
|
|
153
|
+
return TERMINAL_JOB_STATUSES.has(status);
|
|
154
|
+
}
|
|
155
|
+
function enrichJobResult(jobResult) {
|
|
156
|
+
const computedProperties = {};
|
|
157
|
+
if (!Object.getOwnPropertyDescriptor(jobResult, "isTerminal")) {
|
|
158
|
+
computedProperties.isTerminal = {
|
|
159
|
+
get() {
|
|
160
|
+
return isTerminalStatus(this.status);
|
|
161
|
+
},
|
|
162
|
+
enumerable: true,
|
|
163
|
+
configurable: true
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
if (!Object.getOwnPropertyDescriptor(jobResult, "isDone")) {
|
|
167
|
+
computedProperties.isDone = {
|
|
168
|
+
get() {
|
|
169
|
+
return this.status === "done";
|
|
170
|
+
},
|
|
171
|
+
enumerable: true,
|
|
172
|
+
configurable: true
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
if (!Object.getOwnPropertyDescriptor(jobResult, "isFailed")) {
|
|
176
|
+
computedProperties.isFailed = {
|
|
177
|
+
get() {
|
|
178
|
+
return this.status === "failed";
|
|
179
|
+
},
|
|
180
|
+
enumerable: true,
|
|
181
|
+
configurable: true
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
if (Object.keys(computedProperties).length > 0) {
|
|
185
|
+
Object.defineProperties(jobResult, computedProperties);
|
|
186
|
+
}
|
|
187
|
+
return jobResult;
|
|
188
|
+
}
|
|
189
|
+
function enrichParseResult(parseResult, scope) {
|
|
190
|
+
if (scope.namespace !== void 0) {
|
|
191
|
+
parseResult.namespace = scope.namespace;
|
|
192
|
+
}
|
|
193
|
+
if (scope.documentId !== void 0) {
|
|
194
|
+
parseResult.documentId = scope.documentId;
|
|
195
|
+
}
|
|
196
|
+
return parseResult;
|
|
197
|
+
}
|
|
198
|
+
function sanitizePath(path) {
|
|
199
|
+
let sanitized = path.replace(/^\/+/, "");
|
|
200
|
+
sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
|
|
201
|
+
sanitized = sanitized.replace(/\\/g, "/");
|
|
202
|
+
return sanitized;
|
|
203
|
+
}
|
|
204
|
+
function getFileExtension(filename) {
|
|
205
|
+
const match = filename.match(/\.([^.]+)$/);
|
|
206
|
+
return match ? match[1].toLowerCase() : "";
|
|
207
|
+
}
|
|
208
|
+
function jitter(value, percent = 0.2) {
|
|
209
|
+
const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
|
|
210
|
+
return Math.round(value * randomFactor);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// src/storage/parsed-document-storage.ts
|
|
214
|
+
var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
|
|
215
|
+
var jsonContentType = "application/json; charset=utf-8";
|
|
216
|
+
var markdownContentType = "text/markdown; charset=utf-8";
|
|
217
|
+
var csvContentType = "text/csv; charset=utf-8";
|
|
218
|
+
var htmlContentType = "text/html; charset=utf-8";
|
|
219
|
+
var binaryContentType = "application/octet-stream";
|
|
220
|
+
async function syncParseResultToParsedDocumentStorage(params) {
|
|
221
|
+
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
222
|
+
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
223
|
+
for await (const object of collectResultLayoutObjects(params.result)) {
|
|
224
|
+
params.signal?.throwIfAborted();
|
|
225
|
+
await params.storage.writeObject({
|
|
226
|
+
documentId,
|
|
227
|
+
revisionKey,
|
|
228
|
+
...object
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
await params.beforeCommit?.();
|
|
232
|
+
params.signal?.throwIfAborted();
|
|
233
|
+
const commit = {
|
|
234
|
+
version: 1,
|
|
235
|
+
documentId,
|
|
236
|
+
revisionKey,
|
|
237
|
+
source: params.source ?? "resultZip",
|
|
238
|
+
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
239
|
+
};
|
|
240
|
+
await writeParsedDocumentJsonObject(params.storage, {
|
|
241
|
+
documentId,
|
|
242
|
+
revisionKey,
|
|
243
|
+
path: parsedDocumentCommitPath,
|
|
244
|
+
value: commit
|
|
245
|
+
});
|
|
246
|
+
return {
|
|
247
|
+
result: params.result,
|
|
248
|
+
documentId,
|
|
249
|
+
revisionKey,
|
|
250
|
+
commit
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
async function readParsedDocumentCommit(params) {
|
|
254
|
+
const object = await params.storage.readObject({
|
|
255
|
+
documentId: params.documentId,
|
|
256
|
+
revisionKey: params.revisionKey,
|
|
257
|
+
path: parsedDocumentCommitPath
|
|
258
|
+
});
|
|
259
|
+
if (!object) {
|
|
260
|
+
return null;
|
|
261
|
+
}
|
|
262
|
+
const commit = readJsonObject(object);
|
|
263
|
+
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
264
|
+
}
|
|
265
|
+
function readJsonObject(object) {
|
|
266
|
+
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
267
|
+
}
|
|
268
|
+
async function* collectResultLayoutObjects(result) {
|
|
269
|
+
const storedPaths = /* @__PURE__ */ new Set();
|
|
270
|
+
let zip;
|
|
271
|
+
try {
|
|
272
|
+
zip = await JSZip.loadAsync(result.rawZip);
|
|
273
|
+
} catch {
|
|
274
|
+
}
|
|
275
|
+
if (zip) {
|
|
276
|
+
for (const entry of Object.values(zip.files)) {
|
|
277
|
+
if (entry.dir || entry.name === "result.zip") continue;
|
|
278
|
+
const path = normalizeResultPath(entry.name);
|
|
279
|
+
if (isSdkMetadataPath(path)) continue;
|
|
280
|
+
storedPaths.add(path);
|
|
281
|
+
yield {
|
|
282
|
+
path,
|
|
283
|
+
body: await entry.async("nodebuffer"),
|
|
284
|
+
contentType: inferContentTypeFromPath(path)
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
for (const object of generateMissingResultObjects(result, storedPaths)) yield object;
|
|
289
|
+
}
|
|
290
|
+
function* generateMissingResultObjects(result, storedPaths) {
|
|
291
|
+
if (!storedPaths.has("manifest.json")) yield createJsonObject("manifest.json", result.manifest);
|
|
292
|
+
if (!storedPaths.has("chunks.json"))
|
|
293
|
+
yield createJsonObject("chunks.json", keysToSnake(serializeChunks(result.chunks)));
|
|
294
|
+
if (!storedPaths.has("doc_nav.json") && result.docNav)
|
|
295
|
+
yield createJsonObject("doc_nav.json", result.docNav);
|
|
296
|
+
if (!storedPaths.has("full.md") && result.fullMarkdown)
|
|
297
|
+
yield createTextObject("full.md", result.fullMarkdown, markdownContentType);
|
|
298
|
+
if (!storedPaths.has("chunks_slim.json") && result.chunksSlim)
|
|
299
|
+
yield createJsonObject("chunks_slim.json", { chunks: result.chunksSlim });
|
|
300
|
+
if (!storedPaths.has("hierarchy.json") && result.hierarchy)
|
|
301
|
+
yield createJsonObject("hierarchy.json", result.hierarchy);
|
|
302
|
+
if (!storedPaths.has("toc_hierarchies.json") && result.tocHierarchies)
|
|
303
|
+
yield createJsonObject("toc_hierarchies.json", result.tocHierarchies);
|
|
304
|
+
if (!storedPaths.has("kb.csv") && result.kbCsv)
|
|
305
|
+
yield createTextObject("kb.csv", result.kbCsv, csvContentType);
|
|
306
|
+
if (!storedPaths.has("hierarchy_view.html") && result.hierarchyViewHtml)
|
|
307
|
+
yield createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType);
|
|
308
|
+
for (const chunk of result.imageChunks) {
|
|
309
|
+
const path = normalizeResultPath(chunk.filePath);
|
|
310
|
+
if (storedPaths.has(path)) continue;
|
|
311
|
+
yield {
|
|
312
|
+
path,
|
|
313
|
+
body: chunk.data,
|
|
314
|
+
contentType: inferContentTypeFromPath(path),
|
|
315
|
+
metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
for (const chunk of result.tableChunks) {
|
|
319
|
+
const path = normalizeResultPath(chunk.filePath);
|
|
320
|
+
if (storedPaths.has(path)) continue;
|
|
321
|
+
yield {
|
|
322
|
+
path,
|
|
323
|
+
body: Buffer.from(chunk.html, "utf8"),
|
|
324
|
+
contentType: htmlContentType,
|
|
325
|
+
metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
function serializeChunks(chunks) {
|
|
330
|
+
return {
|
|
331
|
+
chunks: chunks.map((chunk) => {
|
|
332
|
+
const rawChunk = {
|
|
333
|
+
chunkId: chunk.chunkId,
|
|
334
|
+
type: chunk.type,
|
|
335
|
+
contentSource: chunk.contentSource,
|
|
336
|
+
content: chunk.content,
|
|
337
|
+
path: chunk.path,
|
|
338
|
+
metadata: chunk.metadata
|
|
339
|
+
};
|
|
340
|
+
if (chunk.type === "image" || chunk.type === "table") {
|
|
341
|
+
return {
|
|
342
|
+
...rawChunk,
|
|
343
|
+
filePath: chunk.filePath,
|
|
344
|
+
assetUrl: chunk.assetUrl
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
return rawChunk;
|
|
348
|
+
})
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
async function writeParsedDocumentJsonObject(storage, params) {
|
|
352
|
+
await storage.writeObject({
|
|
353
|
+
documentId: params.documentId,
|
|
354
|
+
revisionKey: params.revisionKey,
|
|
355
|
+
path: params.path,
|
|
356
|
+
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
357
|
+
contentType: jsonContentType
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
function createJsonObject(path, value) {
|
|
361
|
+
return {
|
|
362
|
+
path,
|
|
363
|
+
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
364
|
+
contentType: jsonContentType
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
function createTextObject(path, value, contentType) {
|
|
368
|
+
return {
|
|
369
|
+
path,
|
|
370
|
+
body: Buffer.from(value, "utf8"),
|
|
371
|
+
contentType
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
function inferContentTypeFromPath(filePath) {
|
|
375
|
+
const lower = filePath.toLowerCase();
|
|
376
|
+
if (lower.endsWith(".json")) return jsonContentType;
|
|
377
|
+
if (lower.endsWith(".md")) return markdownContentType;
|
|
378
|
+
if (lower.endsWith(".csv")) return csvContentType;
|
|
379
|
+
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
380
|
+
if (lower.endsWith(".png")) return "image/png";
|
|
381
|
+
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
382
|
+
if (lower.endsWith(".gif")) return "image/gif";
|
|
383
|
+
if (lower.endsWith(".webp")) return "image/webp";
|
|
384
|
+
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
385
|
+
return binaryContentType;
|
|
386
|
+
}
|
|
387
|
+
function normalizeResultPath(value) {
|
|
388
|
+
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
389
|
+
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
390
|
+
throw new Error(`Invalid parsed result path: ${value}`);
|
|
391
|
+
}
|
|
392
|
+
return normalized;
|
|
393
|
+
}
|
|
394
|
+
function isSdkMetadataPath(path) {
|
|
395
|
+
return path === ".knowhere-sdk" || path.startsWith(".knowhere-sdk/");
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// src/storage/streamed-parsed-document-sync.ts
|
|
399
|
+
import { createHash } from "crypto";
|
|
400
|
+
import { HttpRangeReader, ZipReader } from "@zip.js/zip.js";
|
|
401
|
+
async function syncRemoteResultArchive(params) {
|
|
402
|
+
params.signal?.throwIfAborted();
|
|
403
|
+
const startedAt = Date.now();
|
|
404
|
+
const maxEntries = Math.max(1, params.limits?.maxEntriesPerSync ?? 100);
|
|
405
|
+
const deadlineMs = Math.max(1, params.limits?.archiveSyncDeadlineMs ?? 12e4);
|
|
406
|
+
const previous = await params.storage.readSyncProgress(params);
|
|
407
|
+
const source = await openArchive(params);
|
|
408
|
+
const entryHashes = [];
|
|
409
|
+
const paths = /* @__PURE__ */ new Set();
|
|
410
|
+
let entryIndex = 0;
|
|
411
|
+
let copiedEntries = 0;
|
|
412
|
+
const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
|
|
413
|
+
try {
|
|
414
|
+
if (previous?.archiveIdentity && previous.archiveIdentity !== source.identity) {
|
|
415
|
+
throw new Error("Parsed result archive identity changed; start a new revision.");
|
|
416
|
+
}
|
|
417
|
+
if (previous?.pinnedRevisionKey && previous.pinnedRevisionKey !== params.revisionKey) {
|
|
418
|
+
throw new Error("Parsed result revision changed while resuming archive sync.");
|
|
419
|
+
}
|
|
420
|
+
for await (const entry of source.entries) {
|
|
421
|
+
params.signal?.throwIfAborted();
|
|
422
|
+
if (paths.has(entry.path)) throw new Error(`Duplicate parsed result path: ${entry.path}`);
|
|
423
|
+
paths.add(entry.path);
|
|
424
|
+
if (entryIndex >= nextEntry && copiedEntries > 0 && (copiedEntries >= maxEntries || Date.now() - startedAt >= deadlineMs)) {
|
|
425
|
+
return { documentId: params.documentId, revisionKey: params.revisionKey, completed: false };
|
|
426
|
+
}
|
|
427
|
+
if (entryIndex < nextEntry && source.canSkipEntries) {
|
|
428
|
+
const previousHash = previous?.archiveEntryHashes?.[entryIndex];
|
|
429
|
+
if (!previousHash) throw new Error("Parsed archive checkpoint is missing an entry hash.");
|
|
430
|
+
entryHashes.push(previousHash);
|
|
431
|
+
entryIndex += 1;
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
const hash = createHash("sha256");
|
|
435
|
+
const body = entry.read().pipeThrough(
|
|
436
|
+
new TransformStream({
|
|
437
|
+
transform(chunk, controller) {
|
|
438
|
+
params.signal?.throwIfAborted();
|
|
439
|
+
hash.update(chunk);
|
|
440
|
+
controller.enqueue(chunk);
|
|
441
|
+
}
|
|
442
|
+
})
|
|
443
|
+
);
|
|
444
|
+
if (entryIndex < nextEntry) {
|
|
445
|
+
await body.pipeTo(new WritableStream(), { signal: params.signal });
|
|
446
|
+
} else {
|
|
447
|
+
await writeStream(params.storage, {
|
|
448
|
+
documentId: params.documentId,
|
|
449
|
+
revisionKey: params.revisionKey,
|
|
450
|
+
path: entry.path,
|
|
451
|
+
body,
|
|
452
|
+
contentType: inferContentTypeFromPath(entry.path),
|
|
453
|
+
signal: params.signal
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
const entryHash = createHash("sha256").update(entry.path).update(hash.digest()).digest("hex");
|
|
457
|
+
if (entryIndex < nextEntry && previous?.archiveEntryHashes?.[entryIndex] !== entryHash) {
|
|
458
|
+
throw new Error("Parsed result archive entry changed while resuming.");
|
|
459
|
+
}
|
|
460
|
+
entryHashes.push(entryHash);
|
|
461
|
+
entryIndex += 1;
|
|
462
|
+
if (entryIndex <= nextEntry) continue;
|
|
463
|
+
copiedEntries += 1;
|
|
464
|
+
await params.storage.writeSyncProgress({
|
|
465
|
+
documentId: params.documentId,
|
|
466
|
+
revisionKey: params.revisionKey,
|
|
467
|
+
pinnedRevisionKey: params.revisionKey,
|
|
468
|
+
archiveIdentity: source.identity,
|
|
469
|
+
archiveEntryHashes: [...entryHashes],
|
|
470
|
+
nextArchiveEntry: entryIndex,
|
|
471
|
+
status: "running",
|
|
472
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
await validateRequiredObjects(params, paths);
|
|
476
|
+
await params.verifyRevision?.();
|
|
477
|
+
params.signal?.throwIfAborted();
|
|
478
|
+
const commit = {
|
|
479
|
+
version: 1,
|
|
480
|
+
documentId: params.documentId,
|
|
481
|
+
revisionKey: params.revisionKey,
|
|
482
|
+
source: "resultZip",
|
|
483
|
+
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
484
|
+
};
|
|
485
|
+
await params.storage.writeObject({
|
|
486
|
+
documentId: params.documentId,
|
|
487
|
+
revisionKey: params.revisionKey,
|
|
488
|
+
path: parsedDocumentCommitPath,
|
|
489
|
+
body: Buffer.from(JSON.stringify(commit, null, 2)),
|
|
490
|
+
contentType: "application/json; charset=utf-8"
|
|
491
|
+
});
|
|
492
|
+
await params.storage.writeSyncProgress({
|
|
493
|
+
documentId: params.documentId,
|
|
494
|
+
revisionKey: params.revisionKey,
|
|
495
|
+
pinnedRevisionKey: params.revisionKey,
|
|
496
|
+
archiveIdentity: source.identity,
|
|
497
|
+
archiveEntryHashes: entryHashes,
|
|
498
|
+
nextArchiveEntry: entryIndex,
|
|
499
|
+
status: "completed",
|
|
500
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
501
|
+
});
|
|
502
|
+
if (params.retainResultArchive) {
|
|
503
|
+
try {
|
|
504
|
+
await retainArchive(params, source.downloadIdentity);
|
|
505
|
+
} catch {
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
documentId: params.documentId,
|
|
510
|
+
revisionKey: params.revisionKey,
|
|
511
|
+
completed: true,
|
|
512
|
+
commit
|
|
513
|
+
};
|
|
514
|
+
} finally {
|
|
515
|
+
await source.close();
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
async function openArchive(params) {
|
|
519
|
+
const probe = await fetch(params.resultUrl, {
|
|
520
|
+
headers: { Range: "bytes=0-0" },
|
|
521
|
+
signal: params.signal
|
|
522
|
+
});
|
|
523
|
+
if (!probe.ok) {
|
|
524
|
+
await probe.body?.cancel();
|
|
525
|
+
throw new Error(`Parsed result download failed (HTTP ${probe.status}).`);
|
|
526
|
+
}
|
|
527
|
+
const identity = createDownloadIdentity(params.jobId, probe);
|
|
528
|
+
const supportsRange = probe.status === 206 && probe.headers.has("content-range");
|
|
529
|
+
await probe.body?.cancel();
|
|
530
|
+
if (!supportsRange) {
|
|
531
|
+
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-S56UEARH.mjs");
|
|
532
|
+
return openSequentialArchive(params, identity);
|
|
533
|
+
}
|
|
534
|
+
const options = {
|
|
535
|
+
forceRangeRequests: true,
|
|
536
|
+
preventHeadRequest: true,
|
|
537
|
+
checkResourceChanges: true,
|
|
538
|
+
fetch: async (input, init) => {
|
|
539
|
+
const response = await fetch(input, { ...init, signal: params.signal });
|
|
540
|
+
if (response.status !== 206) {
|
|
541
|
+
await response.body?.cancel();
|
|
542
|
+
throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
|
|
543
|
+
}
|
|
544
|
+
return response;
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
const reader = new ZipReader(new HttpRangeReader(params.resultUrl, options), {
|
|
548
|
+
useWebWorkers: false,
|
|
549
|
+
checkCrc32: true
|
|
550
|
+
});
|
|
551
|
+
try {
|
|
552
|
+
const entries = await reader.getEntries();
|
|
553
|
+
const files = entries.filter(
|
|
554
|
+
(entry) => !entry.directory && isSnapshotPath(entry.filename)
|
|
555
|
+
);
|
|
556
|
+
const archiveIdentity = createHash("sha256").update(identity).update(
|
|
557
|
+
JSON.stringify(
|
|
558
|
+
files.map((entry) => [entry.filename, entry.uncompressedSize, entry.signature])
|
|
559
|
+
)
|
|
560
|
+
).digest("hex");
|
|
561
|
+
return {
|
|
562
|
+
identity: archiveIdentity,
|
|
563
|
+
downloadIdentity: identity,
|
|
564
|
+
canSkipEntries: true,
|
|
565
|
+
entries: (function* () {
|
|
566
|
+
for (const entry of files)
|
|
567
|
+
yield {
|
|
568
|
+
path: normalizeResultPath(entry.filename),
|
|
569
|
+
read: () => readRangeEntry(entry, params.signal)
|
|
570
|
+
};
|
|
571
|
+
})(),
|
|
572
|
+
close: async () => {
|
|
573
|
+
await reader.close();
|
|
574
|
+
}
|
|
575
|
+
};
|
|
576
|
+
} catch (error) {
|
|
577
|
+
await reader.close();
|
|
578
|
+
throw error;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
function readRangeEntry(entry, signal) {
|
|
582
|
+
const stream = new TransformStream();
|
|
583
|
+
void entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).catch(async (error) => {
|
|
584
|
+
try {
|
|
585
|
+
await stream.writable.abort(error);
|
|
586
|
+
} catch {
|
|
587
|
+
}
|
|
588
|
+
});
|
|
589
|
+
return stream.readable;
|
|
590
|
+
}
|
|
591
|
+
async function writeStream(storage, params) {
|
|
592
|
+
if (storage.writeObjectStream) {
|
|
593
|
+
try {
|
|
594
|
+
await storage.writeObjectStream(params);
|
|
595
|
+
} catch (error) {
|
|
596
|
+
await params.body.cancel(error).catch(() => void 0);
|
|
597
|
+
throw error;
|
|
598
|
+
}
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
const parts = [];
|
|
602
|
+
for await (const part of params.body) {
|
|
603
|
+
params.signal?.throwIfAborted();
|
|
604
|
+
parts.push(part);
|
|
605
|
+
}
|
|
606
|
+
await storage.writeObject({ ...params, body: Buffer.concat(parts) });
|
|
607
|
+
}
|
|
608
|
+
async function validateRequiredObjects(params, paths) {
|
|
609
|
+
const required = /* @__PURE__ */ new Set(["manifest.json", "chunks.json"]);
|
|
610
|
+
for (const path of required)
|
|
611
|
+
if (!paths.has(path)) throw new Error(`Required parsed result file missing: ${path}`);
|
|
612
|
+
const manifestObject = await params.storage.readObject({ ...params, path: "manifest.json" });
|
|
613
|
+
const chunksObject = await params.storage.readObject({ ...params, path: "chunks.json" });
|
|
614
|
+
if (!manifestObject || !chunksObject)
|
|
615
|
+
throw new Error("Required parsed result manifest.json or chunks.json was not persisted.");
|
|
616
|
+
const manifest = JSON.parse(Buffer.from(manifestObject.body).toString("utf8"));
|
|
617
|
+
const payload = JSON.parse(Buffer.from(chunksObject.body).toString("utf8"));
|
|
618
|
+
if (!isRecord(manifest)) throw new Error("Invalid parsed result manifest.json.");
|
|
619
|
+
const manifestJobId = manifest.job_id ?? manifest.jobId;
|
|
620
|
+
if (manifestJobId !== params.jobId)
|
|
621
|
+
throw new Error("Parsed result manifest job identity differs from the pinned job.");
|
|
622
|
+
const chunks = Array.isArray(payload) ? payload : isRecord(payload) ? payload.chunks : void 0;
|
|
623
|
+
if (!Array.isArray(chunks)) throw new Error("Invalid parsed result chunks.json.");
|
|
624
|
+
if (isRecord(manifest.files)) {
|
|
625
|
+
for (const path of Object.values(manifest.files))
|
|
626
|
+
if (typeof path === "string") required.add(normalizeResultPath(path));
|
|
627
|
+
}
|
|
628
|
+
for (const chunk of chunks) {
|
|
629
|
+
if (!isRecord(chunk)) throw new Error("Invalid parsed result chunk.");
|
|
630
|
+
const metadata = chunk.metadata;
|
|
631
|
+
const chunkType = chunk.type ?? chunk.chunk_type ?? chunk.chunkType;
|
|
632
|
+
const filePath = chunk.file_path ?? chunk.filePath ?? (isRecord(metadata) ? metadata.file_path ?? metadata.filePath : void 0) ?? (chunkType === "image" || chunkType === "table" ? chunk.path : void 0);
|
|
633
|
+
if (typeof filePath === "string") required.add(normalizeResultPath(filePath));
|
|
634
|
+
if (!isRecord(metadata)) continue;
|
|
635
|
+
const pageAssets = metadata.page_assets ?? metadata.pageAssets;
|
|
636
|
+
if (!Array.isArray(pageAssets)) continue;
|
|
637
|
+
for (const asset of pageAssets)
|
|
638
|
+
if (isRecord(asset)) {
|
|
639
|
+
const ref = asset.artifact_ref ?? asset.artifactRef;
|
|
640
|
+
if (typeof ref === "string") required.add(normalizeResultPath(ref));
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
for (const path of required)
|
|
644
|
+
if (!paths.has(path)) throw new Error(`Required parsed result file missing: ${path}`);
|
|
645
|
+
}
|
|
646
|
+
async function retainArchive(params, expectedIdentity) {
|
|
647
|
+
const response = await fetch(params.resultUrl, { signal: params.signal });
|
|
648
|
+
if (!response.ok || !response.body)
|
|
649
|
+
throw new Error(`Optional archive download failed (HTTP ${response.status}).`);
|
|
650
|
+
if (createDownloadIdentity(params.jobId, response) !== expectedIdentity) {
|
|
651
|
+
await response.body.cancel();
|
|
652
|
+
throw new Error("Optional result archive identity changed.");
|
|
653
|
+
}
|
|
654
|
+
await writeStream(params.storage, {
|
|
655
|
+
documentId: params.documentId,
|
|
656
|
+
revisionKey: params.revisionKey,
|
|
657
|
+
path: "result.zip",
|
|
658
|
+
body: response.body,
|
|
659
|
+
contentType: "application/zip",
|
|
660
|
+
signal: params.signal
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
function isSnapshotPath(path) {
|
|
664
|
+
const normalized = normalizeResultPath(path.replace(/\/$/, ""));
|
|
665
|
+
return normalized !== "result.zip" && normalized !== ".knowhere-sdk" && !normalized.startsWith(".knowhere-sdk/");
|
|
666
|
+
}
|
|
667
|
+
function isRecord(value) {
|
|
668
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
669
|
+
}
|
|
670
|
+
function createDownloadIdentity(jobId, response) {
|
|
671
|
+
return createHash("sha256").update(
|
|
672
|
+
JSON.stringify([
|
|
673
|
+
jobId,
|
|
674
|
+
response.headers.get("etag"),
|
|
675
|
+
response.headers.get("last-modified"),
|
|
676
|
+
response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
|
|
677
|
+
])
|
|
678
|
+
).digest("hex");
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
export {
|
|
682
|
+
DEFAULT_BASE_URL,
|
|
683
|
+
DEFAULT_TIMEOUT,
|
|
684
|
+
DEFAULT_MAX_RETRIES,
|
|
685
|
+
DEFAULT_POLL_INTERVAL,
|
|
686
|
+
MAX_POLL_INTERVAL,
|
|
687
|
+
DEFAULT_POLL_TIMEOUT,
|
|
688
|
+
POLL_INTERVAL_INCREASE_THRESHOLD,
|
|
689
|
+
POLL_INTERVAL_MULTIPLIER,
|
|
690
|
+
INITIAL_RETRY_DELAY,
|
|
691
|
+
MAX_RETRY_DELAY,
|
|
692
|
+
RETRY_DELAY_BASE,
|
|
693
|
+
RETRYABLE_STATUS_CODES,
|
|
694
|
+
ENV,
|
|
695
|
+
KnowhereError,
|
|
696
|
+
NetworkError,
|
|
697
|
+
TimeoutError,
|
|
698
|
+
PollingTimeoutError,
|
|
699
|
+
ChecksumError,
|
|
700
|
+
ValidationError,
|
|
701
|
+
InvalidStateError,
|
|
702
|
+
ParsedDocumentArchiveError,
|
|
703
|
+
sleep,
|
|
704
|
+
keysToCamel,
|
|
705
|
+
keysToSnake,
|
|
706
|
+
parseDates,
|
|
707
|
+
enrichJobResult,
|
|
708
|
+
enrichParseResult,
|
|
709
|
+
sanitizePath,
|
|
710
|
+
getFileExtension,
|
|
711
|
+
jitter,
|
|
712
|
+
syncParseResultToParsedDocumentStorage,
|
|
713
|
+
readParsedDocumentCommit,
|
|
714
|
+
readJsonObject,
|
|
715
|
+
normalizeResultPath,
|
|
716
|
+
syncRemoteResultArchive,
|
|
717
|
+
isSnapshotPath,
|
|
718
|
+
createDownloadIdentity
|
|
719
|
+
};
|