@ontos-ai/knowhere-sdk 2.4.2 → 2.6.0-beta.20261026172248
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/chunk-FESTKAG6.mjs +1343 -0
- package/dist/index.d.mts +49 -3
- package/dist/index.d.ts +49 -3
- package/dist/index.js +1556 -482
- package/dist/index.mjs +145 -565
- package/dist/sequential-parsed-document-archive-YQ5U3VWM.mjs +143 -0
- package/package.json +5 -2
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,54 @@
|
|
|
1
|
+
import {
|
|
2
|
+
APIError,
|
|
3
|
+
AuthenticationError,
|
|
4
|
+
BadRequestError,
|
|
5
|
+
ChecksumError,
|
|
6
|
+
ConflictError,
|
|
7
|
+
DEFAULT_BASE_URL,
|
|
8
|
+
DEFAULT_MAX_RETRIES,
|
|
9
|
+
DEFAULT_POLL_INTERVAL,
|
|
10
|
+
DEFAULT_POLL_TIMEOUT,
|
|
11
|
+
DEFAULT_TIMEOUT,
|
|
12
|
+
ENV,
|
|
13
|
+
GatewayTimeoutError,
|
|
14
|
+
INITIAL_RETRY_DELAY,
|
|
15
|
+
InternalServerError,
|
|
16
|
+
InvalidStateError,
|
|
17
|
+
JobFailedError,
|
|
18
|
+
KnowhereError,
|
|
19
|
+
MAX_POLL_INTERVAL,
|
|
20
|
+
MAX_RETRY_DELAY,
|
|
21
|
+
NetworkError,
|
|
22
|
+
NotFoundError,
|
|
23
|
+
POLL_INTERVAL_INCREASE_THRESHOLD,
|
|
24
|
+
POLL_INTERVAL_MULTIPLIER,
|
|
25
|
+
ParsedDocumentArchiveError,
|
|
26
|
+
PaymentRequiredError,
|
|
27
|
+
PermissionDeniedError,
|
|
28
|
+
PollingTimeoutError,
|
|
29
|
+
RETRYABLE_STATUS_CODES,
|
|
30
|
+
RETRY_DELAY_BASE,
|
|
31
|
+
RateLimitError,
|
|
32
|
+
ServiceUnavailableError,
|
|
33
|
+
TimeoutError,
|
|
34
|
+
ValidationError,
|
|
35
|
+
createAPIError,
|
|
36
|
+
enrichJobResult,
|
|
37
|
+
enrichParseResult,
|
|
38
|
+
getFileExtension,
|
|
39
|
+
jitter,
|
|
40
|
+
keysToCamel,
|
|
41
|
+
keysToSnake,
|
|
42
|
+
parseDates,
|
|
43
|
+
readJsonObject,
|
|
44
|
+
readParsedDocumentCommit,
|
|
45
|
+
sanitizePath,
|
|
46
|
+
sleep,
|
|
47
|
+
syncParseResultToParsedDocumentStorage,
|
|
48
|
+
syncRemoteResultArchive,
|
|
49
|
+
validateArchiveSyncLimits
|
|
50
|
+
} from "./chunk-FESTKAG6.mjs";
|
|
51
|
+
|
|
1
52
|
// src/client.ts
|
|
2
53
|
import path2 from "path";
|
|
3
54
|
|
|
@@ -9,321 +60,6 @@ import axios, {
|
|
|
9
60
|
// src/version.ts
|
|
10
61
|
var VERSION = "2.1.3";
|
|
11
62
|
|
|
12
|
-
// src/constants.ts
|
|
13
|
-
var DEFAULT_BASE_URL = "https://api.knowhereto.ai";
|
|
14
|
-
var DEFAULT_TIMEOUT = 6e4;
|
|
15
|
-
var DEFAULT_MAX_RETRIES = 5;
|
|
16
|
-
var DEFAULT_POLL_INTERVAL = 1e4;
|
|
17
|
-
var MAX_POLL_INTERVAL = 3e4;
|
|
18
|
-
var DEFAULT_POLL_TIMEOUT = 18e5;
|
|
19
|
-
var POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
|
|
20
|
-
var POLL_INTERVAL_MULTIPLIER = 1.2;
|
|
21
|
-
var INITIAL_RETRY_DELAY = 500;
|
|
22
|
-
var MAX_RETRY_DELAY = 3e4;
|
|
23
|
-
var RETRY_DELAY_BASE = 2;
|
|
24
|
-
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
|
|
25
|
-
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
|
|
26
|
-
var ENV = {
|
|
27
|
-
API_KEY: "KNOWHERE_API_KEY",
|
|
28
|
-
BASE_URL: "KNOWHERE_BASE_URL",
|
|
29
|
-
LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
// src/errors/base.ts
|
|
33
|
-
var KnowhereError = class extends Error {
|
|
34
|
-
constructor(message) {
|
|
35
|
-
super(message);
|
|
36
|
-
this.name = "KnowhereError";
|
|
37
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
38
|
-
}
|
|
39
|
-
};
|
|
40
|
-
var NetworkError = class extends KnowhereError {
|
|
41
|
-
constructor(message, cause) {
|
|
42
|
-
super(message);
|
|
43
|
-
this.cause = cause;
|
|
44
|
-
this.name = "NetworkError";
|
|
45
|
-
}
|
|
46
|
-
cause;
|
|
47
|
-
};
|
|
48
|
-
var TimeoutError = class extends NetworkError {
|
|
49
|
-
constructor(message = "Request timed out") {
|
|
50
|
-
super(message);
|
|
51
|
-
this.name = "TimeoutError";
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
var PollingTimeoutError = class extends KnowhereError {
|
|
55
|
-
constructor(message = "Polling timed out", elapsedMs) {
|
|
56
|
-
super(message);
|
|
57
|
-
this.elapsedMs = elapsedMs;
|
|
58
|
-
this.name = "PollingTimeoutError";
|
|
59
|
-
}
|
|
60
|
-
elapsedMs;
|
|
61
|
-
};
|
|
62
|
-
var ChecksumError = class extends KnowhereError {
|
|
63
|
-
constructor(message = "Checksum verification failed", expected, actual) {
|
|
64
|
-
super(message);
|
|
65
|
-
this.expected = expected;
|
|
66
|
-
this.actual = actual;
|
|
67
|
-
this.name = "ChecksumError";
|
|
68
|
-
}
|
|
69
|
-
expected;
|
|
70
|
-
actual;
|
|
71
|
-
};
|
|
72
|
-
var ValidationError = class extends KnowhereError {
|
|
73
|
-
constructor(message) {
|
|
74
|
-
super(message);
|
|
75
|
-
this.name = "ValidationError";
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
|
-
var InvalidStateError = class extends KnowhereError {
|
|
79
|
-
constructor(message) {
|
|
80
|
-
super(message);
|
|
81
|
-
this.name = "InvalidStateError";
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
|
|
85
|
-
// src/errors/api-errors.ts
|
|
86
|
-
var APIError = class extends KnowhereError {
|
|
87
|
-
constructor(message, statusCode, code, requestId, details, body) {
|
|
88
|
-
super(message);
|
|
89
|
-
this.statusCode = statusCode;
|
|
90
|
-
this.code = code;
|
|
91
|
-
this.requestId = requestId;
|
|
92
|
-
this.details = details;
|
|
93
|
-
this.body = body;
|
|
94
|
-
this.name = "APIError";
|
|
95
|
-
}
|
|
96
|
-
statusCode;
|
|
97
|
-
code;
|
|
98
|
-
requestId;
|
|
99
|
-
details;
|
|
100
|
-
body;
|
|
101
|
-
};
|
|
102
|
-
var BadRequestError = class extends APIError {
|
|
103
|
-
constructor(message, code, requestId, details, body) {
|
|
104
|
-
super(message, 400, code, requestId, details, body);
|
|
105
|
-
this.name = "BadRequestError";
|
|
106
|
-
}
|
|
107
|
-
};
|
|
108
|
-
var AuthenticationError = class extends APIError {
|
|
109
|
-
constructor(message = "Authentication failed", code, requestId, details, body) {
|
|
110
|
-
super(message, 401, code, requestId, details, body);
|
|
111
|
-
this.name = "AuthenticationError";
|
|
112
|
-
}
|
|
113
|
-
};
|
|
114
|
-
var PaymentRequiredError = class extends APIError {
|
|
115
|
-
constructor(message = "Payment required", code, requestId, details, body) {
|
|
116
|
-
super(message, 402, code, requestId, details, body);
|
|
117
|
-
this.name = "PaymentRequiredError";
|
|
118
|
-
}
|
|
119
|
-
};
|
|
120
|
-
var PermissionDeniedError = class extends APIError {
|
|
121
|
-
constructor(message = "Permission denied", code, requestId, details, body) {
|
|
122
|
-
super(message, 403, code, requestId, details, body);
|
|
123
|
-
this.name = "PermissionDeniedError";
|
|
124
|
-
}
|
|
125
|
-
};
|
|
126
|
-
var NotFoundError = class extends APIError {
|
|
127
|
-
constructor(message = "Resource not found", code, requestId, details, body) {
|
|
128
|
-
super(message, 404, code, requestId, details, body);
|
|
129
|
-
this.name = "NotFoundError";
|
|
130
|
-
}
|
|
131
|
-
};
|
|
132
|
-
var ConflictError = class extends APIError {
|
|
133
|
-
constructor(message = "Conflict", code, requestId, details, body) {
|
|
134
|
-
super(message, 409, code, requestId, details, body);
|
|
135
|
-
this.name = "ConflictError";
|
|
136
|
-
}
|
|
137
|
-
};
|
|
138
|
-
var RateLimitError = class extends APIError {
|
|
139
|
-
constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
|
|
140
|
-
super(message, 429, code, requestId, details, body);
|
|
141
|
-
this.retryAfter = retryAfter;
|
|
142
|
-
this.name = "RateLimitError";
|
|
143
|
-
}
|
|
144
|
-
retryAfter;
|
|
145
|
-
};
|
|
146
|
-
var InternalServerError = class extends APIError {
|
|
147
|
-
constructor(message = "Internal server error", code, requestId, details, body) {
|
|
148
|
-
super(message, 500, code, requestId, details, body);
|
|
149
|
-
this.name = "InternalServerError";
|
|
150
|
-
}
|
|
151
|
-
};
|
|
152
|
-
var ServiceUnavailableError = class extends APIError {
|
|
153
|
-
constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
|
|
154
|
-
super(message, statusCode, code, requestId, details, body);
|
|
155
|
-
this.name = "ServiceUnavailableError";
|
|
156
|
-
}
|
|
157
|
-
};
|
|
158
|
-
var GatewayTimeoutError = class extends APIError {
|
|
159
|
-
constructor(message = "Gateway timeout", code, requestId, details, body) {
|
|
160
|
-
super(message, 504, code, requestId, details, body);
|
|
161
|
-
this.name = "GatewayTimeoutError";
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
|
-
function createAPIError(statusCode, message, code, requestId, details, body, retryAfter) {
|
|
165
|
-
switch (statusCode) {
|
|
166
|
-
case 400:
|
|
167
|
-
return new BadRequestError(message, code, requestId, details, body);
|
|
168
|
-
case 401:
|
|
169
|
-
return new AuthenticationError(message, code, requestId, details, body);
|
|
170
|
-
case 402:
|
|
171
|
-
return new PaymentRequiredError(message, code, requestId, details, body);
|
|
172
|
-
case 403:
|
|
173
|
-
return new PermissionDeniedError(message, code, requestId, details, body);
|
|
174
|
-
case 404:
|
|
175
|
-
return new NotFoundError(message, code, requestId, details, body);
|
|
176
|
-
case 409:
|
|
177
|
-
return new ConflictError(message, code, requestId, details, body);
|
|
178
|
-
case 429:
|
|
179
|
-
return new RateLimitError(message, code, requestId, details, body, retryAfter);
|
|
180
|
-
case 500:
|
|
181
|
-
return new InternalServerError(message, code, requestId, details, body);
|
|
182
|
-
case 502:
|
|
183
|
-
case 503:
|
|
184
|
-
return new ServiceUnavailableError(message, statusCode, code, requestId, details, body);
|
|
185
|
-
case 504:
|
|
186
|
-
return new GatewayTimeoutError(message, code, requestId, details, body);
|
|
187
|
-
default:
|
|
188
|
-
return new APIError(message, statusCode, code, requestId, details, body);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
// src/errors/job-errors.ts
|
|
193
|
-
var JobFailedError = class extends KnowhereError {
|
|
194
|
-
constructor(message, code, jobResult) {
|
|
195
|
-
super(message);
|
|
196
|
-
this.code = code;
|
|
197
|
-
this.jobResult = jobResult;
|
|
198
|
-
this.name = "JobFailedError";
|
|
199
|
-
}
|
|
200
|
-
code;
|
|
201
|
-
jobResult;
|
|
202
|
-
};
|
|
203
|
-
|
|
204
|
-
// src/lib/utils.ts
|
|
205
|
-
function sleep(ms) {
|
|
206
|
-
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
207
|
-
}
|
|
208
|
-
function snakeToCamel(str) {
|
|
209
|
-
return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
210
|
-
}
|
|
211
|
-
function camelToSnake(str) {
|
|
212
|
-
return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
213
|
-
}
|
|
214
|
-
function keysToCamel(obj) {
|
|
215
|
-
if (obj === null || obj === void 0) {
|
|
216
|
-
return obj;
|
|
217
|
-
}
|
|
218
|
-
if (Array.isArray(obj)) {
|
|
219
|
-
return obj.map((item) => keysToCamel(item));
|
|
220
|
-
}
|
|
221
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
222
|
-
const result = {};
|
|
223
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
224
|
-
result[snakeToCamel(key)] = keysToCamel(value);
|
|
225
|
-
}
|
|
226
|
-
return result;
|
|
227
|
-
}
|
|
228
|
-
return obj;
|
|
229
|
-
}
|
|
230
|
-
function keysToSnake(obj) {
|
|
231
|
-
if (obj === null || obj === void 0) {
|
|
232
|
-
return obj;
|
|
233
|
-
}
|
|
234
|
-
if (Array.isArray(obj)) {
|
|
235
|
-
return obj.map((item) => keysToSnake(item));
|
|
236
|
-
}
|
|
237
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
238
|
-
const result = {};
|
|
239
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
240
|
-
result[camelToSnake(key)] = keysToSnake(value);
|
|
241
|
-
}
|
|
242
|
-
return result;
|
|
243
|
-
}
|
|
244
|
-
return obj;
|
|
245
|
-
}
|
|
246
|
-
function parseDates(obj) {
|
|
247
|
-
if (obj === null || obj === void 0) {
|
|
248
|
-
return obj;
|
|
249
|
-
}
|
|
250
|
-
if (Array.isArray(obj)) {
|
|
251
|
-
return obj.map((item) => parseDates(item));
|
|
252
|
-
}
|
|
253
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
254
|
-
const result = {};
|
|
255
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
256
|
-
if ((key.endsWith("At") || key.endsWith("Date")) && typeof value === "string" && /^\d{4}-\d{2}-\d{2}T/.test(value)) {
|
|
257
|
-
result[key] = new Date(value);
|
|
258
|
-
} else {
|
|
259
|
-
result[key] = parseDates(value);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
return result;
|
|
263
|
-
}
|
|
264
|
-
return obj;
|
|
265
|
-
}
|
|
266
|
-
function isTerminalStatus(status) {
|
|
267
|
-
return TERMINAL_JOB_STATUSES.has(status);
|
|
268
|
-
}
|
|
269
|
-
function enrichJobResult(jobResult) {
|
|
270
|
-
const computedProperties = {};
|
|
271
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isTerminal")) {
|
|
272
|
-
computedProperties.isTerminal = {
|
|
273
|
-
get() {
|
|
274
|
-
return isTerminalStatus(this.status);
|
|
275
|
-
},
|
|
276
|
-
enumerable: true,
|
|
277
|
-
configurable: true
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isDone")) {
|
|
281
|
-
computedProperties.isDone = {
|
|
282
|
-
get() {
|
|
283
|
-
return this.status === "done";
|
|
284
|
-
},
|
|
285
|
-
enumerable: true,
|
|
286
|
-
configurable: true
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isFailed")) {
|
|
290
|
-
computedProperties.isFailed = {
|
|
291
|
-
get() {
|
|
292
|
-
return this.status === "failed";
|
|
293
|
-
},
|
|
294
|
-
enumerable: true,
|
|
295
|
-
configurable: true
|
|
296
|
-
};
|
|
297
|
-
}
|
|
298
|
-
if (Object.keys(computedProperties).length > 0) {
|
|
299
|
-
Object.defineProperties(jobResult, computedProperties);
|
|
300
|
-
}
|
|
301
|
-
return jobResult;
|
|
302
|
-
}
|
|
303
|
-
function enrichParseResult(parseResult2, scope) {
|
|
304
|
-
if (scope.namespace !== void 0) {
|
|
305
|
-
parseResult2.namespace = scope.namespace;
|
|
306
|
-
}
|
|
307
|
-
if (scope.documentId !== void 0) {
|
|
308
|
-
parseResult2.documentId = scope.documentId;
|
|
309
|
-
}
|
|
310
|
-
return parseResult2;
|
|
311
|
-
}
|
|
312
|
-
function sanitizePath(path4) {
|
|
313
|
-
let sanitized = path4.replace(/^\/+/, "");
|
|
314
|
-
sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
|
|
315
|
-
sanitized = sanitized.replace(/\\/g, "/");
|
|
316
|
-
return sanitized;
|
|
317
|
-
}
|
|
318
|
-
function getFileExtension(filename) {
|
|
319
|
-
const match = filename.match(/\.([^.]+)$/);
|
|
320
|
-
return match ? match[1].toLowerCase() : "";
|
|
321
|
-
}
|
|
322
|
-
function jitter(value, percent = 0.2) {
|
|
323
|
-
const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
|
|
324
|
-
return Math.round(value * randomFactor);
|
|
325
|
-
}
|
|
326
|
-
|
|
327
63
|
// src/lib/retry.ts
|
|
328
64
|
function getErrorCode(error) {
|
|
329
65
|
const errorWithResponse = error;
|
|
@@ -1806,215 +1542,6 @@ function isRecord(value) {
|
|
|
1806
1542
|
return typeof value === "object" && value !== null;
|
|
1807
1543
|
}
|
|
1808
1544
|
|
|
1809
|
-
// src/storage/parsed-document-storage.ts
|
|
1810
|
-
import JSZip3 from "jszip";
|
|
1811
|
-
var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
|
|
1812
|
-
var jsonContentType = "application/json; charset=utf-8";
|
|
1813
|
-
var markdownContentType = "text/markdown; charset=utf-8";
|
|
1814
|
-
var csvContentType = "text/csv; charset=utf-8";
|
|
1815
|
-
var htmlContentType = "text/html; charset=utf-8";
|
|
1816
|
-
var binaryContentType = "application/octet-stream";
|
|
1817
|
-
async function syncParseResultToParsedDocumentStorage(params) {
|
|
1818
|
-
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
1819
|
-
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
1820
|
-
const objects = await collectResultLayoutObjects(params.result);
|
|
1821
|
-
for (const object of objects) {
|
|
1822
|
-
await params.storage.writeObject({
|
|
1823
|
-
documentId,
|
|
1824
|
-
revisionKey,
|
|
1825
|
-
...object
|
|
1826
|
-
});
|
|
1827
|
-
}
|
|
1828
|
-
const commit = {
|
|
1829
|
-
version: 1,
|
|
1830
|
-
documentId,
|
|
1831
|
-
revisionKey,
|
|
1832
|
-
source: params.source ?? "resultZip",
|
|
1833
|
-
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1834
|
-
};
|
|
1835
|
-
await writeParsedDocumentJsonObject(params.storage, {
|
|
1836
|
-
documentId,
|
|
1837
|
-
revisionKey,
|
|
1838
|
-
path: parsedDocumentCommitPath,
|
|
1839
|
-
value: commit
|
|
1840
|
-
});
|
|
1841
|
-
return {
|
|
1842
|
-
result: params.result,
|
|
1843
|
-
documentId,
|
|
1844
|
-
revisionKey,
|
|
1845
|
-
commit
|
|
1846
|
-
};
|
|
1847
|
-
}
|
|
1848
|
-
async function readParsedDocumentCommit(params) {
|
|
1849
|
-
const object = await params.storage.readObject({
|
|
1850
|
-
documentId: params.documentId,
|
|
1851
|
-
revisionKey: params.revisionKey,
|
|
1852
|
-
path: parsedDocumentCommitPath
|
|
1853
|
-
});
|
|
1854
|
-
if (!object) {
|
|
1855
|
-
return null;
|
|
1856
|
-
}
|
|
1857
|
-
const commit = readJsonObject(object);
|
|
1858
|
-
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
1859
|
-
}
|
|
1860
|
-
function readJsonObject(object) {
|
|
1861
|
-
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
1862
|
-
}
|
|
1863
|
-
async function collectResultLayoutObjects(result) {
|
|
1864
|
-
const objectsByPath = /* @__PURE__ */ new Map();
|
|
1865
|
-
const rawZipObjects = await collectRawZipObjects(result.rawZip);
|
|
1866
|
-
for (const object of rawZipObjects) {
|
|
1867
|
-
objectsByPath.set(object.path, object);
|
|
1868
|
-
}
|
|
1869
|
-
for (const object of collectGeneratedResultObjects(result)) {
|
|
1870
|
-
if (!objectsByPath.has(object.path)) {
|
|
1871
|
-
objectsByPath.set(object.path, object);
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
|
|
1875
|
-
}
|
|
1876
|
-
async function collectRawZipObjects(rawZip) {
|
|
1877
|
-
try {
|
|
1878
|
-
const zip = await JSZip3.loadAsync(rawZip);
|
|
1879
|
-
const objects = [];
|
|
1880
|
-
for (const entry of Object.values(zip.files)) {
|
|
1881
|
-
if (entry.dir || entry.name === "result.zip") {
|
|
1882
|
-
continue;
|
|
1883
|
-
}
|
|
1884
|
-
objects.push({
|
|
1885
|
-
path: normalizeResultPath(entry.name),
|
|
1886
|
-
body: new Uint8Array(await entry.async("nodebuffer")),
|
|
1887
|
-
contentType: inferContentTypeFromPath(entry.name)
|
|
1888
|
-
});
|
|
1889
|
-
}
|
|
1890
|
-
return objects;
|
|
1891
|
-
} catch {
|
|
1892
|
-
return [];
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
function collectGeneratedResultObjects(result) {
|
|
1896
|
-
const objects = [
|
|
1897
|
-
createJsonObject("manifest.json", result.manifest),
|
|
1898
|
-
createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
|
|
1899
|
-
];
|
|
1900
|
-
if (result.docNav) {
|
|
1901
|
-
objects.push(createJsonObject("doc_nav.json", result.docNav));
|
|
1902
|
-
}
|
|
1903
|
-
if (result.fullMarkdown) {
|
|
1904
|
-
objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
|
|
1905
|
-
}
|
|
1906
|
-
if (result.chunksSlim) {
|
|
1907
|
-
objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
|
|
1908
|
-
}
|
|
1909
|
-
if (result.hierarchy) {
|
|
1910
|
-
objects.push(createJsonObject("hierarchy.json", result.hierarchy));
|
|
1911
|
-
}
|
|
1912
|
-
if (result.tocHierarchies) {
|
|
1913
|
-
objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
|
|
1914
|
-
}
|
|
1915
|
-
if (result.kbCsv) {
|
|
1916
|
-
objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
|
|
1917
|
-
}
|
|
1918
|
-
if (result.hierarchyViewHtml) {
|
|
1919
|
-
objects.push(
|
|
1920
|
-
createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
|
|
1921
|
-
);
|
|
1922
|
-
}
|
|
1923
|
-
for (const chunk of result.imageChunks) {
|
|
1924
|
-
objects.push({
|
|
1925
|
-
path: normalizeResultPath(chunk.filePath),
|
|
1926
|
-
body: new Uint8Array(chunk.data),
|
|
1927
|
-
contentType: inferContentTypeFromPath(chunk.filePath),
|
|
1928
|
-
metadata: {
|
|
1929
|
-
chunkId: chunk.chunkId,
|
|
1930
|
-
chunkType: chunk.type,
|
|
1931
|
-
sourcePath: chunk.filePath
|
|
1932
|
-
}
|
|
1933
|
-
});
|
|
1934
|
-
}
|
|
1935
|
-
for (const chunk of result.tableChunks) {
|
|
1936
|
-
objects.push({
|
|
1937
|
-
path: normalizeResultPath(chunk.filePath),
|
|
1938
|
-
body: Buffer.from(chunk.html, "utf8"),
|
|
1939
|
-
contentType: htmlContentType,
|
|
1940
|
-
metadata: {
|
|
1941
|
-
chunkId: chunk.chunkId,
|
|
1942
|
-
chunkType: chunk.type,
|
|
1943
|
-
sourcePath: chunk.filePath
|
|
1944
|
-
}
|
|
1945
|
-
});
|
|
1946
|
-
}
|
|
1947
|
-
return objects;
|
|
1948
|
-
}
|
|
1949
|
-
function serializeChunks2(chunks) {
|
|
1950
|
-
return {
|
|
1951
|
-
chunks: chunks.map((chunk) => {
|
|
1952
|
-
const rawChunk = {
|
|
1953
|
-
chunkId: chunk.chunkId,
|
|
1954
|
-
type: chunk.type,
|
|
1955
|
-
contentSource: chunk.contentSource,
|
|
1956
|
-
content: chunk.content,
|
|
1957
|
-
path: chunk.path,
|
|
1958
|
-
metadata: chunk.metadata
|
|
1959
|
-
};
|
|
1960
|
-
if (chunk.type === "image" || chunk.type === "table") {
|
|
1961
|
-
return {
|
|
1962
|
-
...rawChunk,
|
|
1963
|
-
filePath: chunk.filePath,
|
|
1964
|
-
assetUrl: chunk.assetUrl
|
|
1965
|
-
};
|
|
1966
|
-
}
|
|
1967
|
-
return rawChunk;
|
|
1968
|
-
})
|
|
1969
|
-
};
|
|
1970
|
-
}
|
|
1971
|
-
async function writeParsedDocumentJsonObject(storage, params) {
|
|
1972
|
-
await storage.writeObject({
|
|
1973
|
-
documentId: params.documentId,
|
|
1974
|
-
revisionKey: params.revisionKey,
|
|
1975
|
-
path: params.path,
|
|
1976
|
-
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
1977
|
-
contentType: jsonContentType
|
|
1978
|
-
});
|
|
1979
|
-
}
|
|
1980
|
-
function createJsonObject(path4, value) {
|
|
1981
|
-
return {
|
|
1982
|
-
path: path4,
|
|
1983
|
-
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
1984
|
-
contentType: jsonContentType
|
|
1985
|
-
};
|
|
1986
|
-
}
|
|
1987
|
-
function createTextObject(path4, value, contentType) {
|
|
1988
|
-
return {
|
|
1989
|
-
path: path4,
|
|
1990
|
-
body: Buffer.from(value, "utf8"),
|
|
1991
|
-
contentType
|
|
1992
|
-
};
|
|
1993
|
-
}
|
|
1994
|
-
function inferContentTypeFromPath(filePath) {
|
|
1995
|
-
const lower = filePath.toLowerCase();
|
|
1996
|
-
if (lower.endsWith(".json")) return jsonContentType;
|
|
1997
|
-
if (lower.endsWith(".md")) return markdownContentType;
|
|
1998
|
-
if (lower.endsWith(".csv")) return csvContentType;
|
|
1999
|
-
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
2000
|
-
if (lower.endsWith(".png")) return "image/png";
|
|
2001
|
-
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
2002
|
-
if (lower.endsWith(".gif")) return "image/gif";
|
|
2003
|
-
if (lower.endsWith(".webp")) return "image/webp";
|
|
2004
|
-
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
2005
|
-
return binaryContentType;
|
|
2006
|
-
}
|
|
2007
|
-
function normalizeResultPath(value) {
|
|
2008
|
-
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
2009
|
-
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
2010
|
-
throw new Error(`Invalid parsed result path: ${value}`);
|
|
2011
|
-
}
|
|
2012
|
-
return normalized;
|
|
2013
|
-
}
|
|
2014
|
-
function isSdkMetadataPath(path4) {
|
|
2015
|
-
return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
1545
|
// src/knowledge/local-store.ts
|
|
2019
1546
|
import { createHash } from "crypto";
|
|
2020
1547
|
import os from "os";
|
|
@@ -2354,29 +1881,35 @@ var Knowledge = class _Knowledge {
|
|
|
2354
1881
|
return this.importJobResult(params);
|
|
2355
1882
|
}
|
|
2356
1883
|
async syncParsedDocument(params) {
|
|
1884
|
+
params.signal?.throwIfAborted();
|
|
2357
1885
|
const normalized = normalizeDocumentReference(params);
|
|
2358
1886
|
const parsedStorageConfig = this.requireParsedStorageConfig();
|
|
1887
|
+
validateArchiveSyncLimits(parsedStorageConfig.limits);
|
|
2359
1888
|
if (normalized.documentId) {
|
|
2360
1889
|
return this.syncRemoteDocumentToParsedStorage({
|
|
2361
1890
|
documentId: normalized.documentId,
|
|
2362
1891
|
revisionKey: params.revisionKey,
|
|
2363
|
-
parsedStorageConfig
|
|
1892
|
+
parsedStorageConfig,
|
|
1893
|
+
signal: params.signal,
|
|
1894
|
+
retainResultArchive: params.retainResultArchive,
|
|
1895
|
+
cacheResultArchive: params.cacheResultArchive
|
|
2364
1896
|
});
|
|
2365
1897
|
}
|
|
2366
1898
|
if (normalized.jobId) {
|
|
2367
|
-
const
|
|
2368
|
-
|
|
2369
|
-
result
|
|
1899
|
+
const job = await this.client.jobs.get(normalized.jobId);
|
|
1900
|
+
if (!job.isDone || !job.resultUrl)
|
|
1901
|
+
throw new ValidationError("Completed job result archive is unavailable");
|
|
1902
|
+
return syncRemoteResultArchive({
|
|
1903
|
+
documentId: job.documentId ?? normalized.localDocumentId ?? normalized.jobId,
|
|
1904
|
+
revisionKey: params.revisionKey ?? normalized.jobId,
|
|
1905
|
+
jobId: normalized.jobId,
|
|
1906
|
+
resultUrl: job.resultUrl,
|
|
2370
1907
|
storage: parsedStorageConfig.storage,
|
|
2371
|
-
|
|
2372
|
-
|
|
1908
|
+
limits: parsedStorageConfig.limits,
|
|
1909
|
+
signal: params.signal,
|
|
1910
|
+
retainResultArchive: params.retainResultArchive,
|
|
1911
|
+
cacheResultArchive: params.cacheResultArchive
|
|
2373
1912
|
});
|
|
2374
|
-
return {
|
|
2375
|
-
documentId: synced2.documentId,
|
|
2376
|
-
revisionKey: synced2.revisionKey,
|
|
2377
|
-
completed: true,
|
|
2378
|
-
commit: synced2.commit
|
|
2379
|
-
};
|
|
2380
1913
|
}
|
|
2381
1914
|
if (!normalized.localDocumentId) {
|
|
2382
1915
|
throw new ValidationError("localDocumentId, documentId, or jobId is required");
|
|
@@ -3074,10 +2607,14 @@ var Knowledge = class _Knowledge {
|
|
|
3074
2607
|
parsedStorageConfig
|
|
3075
2608
|
});
|
|
3076
2609
|
} catch (error) {
|
|
2610
|
+
const previous = await parsedStorageConfig.storage.readSyncProgress({
|
|
2611
|
+
documentId,
|
|
2612
|
+
revisionKey
|
|
2613
|
+
});
|
|
3077
2614
|
await parsedStorageConfig.storage.writeSyncProgress({
|
|
2615
|
+
...previous,
|
|
3078
2616
|
documentId,
|
|
3079
2617
|
revisionKey,
|
|
3080
|
-
nextChunkPage: 1,
|
|
3081
2618
|
status: "failed",
|
|
3082
2619
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3083
2620
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -3087,6 +2624,7 @@ var Knowledge = class _Knowledge {
|
|
|
3087
2624
|
void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
|
|
3088
2625
|
}
|
|
3089
2626
|
async syncRemoteDocumentToParsedStorage(params) {
|
|
2627
|
+
params.signal?.throwIfAborted();
|
|
3090
2628
|
const firstPage = await this.client.documents.listChunks(params.documentId, {
|
|
3091
2629
|
page: 1,
|
|
3092
2630
|
pageSize: 1,
|
|
@@ -3094,55 +2632,68 @@ var Knowledge = class _Knowledge {
|
|
|
3094
2632
|
});
|
|
3095
2633
|
const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
|
|
3096
2634
|
if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
|
|
3097
|
-
|
|
3098
|
-
documentId: params.documentId,
|
|
3099
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
3100
|
-
});
|
|
2635
|
+
throw new ValidationError("Published document revision changed before parsed snapshot sync");
|
|
3101
2636
|
}
|
|
3102
2637
|
if (firstPage.jobId) {
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
});
|
|
3112
|
-
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
2638
|
+
const job = await this.client.jobs.get(firstPage.jobId).catch((error) => {
|
|
2639
|
+
if (error instanceof NotFoundError) return null;
|
|
2640
|
+
throw error;
|
|
2641
|
+
});
|
|
2642
|
+
if (job?.isDone && job.resultUrl) {
|
|
2643
|
+
if (job.documentId && job.documentId !== params.documentId)
|
|
2644
|
+
throw new ValidationError("Parsed job document identity differs from requested document");
|
|
2645
|
+
return syncRemoteResultArchive({
|
|
3113
2646
|
documentId: params.documentId,
|
|
3114
2647
|
revisionKey,
|
|
3115
|
-
|
|
3116
|
-
|
|
2648
|
+
jobId: firstPage.jobId,
|
|
2649
|
+
resultUrl: job.resultUrl,
|
|
2650
|
+
storage: params.parsedStorageConfig.storage,
|
|
2651
|
+
limits: params.parsedStorageConfig.limits,
|
|
2652
|
+
signal: params.signal,
|
|
2653
|
+
retainResultArchive: params.retainResultArchive,
|
|
2654
|
+
cacheResultArchive: params.cacheResultArchive,
|
|
2655
|
+
verifyRevision: async () => {
|
|
2656
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
2657
|
+
page: 1,
|
|
2658
|
+
pageSize: 1,
|
|
2659
|
+
includeAssetUrls: true
|
|
2660
|
+
});
|
|
2661
|
+
if (getRemoteRevisionKey(current) !== revisionKey || current.jobId !== firstPage.jobId)
|
|
2662
|
+
throw new ValidationError(
|
|
2663
|
+
"Published document revision changed during parsed snapshot sync"
|
|
2664
|
+
);
|
|
2665
|
+
}
|
|
3117
2666
|
});
|
|
3118
|
-
return {
|
|
3119
|
-
documentId: synced.documentId,
|
|
3120
|
-
revisionKey: synced.revisionKey,
|
|
3121
|
-
completed: true,
|
|
3122
|
-
commit: synced.commit
|
|
3123
|
-
};
|
|
3124
|
-
} catch {
|
|
3125
2667
|
}
|
|
3126
2668
|
}
|
|
3127
2669
|
return this.syncRemoteDocumentChunksToParsedStorage({
|
|
3128
2670
|
documentId: params.documentId,
|
|
3129
2671
|
revisionKey,
|
|
3130
2672
|
firstPage,
|
|
3131
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
2673
|
+
parsedStorageConfig: params.parsedStorageConfig,
|
|
2674
|
+
signal: params.signal
|
|
3132
2675
|
});
|
|
3133
2676
|
}
|
|
3134
2677
|
async syncRemoteDocumentChunksToParsedStorage(params) {
|
|
2678
|
+
params.signal?.throwIfAborted();
|
|
3135
2679
|
const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
|
|
3136
2680
|
const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
|
|
3137
2681
|
documentId: params.documentId,
|
|
3138
2682
|
revisionKey: params.revisionKey
|
|
3139
2683
|
});
|
|
3140
|
-
let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
2684
|
+
let nextPage = existingProgress?.status === "running" || existingProgress?.status === "failed" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
3141
2685
|
let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
|
|
3142
2686
|
page: nextPage,
|
|
3143
2687
|
pageSize: limits.remotePageSize,
|
|
3144
2688
|
includeAssetUrls: true
|
|
3145
2689
|
});
|
|
2690
|
+
validateRemoteReconstructionRevision(
|
|
2691
|
+
params.documentId,
|
|
2692
|
+
params.revisionKey,
|
|
2693
|
+
params.firstPage.jobId,
|
|
2694
|
+
response
|
|
2695
|
+
);
|
|
2696
|
+
params.signal?.throwIfAborted();
|
|
3146
2697
|
const totalPages = response.pagination.totalPages;
|
|
3147
2698
|
let syncedPages = 0;
|
|
3148
2699
|
const startedAt = Date.now();
|
|
@@ -3154,6 +2705,13 @@ var Knowledge = class _Knowledge {
|
|
|
3154
2705
|
includeAssetUrls: true
|
|
3155
2706
|
});
|
|
3156
2707
|
}
|
|
2708
|
+
params.signal?.throwIfAborted();
|
|
2709
|
+
validateRemoteReconstructionRevision(
|
|
2710
|
+
params.documentId,
|
|
2711
|
+
params.revisionKey,
|
|
2712
|
+
params.firstPage.jobId,
|
|
2713
|
+
response
|
|
2714
|
+
);
|
|
3157
2715
|
await params.parsedStorageConfig.storage.writeObject({
|
|
3158
2716
|
documentId: params.documentId,
|
|
3159
2717
|
revisionKey: params.revisionKey,
|
|
@@ -3201,7 +2759,23 @@ var Knowledge = class _Knowledge {
|
|
|
3201
2759
|
storage: params.parsedStorageConfig.storage,
|
|
3202
2760
|
documentId: params.documentId,
|
|
3203
2761
|
revisionKey: params.revisionKey,
|
|
3204
|
-
source: "remoteReconstruction"
|
|
2762
|
+
source: "remoteReconstruction",
|
|
2763
|
+
signal: params.signal,
|
|
2764
|
+
beforeCommit: async () => {
|
|
2765
|
+
params.signal?.throwIfAborted();
|
|
2766
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
2767
|
+
page: 1,
|
|
2768
|
+
pageSize: 1,
|
|
2769
|
+
includeAssetUrls: true
|
|
2770
|
+
});
|
|
2771
|
+
params.signal?.throwIfAborted();
|
|
2772
|
+
validateRemoteReconstructionRevision(
|
|
2773
|
+
params.documentId,
|
|
2774
|
+
params.revisionKey,
|
|
2775
|
+
params.firstPage.jobId,
|
|
2776
|
+
current
|
|
2777
|
+
);
|
|
2778
|
+
}
|
|
3205
2779
|
});
|
|
3206
2780
|
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3207
2781
|
documentId: params.documentId,
|
|
@@ -3401,7 +2975,7 @@ function toMinimalChunkFromRemoteChunk(chunk) {
|
|
|
3401
2975
|
filePath: chunk.filePath ?? "",
|
|
3402
2976
|
assetUrl: chunk.assetUrl ?? void 0,
|
|
3403
2977
|
data: Buffer.alloc(0),
|
|
3404
|
-
format:
|
|
2978
|
+
format: inferContentTypeFromPath(chunk.filePath ?? "").replace("image/", ""),
|
|
3405
2979
|
save: () => Promise.resolve(chunk.filePath ?? "")
|
|
3406
2980
|
};
|
|
3407
2981
|
}
|
|
@@ -3753,7 +3327,7 @@ function cloneMetadata(metadata) {
|
|
|
3753
3327
|
function toUnknownArray(value) {
|
|
3754
3328
|
return Array.isArray(value) ? value : null;
|
|
3755
3329
|
}
|
|
3756
|
-
function
|
|
3330
|
+
function inferContentTypeFromPath(filePath) {
|
|
3757
3331
|
const extension = filePath.split(".").at(-1)?.toLowerCase();
|
|
3758
3332
|
if (extension === "png") return "image/png";
|
|
3759
3333
|
if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
|
|
@@ -4067,6 +3641,11 @@ function toRemoteSearchResult(result, documentByServerId) {
|
|
|
4067
3641
|
sourceFileName: result.source.sourceFileName ?? void 0
|
|
4068
3642
|
};
|
|
4069
3643
|
}
|
|
3644
|
+
function validateRemoteReconstructionRevision(documentId, revisionKey, jobId, response) {
|
|
3645
|
+
if (response.documentId !== documentId || getRemoteRevisionKey(response) !== revisionKey || (response.jobId ?? void 0) !== (jobId ?? void 0)) {
|
|
3646
|
+
throw new ValidationError("Published document revision changed during remote reconstruction");
|
|
3647
|
+
}
|
|
3648
|
+
}
|
|
4070
3649
|
|
|
4071
3650
|
// src/client.ts
|
|
4072
3651
|
function inferFileName(file, explicitFileName) {
|
|
@@ -4395,6 +3974,7 @@ export {
|
|
|
4395
3974
|
NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
|
|
4396
3975
|
NetworkError,
|
|
4397
3976
|
NotFoundError,
|
|
3977
|
+
ParsedDocumentArchiveError,
|
|
4398
3978
|
PaymentRequiredError,
|
|
4399
3979
|
PermissionDeniedError,
|
|
4400
3980
|
PollingTimeoutError,
|