@ontos-ai/knowhere-sdk 2.4.1 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/chunk-4PWLXJGH.mjs +719 -0
- package/dist/index.d.mts +45 -8
- package/dist/index.d.ts +45 -8
- package/dist/index.js +1051 -513
- package/dist/index.mjs +127 -446
- package/dist/sequential-parsed-document-archive-S56UEARH.mjs +139 -0
- package/package.json +5 -2
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1,40 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ChecksumError,
|
|
3
|
+
DEFAULT_BASE_URL,
|
|
4
|
+
DEFAULT_MAX_RETRIES,
|
|
5
|
+
DEFAULT_POLL_INTERVAL,
|
|
6
|
+
DEFAULT_POLL_TIMEOUT,
|
|
7
|
+
DEFAULT_TIMEOUT,
|
|
8
|
+
ENV,
|
|
9
|
+
INITIAL_RETRY_DELAY,
|
|
10
|
+
InvalidStateError,
|
|
11
|
+
KnowhereError,
|
|
12
|
+
MAX_POLL_INTERVAL,
|
|
13
|
+
MAX_RETRY_DELAY,
|
|
14
|
+
NetworkError,
|
|
15
|
+
POLL_INTERVAL_INCREASE_THRESHOLD,
|
|
16
|
+
POLL_INTERVAL_MULTIPLIER,
|
|
17
|
+
ParsedDocumentArchiveError,
|
|
18
|
+
PollingTimeoutError,
|
|
19
|
+
RETRYABLE_STATUS_CODES,
|
|
20
|
+
RETRY_DELAY_BASE,
|
|
21
|
+
TimeoutError,
|
|
22
|
+
ValidationError,
|
|
23
|
+
enrichJobResult,
|
|
24
|
+
enrichParseResult,
|
|
25
|
+
getFileExtension,
|
|
26
|
+
jitter,
|
|
27
|
+
keysToCamel,
|
|
28
|
+
keysToSnake,
|
|
29
|
+
parseDates,
|
|
30
|
+
readJsonObject,
|
|
31
|
+
readParsedDocumentCommit,
|
|
32
|
+
sanitizePath,
|
|
33
|
+
sleep,
|
|
34
|
+
syncParseResultToParsedDocumentStorage,
|
|
35
|
+
syncRemoteResultArchive
|
|
36
|
+
} from "./chunk-4PWLXJGH.mjs";
|
|
37
|
+
|
|
1
38
|
// src/client.ts
|
|
2
39
|
import path2 from "path";
|
|
3
40
|
|
|
@@ -9,79 +46,6 @@ import axios, {
|
|
|
9
46
|
// src/version.ts
|
|
10
47
|
var VERSION = "2.1.3";
|
|
11
48
|
|
|
12
|
-
// src/constants.ts
|
|
13
|
-
var DEFAULT_BASE_URL = "https://api.knowhereto.ai";
|
|
14
|
-
var DEFAULT_TIMEOUT = 6e4;
|
|
15
|
-
var DEFAULT_MAX_RETRIES = 5;
|
|
16
|
-
var DEFAULT_POLL_INTERVAL = 1e4;
|
|
17
|
-
var MAX_POLL_INTERVAL = 3e4;
|
|
18
|
-
var DEFAULT_POLL_TIMEOUT = 18e5;
|
|
19
|
-
var POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
|
|
20
|
-
var POLL_INTERVAL_MULTIPLIER = 1.2;
|
|
21
|
-
var INITIAL_RETRY_DELAY = 500;
|
|
22
|
-
var MAX_RETRY_DELAY = 3e4;
|
|
23
|
-
var RETRY_DELAY_BASE = 2;
|
|
24
|
-
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
|
|
25
|
-
var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
|
|
26
|
-
var ENV = {
|
|
27
|
-
API_KEY: "KNOWHERE_API_KEY",
|
|
28
|
-
BASE_URL: "KNOWHERE_BASE_URL",
|
|
29
|
-
LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
// src/errors/base.ts
|
|
33
|
-
var KnowhereError = class extends Error {
|
|
34
|
-
constructor(message) {
|
|
35
|
-
super(message);
|
|
36
|
-
this.name = "KnowhereError";
|
|
37
|
-
Object.setPrototypeOf(this, new.target.prototype);
|
|
38
|
-
}
|
|
39
|
-
};
|
|
40
|
-
var NetworkError = class extends KnowhereError {
|
|
41
|
-
constructor(message, cause) {
|
|
42
|
-
super(message);
|
|
43
|
-
this.cause = cause;
|
|
44
|
-
this.name = "NetworkError";
|
|
45
|
-
}
|
|
46
|
-
cause;
|
|
47
|
-
};
|
|
48
|
-
var TimeoutError = class extends NetworkError {
|
|
49
|
-
constructor(message = "Request timed out") {
|
|
50
|
-
super(message);
|
|
51
|
-
this.name = "TimeoutError";
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
var PollingTimeoutError = class extends KnowhereError {
|
|
55
|
-
constructor(message = "Polling timed out", elapsedMs) {
|
|
56
|
-
super(message);
|
|
57
|
-
this.elapsedMs = elapsedMs;
|
|
58
|
-
this.name = "PollingTimeoutError";
|
|
59
|
-
}
|
|
60
|
-
elapsedMs;
|
|
61
|
-
};
|
|
62
|
-
var ChecksumError = class extends KnowhereError {
|
|
63
|
-
constructor(message = "Checksum verification failed", expected, actual) {
|
|
64
|
-
super(message);
|
|
65
|
-
this.expected = expected;
|
|
66
|
-
this.actual = actual;
|
|
67
|
-
this.name = "ChecksumError";
|
|
68
|
-
}
|
|
69
|
-
expected;
|
|
70
|
-
actual;
|
|
71
|
-
};
|
|
72
|
-
var ValidationError = class extends KnowhereError {
|
|
73
|
-
constructor(message) {
|
|
74
|
-
super(message);
|
|
75
|
-
this.name = "ValidationError";
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
|
-
var InvalidStateError = class extends KnowhereError {
|
|
79
|
-
constructor(message) {
|
|
80
|
-
super(message);
|
|
81
|
-
this.name = "InvalidStateError";
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
|
|
85
49
|
// src/errors/api-errors.ts
|
|
86
50
|
var APIError = class extends KnowhereError {
|
|
87
51
|
constructor(message, statusCode, code, requestId, details, body) {
|
|
@@ -201,129 +165,6 @@ var JobFailedError = class extends KnowhereError {
|
|
|
201
165
|
jobResult;
|
|
202
166
|
};
|
|
203
167
|
|
|
204
|
-
// src/lib/utils.ts
|
|
205
|
-
function sleep(ms) {
|
|
206
|
-
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
207
|
-
}
|
|
208
|
-
function snakeToCamel(str) {
|
|
209
|
-
return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
210
|
-
}
|
|
211
|
-
function camelToSnake(str) {
|
|
212
|
-
return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
|
|
213
|
-
}
|
|
214
|
-
function keysToCamel(obj) {
|
|
215
|
-
if (obj === null || obj === void 0) {
|
|
216
|
-
return obj;
|
|
217
|
-
}
|
|
218
|
-
if (Array.isArray(obj)) {
|
|
219
|
-
return obj.map((item) => keysToCamel(item));
|
|
220
|
-
}
|
|
221
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
222
|
-
const result = {};
|
|
223
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
224
|
-
result[snakeToCamel(key)] = keysToCamel(value);
|
|
225
|
-
}
|
|
226
|
-
return result;
|
|
227
|
-
}
|
|
228
|
-
return obj;
|
|
229
|
-
}
|
|
230
|
-
function keysToSnake(obj) {
|
|
231
|
-
if (obj === null || obj === void 0) {
|
|
232
|
-
return obj;
|
|
233
|
-
}
|
|
234
|
-
if (Array.isArray(obj)) {
|
|
235
|
-
return obj.map((item) => keysToSnake(item));
|
|
236
|
-
}
|
|
237
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
238
|
-
const result = {};
|
|
239
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
240
|
-
result[camelToSnake(key)] = keysToSnake(value);
|
|
241
|
-
}
|
|
242
|
-
return result;
|
|
243
|
-
}
|
|
244
|
-
return obj;
|
|
245
|
-
}
|
|
246
|
-
function parseDates(obj) {
|
|
247
|
-
if (obj === null || obj === void 0) {
|
|
248
|
-
return obj;
|
|
249
|
-
}
|
|
250
|
-
if (Array.isArray(obj)) {
|
|
251
|
-
return obj.map((item) => parseDates(item));
|
|
252
|
-
}
|
|
253
|
-
if (typeof obj === "object" && obj.constructor === Object) {
|
|
254
|
-
const result = {};
|
|
255
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
256
|
-
if ((key.endsWith("At") || key.endsWith("Date")) && typeof value === "string" && /^\d{4}-\d{2}-\d{2}T/.test(value)) {
|
|
257
|
-
result[key] = new Date(value);
|
|
258
|
-
} else {
|
|
259
|
-
result[key] = parseDates(value);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
return result;
|
|
263
|
-
}
|
|
264
|
-
return obj;
|
|
265
|
-
}
|
|
266
|
-
function isTerminalStatus(status) {
|
|
267
|
-
return TERMINAL_JOB_STATUSES.has(status);
|
|
268
|
-
}
|
|
269
|
-
function enrichJobResult(jobResult) {
|
|
270
|
-
const computedProperties = {};
|
|
271
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isTerminal")) {
|
|
272
|
-
computedProperties.isTerminal = {
|
|
273
|
-
get() {
|
|
274
|
-
return isTerminalStatus(this.status);
|
|
275
|
-
},
|
|
276
|
-
enumerable: true,
|
|
277
|
-
configurable: true
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isDone")) {
|
|
281
|
-
computedProperties.isDone = {
|
|
282
|
-
get() {
|
|
283
|
-
return this.status === "done";
|
|
284
|
-
},
|
|
285
|
-
enumerable: true,
|
|
286
|
-
configurable: true
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
if (!Object.getOwnPropertyDescriptor(jobResult, "isFailed")) {
|
|
290
|
-
computedProperties.isFailed = {
|
|
291
|
-
get() {
|
|
292
|
-
return this.status === "failed";
|
|
293
|
-
},
|
|
294
|
-
enumerable: true,
|
|
295
|
-
configurable: true
|
|
296
|
-
};
|
|
297
|
-
}
|
|
298
|
-
if (Object.keys(computedProperties).length > 0) {
|
|
299
|
-
Object.defineProperties(jobResult, computedProperties);
|
|
300
|
-
}
|
|
301
|
-
return jobResult;
|
|
302
|
-
}
|
|
303
|
-
function enrichParseResult(parseResult2, scope) {
|
|
304
|
-
if (scope.namespace !== void 0) {
|
|
305
|
-
parseResult2.namespace = scope.namespace;
|
|
306
|
-
}
|
|
307
|
-
if (scope.documentId !== void 0) {
|
|
308
|
-
parseResult2.documentId = scope.documentId;
|
|
309
|
-
}
|
|
310
|
-
return parseResult2;
|
|
311
|
-
}
|
|
312
|
-
function sanitizePath(path4) {
|
|
313
|
-
let sanitized = path4.replace(/^\/+/, "");
|
|
314
|
-
sanitized = sanitized.replace(/\.\.(\/|\\)/g, "");
|
|
315
|
-
sanitized = sanitized.replace(/\\/g, "/");
|
|
316
|
-
return sanitized;
|
|
317
|
-
}
|
|
318
|
-
function getFileExtension(filename) {
|
|
319
|
-
const match = filename.match(/\.([^.]+)$/);
|
|
320
|
-
return match ? match[1].toLowerCase() : "";
|
|
321
|
-
}
|
|
322
|
-
function jitter(value, percent = 0.2) {
|
|
323
|
-
const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
|
|
324
|
-
return Math.round(value * randomFactor);
|
|
325
|
-
}
|
|
326
|
-
|
|
327
168
|
// src/lib/retry.ts
|
|
328
169
|
function getErrorCode(error) {
|
|
329
170
|
const errorWithResponse = error;
|
|
@@ -1806,215 +1647,6 @@ function isRecord(value) {
|
|
|
1806
1647
|
return typeof value === "object" && value !== null;
|
|
1807
1648
|
}
|
|
1808
1649
|
|
|
1809
|
-
// src/storage/parsed-document-storage.ts
|
|
1810
|
-
import JSZip3 from "jszip";
|
|
1811
|
-
var parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
|
|
1812
|
-
var jsonContentType = "application/json; charset=utf-8";
|
|
1813
|
-
var markdownContentType = "text/markdown; charset=utf-8";
|
|
1814
|
-
var csvContentType = "text/csv; charset=utf-8";
|
|
1815
|
-
var htmlContentType = "text/html; charset=utf-8";
|
|
1816
|
-
var binaryContentType = "application/octet-stream";
|
|
1817
|
-
async function syncParseResultToParsedDocumentStorage(params) {
|
|
1818
|
-
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
1819
|
-
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
1820
|
-
const objects = await collectResultLayoutObjects(params.result);
|
|
1821
|
-
for (const object of objects) {
|
|
1822
|
-
await params.storage.writeObject({
|
|
1823
|
-
documentId,
|
|
1824
|
-
revisionKey,
|
|
1825
|
-
...object
|
|
1826
|
-
});
|
|
1827
|
-
}
|
|
1828
|
-
const commit = {
|
|
1829
|
-
version: 1,
|
|
1830
|
-
documentId,
|
|
1831
|
-
revisionKey,
|
|
1832
|
-
source: params.source ?? "resultZip",
|
|
1833
|
-
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1834
|
-
};
|
|
1835
|
-
await writeParsedDocumentJsonObject(params.storage, {
|
|
1836
|
-
documentId,
|
|
1837
|
-
revisionKey,
|
|
1838
|
-
path: parsedDocumentCommitPath,
|
|
1839
|
-
value: commit
|
|
1840
|
-
});
|
|
1841
|
-
return {
|
|
1842
|
-
result: params.result,
|
|
1843
|
-
documentId,
|
|
1844
|
-
revisionKey,
|
|
1845
|
-
commit
|
|
1846
|
-
};
|
|
1847
|
-
}
|
|
1848
|
-
async function readParsedDocumentCommit(params) {
|
|
1849
|
-
const object = await params.storage.readObject({
|
|
1850
|
-
documentId: params.documentId,
|
|
1851
|
-
revisionKey: params.revisionKey,
|
|
1852
|
-
path: parsedDocumentCommitPath
|
|
1853
|
-
});
|
|
1854
|
-
if (!object) {
|
|
1855
|
-
return null;
|
|
1856
|
-
}
|
|
1857
|
-
const commit = readJsonObject(object);
|
|
1858
|
-
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
1859
|
-
}
|
|
1860
|
-
function readJsonObject(object) {
|
|
1861
|
-
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
1862
|
-
}
|
|
1863
|
-
async function collectResultLayoutObjects(result) {
|
|
1864
|
-
const objectsByPath = /* @__PURE__ */ new Map();
|
|
1865
|
-
const rawZipObjects = await collectRawZipObjects(result.rawZip);
|
|
1866
|
-
for (const object of rawZipObjects) {
|
|
1867
|
-
objectsByPath.set(object.path, object);
|
|
1868
|
-
}
|
|
1869
|
-
for (const object of collectGeneratedResultObjects(result)) {
|
|
1870
|
-
if (!objectsByPath.has(object.path)) {
|
|
1871
|
-
objectsByPath.set(object.path, object);
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
|
|
1875
|
-
}
|
|
1876
|
-
async function collectRawZipObjects(rawZip) {
|
|
1877
|
-
try {
|
|
1878
|
-
const zip = await JSZip3.loadAsync(rawZip);
|
|
1879
|
-
const objects = [];
|
|
1880
|
-
for (const entry of Object.values(zip.files)) {
|
|
1881
|
-
if (entry.dir || entry.name === "result.zip") {
|
|
1882
|
-
continue;
|
|
1883
|
-
}
|
|
1884
|
-
objects.push({
|
|
1885
|
-
path: normalizeResultPath(entry.name),
|
|
1886
|
-
body: new Uint8Array(await entry.async("nodebuffer")),
|
|
1887
|
-
contentType: inferContentTypeFromPath(entry.name)
|
|
1888
|
-
});
|
|
1889
|
-
}
|
|
1890
|
-
return objects;
|
|
1891
|
-
} catch {
|
|
1892
|
-
return [];
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
function collectGeneratedResultObjects(result) {
|
|
1896
|
-
const objects = [
|
|
1897
|
-
createJsonObject("manifest.json", result.manifest),
|
|
1898
|
-
createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
|
|
1899
|
-
];
|
|
1900
|
-
if (result.docNav) {
|
|
1901
|
-
objects.push(createJsonObject("doc_nav.json", result.docNav));
|
|
1902
|
-
}
|
|
1903
|
-
if (result.fullMarkdown) {
|
|
1904
|
-
objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
|
|
1905
|
-
}
|
|
1906
|
-
if (result.chunksSlim) {
|
|
1907
|
-
objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
|
|
1908
|
-
}
|
|
1909
|
-
if (result.hierarchy) {
|
|
1910
|
-
objects.push(createJsonObject("hierarchy.json", result.hierarchy));
|
|
1911
|
-
}
|
|
1912
|
-
if (result.tocHierarchies) {
|
|
1913
|
-
objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
|
|
1914
|
-
}
|
|
1915
|
-
if (result.kbCsv) {
|
|
1916
|
-
objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
|
|
1917
|
-
}
|
|
1918
|
-
if (result.hierarchyViewHtml) {
|
|
1919
|
-
objects.push(
|
|
1920
|
-
createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
|
|
1921
|
-
);
|
|
1922
|
-
}
|
|
1923
|
-
for (const chunk of result.imageChunks) {
|
|
1924
|
-
objects.push({
|
|
1925
|
-
path: normalizeResultPath(chunk.filePath),
|
|
1926
|
-
body: new Uint8Array(chunk.data),
|
|
1927
|
-
contentType: inferContentTypeFromPath(chunk.filePath),
|
|
1928
|
-
metadata: {
|
|
1929
|
-
chunkId: chunk.chunkId,
|
|
1930
|
-
chunkType: chunk.type,
|
|
1931
|
-
sourcePath: chunk.filePath
|
|
1932
|
-
}
|
|
1933
|
-
});
|
|
1934
|
-
}
|
|
1935
|
-
for (const chunk of result.tableChunks) {
|
|
1936
|
-
objects.push({
|
|
1937
|
-
path: normalizeResultPath(chunk.filePath),
|
|
1938
|
-
body: Buffer.from(chunk.html, "utf8"),
|
|
1939
|
-
contentType: htmlContentType,
|
|
1940
|
-
metadata: {
|
|
1941
|
-
chunkId: chunk.chunkId,
|
|
1942
|
-
chunkType: chunk.type,
|
|
1943
|
-
sourcePath: chunk.filePath
|
|
1944
|
-
}
|
|
1945
|
-
});
|
|
1946
|
-
}
|
|
1947
|
-
return objects;
|
|
1948
|
-
}
|
|
1949
|
-
function serializeChunks2(chunks) {
|
|
1950
|
-
return {
|
|
1951
|
-
chunks: chunks.map((chunk) => {
|
|
1952
|
-
const rawChunk = {
|
|
1953
|
-
chunkId: chunk.chunkId,
|
|
1954
|
-
type: chunk.type,
|
|
1955
|
-
contentSource: chunk.contentSource,
|
|
1956
|
-
content: chunk.content,
|
|
1957
|
-
path: chunk.path,
|
|
1958
|
-
metadata: chunk.metadata
|
|
1959
|
-
};
|
|
1960
|
-
if (chunk.type === "image" || chunk.type === "table") {
|
|
1961
|
-
return {
|
|
1962
|
-
...rawChunk,
|
|
1963
|
-
filePath: chunk.filePath,
|
|
1964
|
-
assetUrl: chunk.assetUrl
|
|
1965
|
-
};
|
|
1966
|
-
}
|
|
1967
|
-
return rawChunk;
|
|
1968
|
-
})
|
|
1969
|
-
};
|
|
1970
|
-
}
|
|
1971
|
-
async function writeParsedDocumentJsonObject(storage, params) {
|
|
1972
|
-
await storage.writeObject({
|
|
1973
|
-
documentId: params.documentId,
|
|
1974
|
-
revisionKey: params.revisionKey,
|
|
1975
|
-
path: params.path,
|
|
1976
|
-
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
1977
|
-
contentType: jsonContentType
|
|
1978
|
-
});
|
|
1979
|
-
}
|
|
1980
|
-
function createJsonObject(path4, value) {
|
|
1981
|
-
return {
|
|
1982
|
-
path: path4,
|
|
1983
|
-
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
1984
|
-
contentType: jsonContentType
|
|
1985
|
-
};
|
|
1986
|
-
}
|
|
1987
|
-
function createTextObject(path4, value, contentType) {
|
|
1988
|
-
return {
|
|
1989
|
-
path: path4,
|
|
1990
|
-
body: Buffer.from(value, "utf8"),
|
|
1991
|
-
contentType
|
|
1992
|
-
};
|
|
1993
|
-
}
|
|
1994
|
-
function inferContentTypeFromPath(filePath) {
|
|
1995
|
-
const lower = filePath.toLowerCase();
|
|
1996
|
-
if (lower.endsWith(".json")) return jsonContentType;
|
|
1997
|
-
if (lower.endsWith(".md")) return markdownContentType;
|
|
1998
|
-
if (lower.endsWith(".csv")) return csvContentType;
|
|
1999
|
-
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
2000
|
-
if (lower.endsWith(".png")) return "image/png";
|
|
2001
|
-
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
2002
|
-
if (lower.endsWith(".gif")) return "image/gif";
|
|
2003
|
-
if (lower.endsWith(".webp")) return "image/webp";
|
|
2004
|
-
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
2005
|
-
return binaryContentType;
|
|
2006
|
-
}
|
|
2007
|
-
function normalizeResultPath(value) {
|
|
2008
|
-
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
2009
|
-
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
2010
|
-
throw new Error(`Invalid parsed result path: ${value}`);
|
|
2011
|
-
}
|
|
2012
|
-
return normalized;
|
|
2013
|
-
}
|
|
2014
|
-
function isSdkMetadataPath(path4) {
|
|
2015
|
-
return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
1650
|
// src/knowledge/local-store.ts
|
|
2019
1651
|
import { createHash } from "crypto";
|
|
2020
1652
|
import os from "os";
|
|
@@ -2354,29 +1986,32 @@ var Knowledge = class _Knowledge {
|
|
|
2354
1986
|
return this.importJobResult(params);
|
|
2355
1987
|
}
|
|
2356
1988
|
async syncParsedDocument(params) {
|
|
1989
|
+
params.signal?.throwIfAborted();
|
|
2357
1990
|
const normalized = normalizeDocumentReference(params);
|
|
2358
1991
|
const parsedStorageConfig = this.requireParsedStorageConfig();
|
|
2359
1992
|
if (normalized.documentId) {
|
|
2360
1993
|
return this.syncRemoteDocumentToParsedStorage({
|
|
2361
1994
|
documentId: normalized.documentId,
|
|
2362
1995
|
revisionKey: params.revisionKey,
|
|
2363
|
-
parsedStorageConfig
|
|
1996
|
+
parsedStorageConfig,
|
|
1997
|
+
signal: params.signal,
|
|
1998
|
+
retainResultArchive: params.retainResultArchive
|
|
2364
1999
|
});
|
|
2365
2000
|
}
|
|
2366
2001
|
if (normalized.jobId) {
|
|
2367
|
-
const
|
|
2368
|
-
|
|
2369
|
-
result
|
|
2002
|
+
const job = await this.client.jobs.get(normalized.jobId);
|
|
2003
|
+
if (!job.isDone || !job.resultUrl)
|
|
2004
|
+
throw new ValidationError("Completed job result archive is unavailable");
|
|
2005
|
+
return syncRemoteResultArchive({
|
|
2006
|
+
documentId: job.documentId ?? normalized.localDocumentId ?? normalized.jobId,
|
|
2007
|
+
revisionKey: params.revisionKey ?? normalized.jobId,
|
|
2008
|
+
jobId: normalized.jobId,
|
|
2009
|
+
resultUrl: job.resultUrl,
|
|
2370
2010
|
storage: parsedStorageConfig.storage,
|
|
2371
|
-
|
|
2372
|
-
|
|
2011
|
+
limits: parsedStorageConfig.limits,
|
|
2012
|
+
signal: params.signal,
|
|
2013
|
+
retainResultArchive: params.retainResultArchive
|
|
2373
2014
|
});
|
|
2374
|
-
return {
|
|
2375
|
-
documentId: synced2.documentId,
|
|
2376
|
-
revisionKey: synced2.revisionKey,
|
|
2377
|
-
completed: true,
|
|
2378
|
-
commit: synced2.commit
|
|
2379
|
-
};
|
|
2380
2015
|
}
|
|
2381
2016
|
if (!normalized.localDocumentId) {
|
|
2382
2017
|
throw new ValidationError("localDocumentId, documentId, or jobId is required");
|
|
@@ -3074,10 +2709,14 @@ var Knowledge = class _Knowledge {
|
|
|
3074
2709
|
parsedStorageConfig
|
|
3075
2710
|
});
|
|
3076
2711
|
} catch (error) {
|
|
2712
|
+
const previous = await parsedStorageConfig.storage.readSyncProgress({
|
|
2713
|
+
documentId,
|
|
2714
|
+
revisionKey
|
|
2715
|
+
});
|
|
3077
2716
|
await parsedStorageConfig.storage.writeSyncProgress({
|
|
2717
|
+
...previous,
|
|
3078
2718
|
documentId,
|
|
3079
2719
|
revisionKey,
|
|
3080
|
-
nextChunkPage: 1,
|
|
3081
2720
|
status: "failed",
|
|
3082
2721
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3083
2722
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -3087,6 +2726,7 @@ var Knowledge = class _Knowledge {
|
|
|
3087
2726
|
void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
|
|
3088
2727
|
}
|
|
3089
2728
|
async syncRemoteDocumentToParsedStorage(params) {
|
|
2729
|
+
params.signal?.throwIfAborted();
|
|
3090
2730
|
const firstPage = await this.client.documents.listChunks(params.documentId, {
|
|
3091
2731
|
page: 1,
|
|
3092
2732
|
pageSize: 1,
|
|
@@ -3094,55 +2734,67 @@ var Knowledge = class _Knowledge {
|
|
|
3094
2734
|
});
|
|
3095
2735
|
const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
|
|
3096
2736
|
if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
|
|
3097
|
-
|
|
3098
|
-
documentId: params.documentId,
|
|
3099
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
3100
|
-
});
|
|
2737
|
+
throw new ValidationError("Published document revision changed before parsed snapshot sync");
|
|
3101
2738
|
}
|
|
3102
2739
|
if (firstPage.jobId) {
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
});
|
|
3112
|
-
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
2740
|
+
const job = await this.client.jobs.get(firstPage.jobId).catch((error) => {
|
|
2741
|
+
if (error instanceof NotFoundError) return null;
|
|
2742
|
+
throw error;
|
|
2743
|
+
});
|
|
2744
|
+
if (job?.isDone && job.resultUrl) {
|
|
2745
|
+
if (job.documentId && job.documentId !== params.documentId)
|
|
2746
|
+
throw new ValidationError("Parsed job document identity differs from requested document");
|
|
2747
|
+
return syncRemoteResultArchive({
|
|
3113
2748
|
documentId: params.documentId,
|
|
3114
2749
|
revisionKey,
|
|
3115
|
-
|
|
3116
|
-
|
|
2750
|
+
jobId: firstPage.jobId,
|
|
2751
|
+
resultUrl: job.resultUrl,
|
|
2752
|
+
storage: params.parsedStorageConfig.storage,
|
|
2753
|
+
limits: params.parsedStorageConfig.limits,
|
|
2754
|
+
signal: params.signal,
|
|
2755
|
+
retainResultArchive: params.retainResultArchive,
|
|
2756
|
+
verifyRevision: async () => {
|
|
2757
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
2758
|
+
page: 1,
|
|
2759
|
+
pageSize: 1,
|
|
2760
|
+
includeAssetUrls: true
|
|
2761
|
+
});
|
|
2762
|
+
if (getRemoteRevisionKey(current) !== revisionKey || current.jobId !== firstPage.jobId)
|
|
2763
|
+
throw new ValidationError(
|
|
2764
|
+
"Published document revision changed during parsed snapshot sync"
|
|
2765
|
+
);
|
|
2766
|
+
}
|
|
3117
2767
|
});
|
|
3118
|
-
return {
|
|
3119
|
-
documentId: synced.documentId,
|
|
3120
|
-
revisionKey: synced.revisionKey,
|
|
3121
|
-
completed: true,
|
|
3122
|
-
commit: synced.commit
|
|
3123
|
-
};
|
|
3124
|
-
} catch {
|
|
3125
2768
|
}
|
|
3126
2769
|
}
|
|
3127
2770
|
return this.syncRemoteDocumentChunksToParsedStorage({
|
|
3128
2771
|
documentId: params.documentId,
|
|
3129
2772
|
revisionKey,
|
|
3130
2773
|
firstPage,
|
|
3131
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
2774
|
+
parsedStorageConfig: params.parsedStorageConfig,
|
|
2775
|
+
signal: params.signal
|
|
3132
2776
|
});
|
|
3133
2777
|
}
|
|
3134
2778
|
async syncRemoteDocumentChunksToParsedStorage(params) {
|
|
2779
|
+
params.signal?.throwIfAborted();
|
|
3135
2780
|
const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
|
|
3136
2781
|
const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
|
|
3137
2782
|
documentId: params.documentId,
|
|
3138
2783
|
revisionKey: params.revisionKey
|
|
3139
2784
|
});
|
|
3140
|
-
let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
2785
|
+
let nextPage = existingProgress?.status === "running" || existingProgress?.status === "failed" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
3141
2786
|
let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
|
|
3142
2787
|
page: nextPage,
|
|
3143
2788
|
pageSize: limits.remotePageSize,
|
|
3144
2789
|
includeAssetUrls: true
|
|
3145
2790
|
});
|
|
2791
|
+
validateRemoteReconstructionRevision(
|
|
2792
|
+
params.documentId,
|
|
2793
|
+
params.revisionKey,
|
|
2794
|
+
params.firstPage.jobId,
|
|
2795
|
+
response
|
|
2796
|
+
);
|
|
2797
|
+
params.signal?.throwIfAborted();
|
|
3146
2798
|
const totalPages = response.pagination.totalPages;
|
|
3147
2799
|
let syncedPages = 0;
|
|
3148
2800
|
const startedAt = Date.now();
|
|
@@ -3154,6 +2806,13 @@ var Knowledge = class _Knowledge {
|
|
|
3154
2806
|
includeAssetUrls: true
|
|
3155
2807
|
});
|
|
3156
2808
|
}
|
|
2809
|
+
params.signal?.throwIfAborted();
|
|
2810
|
+
validateRemoteReconstructionRevision(
|
|
2811
|
+
params.documentId,
|
|
2812
|
+
params.revisionKey,
|
|
2813
|
+
params.firstPage.jobId,
|
|
2814
|
+
response
|
|
2815
|
+
);
|
|
3157
2816
|
await params.parsedStorageConfig.storage.writeObject({
|
|
3158
2817
|
documentId: params.documentId,
|
|
3159
2818
|
revisionKey: params.revisionKey,
|
|
@@ -3201,7 +2860,23 @@ var Knowledge = class _Knowledge {
|
|
|
3201
2860
|
storage: params.parsedStorageConfig.storage,
|
|
3202
2861
|
documentId: params.documentId,
|
|
3203
2862
|
revisionKey: params.revisionKey,
|
|
3204
|
-
source: "remoteReconstruction"
|
|
2863
|
+
source: "remoteReconstruction",
|
|
2864
|
+
signal: params.signal,
|
|
2865
|
+
beforeCommit: async () => {
|
|
2866
|
+
params.signal?.throwIfAborted();
|
|
2867
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
2868
|
+
page: 1,
|
|
2869
|
+
pageSize: 1,
|
|
2870
|
+
includeAssetUrls: true
|
|
2871
|
+
});
|
|
2872
|
+
params.signal?.throwIfAborted();
|
|
2873
|
+
validateRemoteReconstructionRevision(
|
|
2874
|
+
params.documentId,
|
|
2875
|
+
params.revisionKey,
|
|
2876
|
+
params.firstPage.jobId,
|
|
2877
|
+
current
|
|
2878
|
+
);
|
|
2879
|
+
}
|
|
3205
2880
|
});
|
|
3206
2881
|
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3207
2882
|
documentId: params.documentId,
|
|
@@ -3401,7 +3076,7 @@ function toMinimalChunkFromRemoteChunk(chunk) {
|
|
|
3401
3076
|
filePath: chunk.filePath ?? "",
|
|
3402
3077
|
assetUrl: chunk.assetUrl ?? void 0,
|
|
3403
3078
|
data: Buffer.alloc(0),
|
|
3404
|
-
format:
|
|
3079
|
+
format: inferContentTypeFromPath(chunk.filePath ?? "").replace("image/", ""),
|
|
3405
3080
|
save: () => Promise.resolve(chunk.filePath ?? "")
|
|
3406
3081
|
};
|
|
3407
3082
|
}
|
|
@@ -3753,7 +3428,7 @@ function cloneMetadata(metadata) {
|
|
|
3753
3428
|
function toUnknownArray(value) {
|
|
3754
3429
|
return Array.isArray(value) ? value : null;
|
|
3755
3430
|
}
|
|
3756
|
-
function
|
|
3431
|
+
function inferContentTypeFromPath(filePath) {
|
|
3757
3432
|
const extension = filePath.split(".").at(-1)?.toLowerCase();
|
|
3758
3433
|
if (extension === "png") return "image/png";
|
|
3759
3434
|
if (extension === "jpg" || extension === "jpeg") return "image/jpeg";
|
|
@@ -4067,6 +3742,11 @@ function toRemoteSearchResult(result, documentByServerId) {
|
|
|
4067
3742
|
sourceFileName: result.source.sourceFileName ?? void 0
|
|
4068
3743
|
};
|
|
4069
3744
|
}
|
|
3745
|
+
function validateRemoteReconstructionRevision(documentId, revisionKey, jobId, response) {
|
|
3746
|
+
if (response.documentId !== documentId || getRemoteRevisionKey(response) !== revisionKey || (response.jobId ?? void 0) !== (jobId ?? void 0)) {
|
|
3747
|
+
throw new ValidationError("Published document revision changed during remote reconstruction");
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
4070
3750
|
|
|
4071
3751
|
// src/client.ts
|
|
4072
3752
|
function inferFileName(file, explicitFileName) {
|
|
@@ -4395,6 +4075,7 @@ export {
|
|
|
4395
4075
|
NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
|
|
4396
4076
|
NetworkError,
|
|
4397
4077
|
NotFoundError,
|
|
4078
|
+
ParsedDocumentArchiveError,
|
|
4398
4079
|
PaymentRequiredError,
|
|
4399
4080
|
PermissionDeniedError,
|
|
4400
4081
|
PollingTimeoutError,
|