@ontos-ai/knowhere-sdk 2.4.2 → 2.6.0-beta.20261026172248
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/chunk-FESTKAG6.mjs +1343 -0
- package/dist/index.d.mts +49 -3
- package/dist/index.d.ts +49 -3
- package/dist/index.js +1556 -482
- package/dist/index.mjs +145 -565
- package/dist/sequential-parsed-document-archive-YQ5U3VWM.mjs +143 -0
- package/package.json +5 -2
package/dist/index.js
CHANGED
|
@@ -5,6 +5,9 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
|
5
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
6
|
var __getProtoOf = Object.getPrototypeOf;
|
|
7
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __esm = (fn, res) => function __init() {
|
|
9
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
10
|
+
};
|
|
8
11
|
var __export = (target, all) => {
|
|
9
12
|
for (var name in all)
|
|
10
13
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
@@ -27,205 +30,92 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
27
30
|
));
|
|
28
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
32
|
|
|
30
|
-
// src/index.ts
|
|
31
|
-
var index_exports = {};
|
|
32
|
-
__export(index_exports, {
|
|
33
|
-
APIError: () => APIError,
|
|
34
|
-
AuthenticationError: () => AuthenticationError,
|
|
35
|
-
BadRequestError: () => BadRequestError,
|
|
36
|
-
ChecksumError: () => ChecksumError,
|
|
37
|
-
ConflictError: () => ConflictError,
|
|
38
|
-
DiskParsedDocumentStorage: () => DiskParsedDocumentStorage,
|
|
39
|
-
Documents: () => Documents,
|
|
40
|
-
GatewayTimeoutError: () => GatewayTimeoutError,
|
|
41
|
-
InternalServerError: () => InternalServerError,
|
|
42
|
-
InvalidStateError: () => InvalidStateError,
|
|
43
|
-
JobFailedError: () => JobFailedError,
|
|
44
|
-
Jobs: () => Jobs,
|
|
45
|
-
Knowhere: () => Knowhere,
|
|
46
|
-
KnowhereError: () => KnowhereError,
|
|
47
|
-
Knowledge: () => Knowledge,
|
|
48
|
-
LocalKnowledgeStore: () => LocalKnowledgeStore,
|
|
49
|
-
NODE_SDK_DOCUMENT_METADATA_DEFAULTS: () => NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
|
|
50
|
-
NetworkError: () => NetworkError,
|
|
51
|
-
NotFoundError: () => NotFoundError,
|
|
52
|
-
PaymentRequiredError: () => PaymentRequiredError,
|
|
53
|
-
PermissionDeniedError: () => PermissionDeniedError,
|
|
54
|
-
PollingTimeoutError: () => PollingTimeoutError,
|
|
55
|
-
RateLimitError: () => RateLimitError,
|
|
56
|
-
Retrieval: () => Retrieval,
|
|
57
|
-
ServiceUnavailableError: () => ServiceUnavailableError,
|
|
58
|
-
TimeoutError: () => TimeoutError,
|
|
59
|
-
VERSION: () => VERSION,
|
|
60
|
-
ValidationError: () => ValidationError,
|
|
61
|
-
default: () => Knowhere,
|
|
62
|
-
mergeDocumentMetadataDefaults: () => mergeDocumentMetadataDefaults,
|
|
63
|
-
storeParseResultAssets: () => storeParseResultAssets,
|
|
64
|
-
syncParseResultToParsedDocumentStorage: () => syncParseResultToParsedDocumentStorage
|
|
65
|
-
});
|
|
66
|
-
module.exports = __toCommonJS(index_exports);
|
|
67
|
-
|
|
68
|
-
// src/client.ts
|
|
69
|
-
var import_path3 = __toESM(require("path"));
|
|
70
|
-
|
|
71
|
-
// src/lib/http-client.ts
|
|
72
|
-
var import_axios = __toESM(require("axios"));
|
|
73
|
-
|
|
74
|
-
// src/version.ts
|
|
75
|
-
var VERSION = "2.1.3";
|
|
76
|
-
|
|
77
33
|
// src/constants.ts
|
|
78
|
-
var DEFAULT_BASE_URL
|
|
79
|
-
var
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
34
|
+
var DEFAULT_BASE_URL, DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES, DEFAULT_POLL_INTERVAL, MAX_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT, POLL_INTERVAL_INCREASE_THRESHOLD, POLL_INTERVAL_MULTIPLIER, INITIAL_RETRY_DELAY, MAX_RETRY_DELAY, RETRY_DELAY_BASE, RETRYABLE_STATUS_CODES, TERMINAL_JOB_STATUSES, ENV;
|
|
35
|
+
var init_constants = __esm({
|
|
36
|
+
"src/constants.ts"() {
|
|
37
|
+
"use strict";
|
|
38
|
+
DEFAULT_BASE_URL = "https://api.knowhereto.ai";
|
|
39
|
+
DEFAULT_TIMEOUT = 6e4;
|
|
40
|
+
DEFAULT_MAX_RETRIES = 5;
|
|
41
|
+
DEFAULT_POLL_INTERVAL = 1e4;
|
|
42
|
+
MAX_POLL_INTERVAL = 3e4;
|
|
43
|
+
DEFAULT_POLL_TIMEOUT = 18e5;
|
|
44
|
+
POLL_INTERVAL_INCREASE_THRESHOLD = 6e4;
|
|
45
|
+
POLL_INTERVAL_MULTIPLIER = 1.2;
|
|
46
|
+
INITIAL_RETRY_DELAY = 500;
|
|
47
|
+
MAX_RETRY_DELAY = 3e4;
|
|
48
|
+
RETRY_DELAY_BASE = 2;
|
|
49
|
+
RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([409, 429, 502, 503, 504]);
|
|
50
|
+
TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["done", "failed"]);
|
|
51
|
+
ENV = {
|
|
52
|
+
API_KEY: "KNOWHERE_API_KEY",
|
|
53
|
+
BASE_URL: "KNOWHERE_BASE_URL",
|
|
54
|
+
LOG_LEVEL: "KNOWHERE_LOG_LEVEL"
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
});
|
|
96
58
|
|
|
97
59
|
// src/errors/base.ts
|
|
98
|
-
var KnowhereError
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
60
|
+
var KnowhereError, NetworkError, TimeoutError, PollingTimeoutError, ChecksumError, ValidationError, InvalidStateError;
|
|
61
|
+
var init_base = __esm({
|
|
62
|
+
"src/errors/base.ts"() {
|
|
63
|
+
"use strict";
|
|
64
|
+
KnowhereError = class extends Error {
|
|
65
|
+
constructor(message) {
|
|
66
|
+
super(message);
|
|
67
|
+
this.name = "KnowhereError";
|
|
68
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
NetworkError = class extends KnowhereError {
|
|
72
|
+
constructor(message, cause) {
|
|
73
|
+
super(message);
|
|
74
|
+
this.cause = cause;
|
|
75
|
+
this.name = "NetworkError";
|
|
76
|
+
}
|
|
77
|
+
cause;
|
|
78
|
+
};
|
|
79
|
+
TimeoutError = class extends NetworkError {
|
|
80
|
+
constructor(message = "Request timed out") {
|
|
81
|
+
super(message);
|
|
82
|
+
this.name = "TimeoutError";
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
PollingTimeoutError = class extends KnowhereError {
|
|
86
|
+
constructor(message = "Polling timed out", elapsedMs) {
|
|
87
|
+
super(message);
|
|
88
|
+
this.elapsedMs = elapsedMs;
|
|
89
|
+
this.name = "PollingTimeoutError";
|
|
90
|
+
}
|
|
91
|
+
elapsedMs;
|
|
92
|
+
};
|
|
93
|
+
ChecksumError = class extends KnowhereError {
|
|
94
|
+
constructor(message = "Checksum verification failed", expected, actual) {
|
|
95
|
+
super(message);
|
|
96
|
+
this.expected = expected;
|
|
97
|
+
this.actual = actual;
|
|
98
|
+
this.name = "ChecksumError";
|
|
99
|
+
}
|
|
100
|
+
expected;
|
|
101
|
+
actual;
|
|
102
|
+
};
|
|
103
|
+
ValidationError = class extends KnowhereError {
|
|
104
|
+
constructor(message) {
|
|
105
|
+
super(message);
|
|
106
|
+
this.name = "ValidationError";
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
InvalidStateError = class extends KnowhereError {
|
|
110
|
+
constructor(message) {
|
|
111
|
+
super(message);
|
|
112
|
+
this.name = "InvalidStateError";
|
|
113
|
+
}
|
|
114
|
+
};
|
|
147
115
|
}
|
|
148
|
-
};
|
|
116
|
+
});
|
|
149
117
|
|
|
150
118
|
// src/errors/api-errors.ts
|
|
151
|
-
var APIError = class extends KnowhereError {
|
|
152
|
-
constructor(message, statusCode, code, requestId, details, body) {
|
|
153
|
-
super(message);
|
|
154
|
-
this.statusCode = statusCode;
|
|
155
|
-
this.code = code;
|
|
156
|
-
this.requestId = requestId;
|
|
157
|
-
this.details = details;
|
|
158
|
-
this.body = body;
|
|
159
|
-
this.name = "APIError";
|
|
160
|
-
}
|
|
161
|
-
statusCode;
|
|
162
|
-
code;
|
|
163
|
-
requestId;
|
|
164
|
-
details;
|
|
165
|
-
body;
|
|
166
|
-
};
|
|
167
|
-
var BadRequestError = class extends APIError {
|
|
168
|
-
constructor(message, code, requestId, details, body) {
|
|
169
|
-
super(message, 400, code, requestId, details, body);
|
|
170
|
-
this.name = "BadRequestError";
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
var AuthenticationError = class extends APIError {
|
|
174
|
-
constructor(message = "Authentication failed", code, requestId, details, body) {
|
|
175
|
-
super(message, 401, code, requestId, details, body);
|
|
176
|
-
this.name = "AuthenticationError";
|
|
177
|
-
}
|
|
178
|
-
};
|
|
179
|
-
var PaymentRequiredError = class extends APIError {
|
|
180
|
-
constructor(message = "Payment required", code, requestId, details, body) {
|
|
181
|
-
super(message, 402, code, requestId, details, body);
|
|
182
|
-
this.name = "PaymentRequiredError";
|
|
183
|
-
}
|
|
184
|
-
};
|
|
185
|
-
var PermissionDeniedError = class extends APIError {
|
|
186
|
-
constructor(message = "Permission denied", code, requestId, details, body) {
|
|
187
|
-
super(message, 403, code, requestId, details, body);
|
|
188
|
-
this.name = "PermissionDeniedError";
|
|
189
|
-
}
|
|
190
|
-
};
|
|
191
|
-
var NotFoundError = class extends APIError {
|
|
192
|
-
constructor(message = "Resource not found", code, requestId, details, body) {
|
|
193
|
-
super(message, 404, code, requestId, details, body);
|
|
194
|
-
this.name = "NotFoundError";
|
|
195
|
-
}
|
|
196
|
-
};
|
|
197
|
-
var ConflictError = class extends APIError {
|
|
198
|
-
constructor(message = "Conflict", code, requestId, details, body) {
|
|
199
|
-
super(message, 409, code, requestId, details, body);
|
|
200
|
-
this.name = "ConflictError";
|
|
201
|
-
}
|
|
202
|
-
};
|
|
203
|
-
var RateLimitError = class extends APIError {
|
|
204
|
-
constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
|
|
205
|
-
super(message, 429, code, requestId, details, body);
|
|
206
|
-
this.retryAfter = retryAfter;
|
|
207
|
-
this.name = "RateLimitError";
|
|
208
|
-
}
|
|
209
|
-
retryAfter;
|
|
210
|
-
};
|
|
211
|
-
var InternalServerError = class extends APIError {
|
|
212
|
-
constructor(message = "Internal server error", code, requestId, details, body) {
|
|
213
|
-
super(message, 500, code, requestId, details, body);
|
|
214
|
-
this.name = "InternalServerError";
|
|
215
|
-
}
|
|
216
|
-
};
|
|
217
|
-
var ServiceUnavailableError = class extends APIError {
|
|
218
|
-
constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
|
|
219
|
-
super(message, statusCode, code, requestId, details, body);
|
|
220
|
-
this.name = "ServiceUnavailableError";
|
|
221
|
-
}
|
|
222
|
-
};
|
|
223
|
-
var GatewayTimeoutError = class extends APIError {
|
|
224
|
-
constructor(message = "Gateway timeout", code, requestId, details, body) {
|
|
225
|
-
super(message, 504, code, requestId, details, body);
|
|
226
|
-
this.name = "GatewayTimeoutError";
|
|
227
|
-
}
|
|
228
|
-
};
|
|
229
119
|
function createAPIError(statusCode, message, code, requestId, details, body, retryAfter) {
|
|
230
120
|
switch (statusCode) {
|
|
231
121
|
case 400:
|
|
@@ -253,18 +143,141 @@ function createAPIError(statusCode, message, code, requestId, details, body, ret
|
|
|
253
143
|
return new APIError(message, statusCode, code, requestId, details, body);
|
|
254
144
|
}
|
|
255
145
|
}
|
|
146
|
+
var APIError, BadRequestError, AuthenticationError, PaymentRequiredError, PermissionDeniedError, NotFoundError, ConflictError, RateLimitError, InternalServerError, ServiceUnavailableError, GatewayTimeoutError;
|
|
147
|
+
var init_api_errors = __esm({
|
|
148
|
+
"src/errors/api-errors.ts"() {
|
|
149
|
+
"use strict";
|
|
150
|
+
init_base();
|
|
151
|
+
APIError = class extends KnowhereError {
|
|
152
|
+
constructor(message, statusCode, code, requestId, details, body) {
|
|
153
|
+
super(message);
|
|
154
|
+
this.statusCode = statusCode;
|
|
155
|
+
this.code = code;
|
|
156
|
+
this.requestId = requestId;
|
|
157
|
+
this.details = details;
|
|
158
|
+
this.body = body;
|
|
159
|
+
this.name = "APIError";
|
|
160
|
+
}
|
|
161
|
+
statusCode;
|
|
162
|
+
code;
|
|
163
|
+
requestId;
|
|
164
|
+
details;
|
|
165
|
+
body;
|
|
166
|
+
};
|
|
167
|
+
BadRequestError = class extends APIError {
|
|
168
|
+
constructor(message, code, requestId, details, body) {
|
|
169
|
+
super(message, 400, code, requestId, details, body);
|
|
170
|
+
this.name = "BadRequestError";
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
AuthenticationError = class extends APIError {
|
|
174
|
+
constructor(message = "Authentication failed", code, requestId, details, body) {
|
|
175
|
+
super(message, 401, code, requestId, details, body);
|
|
176
|
+
this.name = "AuthenticationError";
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
PaymentRequiredError = class extends APIError {
|
|
180
|
+
constructor(message = "Payment required", code, requestId, details, body) {
|
|
181
|
+
super(message, 402, code, requestId, details, body);
|
|
182
|
+
this.name = "PaymentRequiredError";
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
PermissionDeniedError = class extends APIError {
|
|
186
|
+
constructor(message = "Permission denied", code, requestId, details, body) {
|
|
187
|
+
super(message, 403, code, requestId, details, body);
|
|
188
|
+
this.name = "PermissionDeniedError";
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
NotFoundError = class extends APIError {
|
|
192
|
+
constructor(message = "Resource not found", code, requestId, details, body) {
|
|
193
|
+
super(message, 404, code, requestId, details, body);
|
|
194
|
+
this.name = "NotFoundError";
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
ConflictError = class extends APIError {
|
|
198
|
+
constructor(message = "Conflict", code, requestId, details, body) {
|
|
199
|
+
super(message, 409, code, requestId, details, body);
|
|
200
|
+
this.name = "ConflictError";
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
RateLimitError = class extends APIError {
|
|
204
|
+
constructor(message = "Rate limit exceeded", code, requestId, details, body, retryAfter) {
|
|
205
|
+
super(message, 429, code, requestId, details, body);
|
|
206
|
+
this.retryAfter = retryAfter;
|
|
207
|
+
this.name = "RateLimitError";
|
|
208
|
+
}
|
|
209
|
+
retryAfter;
|
|
210
|
+
};
|
|
211
|
+
InternalServerError = class extends APIError {
|
|
212
|
+
constructor(message = "Internal server error", code, requestId, details, body) {
|
|
213
|
+
super(message, 500, code, requestId, details, body);
|
|
214
|
+
this.name = "InternalServerError";
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
ServiceUnavailableError = class extends APIError {
|
|
218
|
+
constructor(message = "Service unavailable", statusCode = 503, code, requestId, details, body) {
|
|
219
|
+
super(message, statusCode, code, requestId, details, body);
|
|
220
|
+
this.name = "ServiceUnavailableError";
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
GatewayTimeoutError = class extends APIError {
|
|
224
|
+
constructor(message = "Gateway timeout", code, requestId, details, body) {
|
|
225
|
+
super(message, 504, code, requestId, details, body);
|
|
226
|
+
this.name = "GatewayTimeoutError";
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
});
|
|
256
231
|
|
|
257
232
|
// src/errors/job-errors.ts
|
|
258
|
-
var JobFailedError
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
233
|
+
var JobFailedError;
|
|
234
|
+
var init_job_errors = __esm({
|
|
235
|
+
"src/errors/job-errors.ts"() {
|
|
236
|
+
"use strict";
|
|
237
|
+
init_base();
|
|
238
|
+
JobFailedError = class extends KnowhereError {
|
|
239
|
+
constructor(message, code, jobResult) {
|
|
240
|
+
super(message);
|
|
241
|
+
this.code = code;
|
|
242
|
+
this.jobResult = jobResult;
|
|
243
|
+
this.name = "JobFailedError";
|
|
244
|
+
}
|
|
245
|
+
code;
|
|
246
|
+
jobResult;
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
// src/errors/parsed-document-archive-error.ts
|
|
252
|
+
var ParsedDocumentArchiveError;
|
|
253
|
+
var init_parsed_document_archive_error = __esm({
|
|
254
|
+
"src/errors/parsed-document-archive-error.ts"() {
|
|
255
|
+
"use strict";
|
|
256
|
+
init_base();
|
|
257
|
+
ParsedDocumentArchiveError = class extends KnowhereError {
|
|
258
|
+
code = "ARCHIVE_SPOOL_FAILED";
|
|
259
|
+
cause;
|
|
260
|
+
constructor(cause) {
|
|
261
|
+
super(
|
|
262
|
+
"Parsed result ZIP temporary storage is unavailable. Range-enabled object storage avoids temporary disk usage."
|
|
263
|
+
);
|
|
264
|
+
this.name = "ParsedDocumentArchiveError";
|
|
265
|
+
this.cause = cause;
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
// src/errors/index.ts
|
|
272
|
+
var init_errors = __esm({
|
|
273
|
+
"src/errors/index.ts"() {
|
|
274
|
+
"use strict";
|
|
275
|
+
init_base();
|
|
276
|
+
init_api_errors();
|
|
277
|
+
init_job_errors();
|
|
278
|
+
init_parsed_document_archive_error();
|
|
279
|
+
}
|
|
280
|
+
});
|
|
268
281
|
|
|
269
282
|
// src/lib/utils.ts
|
|
270
283
|
function sleep(ms) {
|
|
@@ -388,8 +401,1204 @@ function jitter(value, percent = 0.2) {
|
|
|
388
401
|
const randomFactor = 1 + (Math.random() * 2 - 1) * percent;
|
|
389
402
|
return Math.round(value * randomFactor);
|
|
390
403
|
}
|
|
404
|
+
var init_utils = __esm({
|
|
405
|
+
"src/lib/utils.ts"() {
|
|
406
|
+
"use strict";
|
|
407
|
+
init_constants();
|
|
408
|
+
}
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
// src/storage/parsed-document-storage.ts
|
|
412
|
+
async function syncParseResultToParsedDocumentStorage(params) {
|
|
413
|
+
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
414
|
+
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
415
|
+
for await (const object of collectResultLayoutObjects(params.result)) {
|
|
416
|
+
params.signal?.throwIfAborted();
|
|
417
|
+
await params.storage.writeObject({
|
|
418
|
+
documentId,
|
|
419
|
+
revisionKey,
|
|
420
|
+
...object
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
await params.beforeCommit?.();
|
|
424
|
+
params.signal?.throwIfAborted();
|
|
425
|
+
const commit = {
|
|
426
|
+
version: 1,
|
|
427
|
+
documentId,
|
|
428
|
+
revisionKey,
|
|
429
|
+
source: params.source ?? "resultZip",
|
|
430
|
+
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
431
|
+
};
|
|
432
|
+
await writeParsedDocumentJsonObject(params.storage, {
|
|
433
|
+
documentId,
|
|
434
|
+
revisionKey,
|
|
435
|
+
path: parsedDocumentCommitPath,
|
|
436
|
+
value: commit
|
|
437
|
+
});
|
|
438
|
+
return {
|
|
439
|
+
result: params.result,
|
|
440
|
+
documentId,
|
|
441
|
+
revisionKey,
|
|
442
|
+
commit
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
async function readParsedDocumentCommit(params) {
|
|
446
|
+
const object = await params.storage.readObject({
|
|
447
|
+
documentId: params.documentId,
|
|
448
|
+
revisionKey: params.revisionKey,
|
|
449
|
+
path: parsedDocumentCommitPath
|
|
450
|
+
});
|
|
451
|
+
if (!object) {
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
const commit = readJsonObject(object);
|
|
455
|
+
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
456
|
+
}
|
|
457
|
+
function readJsonObject(object) {
|
|
458
|
+
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
459
|
+
}
|
|
460
|
+
async function* collectResultLayoutObjects(result) {
|
|
461
|
+
const storedPaths = /* @__PURE__ */ new Set();
|
|
462
|
+
let zip;
|
|
463
|
+
try {
|
|
464
|
+
zip = await import_jszip3.default.loadAsync(result.rawZip);
|
|
465
|
+
} catch {
|
|
466
|
+
}
|
|
467
|
+
if (zip) {
|
|
468
|
+
for (const entry of Object.values(zip.files)) {
|
|
469
|
+
if (entry.dir || entry.name === "result.zip") continue;
|
|
470
|
+
const path4 = normalizeResultPath(entry.name);
|
|
471
|
+
if (isSdkMetadataPath(path4)) continue;
|
|
472
|
+
storedPaths.add(path4);
|
|
473
|
+
yield {
|
|
474
|
+
path: path4,
|
|
475
|
+
body: await entry.async("nodebuffer"),
|
|
476
|
+
contentType: inferContentTypeFromPath(path4)
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
for (const object of generateMissingResultObjects(result, storedPaths)) yield object;
|
|
481
|
+
}
|
|
482
|
+
function* generateMissingResultObjects(result, storedPaths) {
|
|
483
|
+
if (!storedPaths.has("manifest.json")) yield createJsonObject("manifest.json", result.manifest);
|
|
484
|
+
if (!storedPaths.has("chunks.json"))
|
|
485
|
+
yield createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)));
|
|
486
|
+
if (!storedPaths.has("doc_nav.json") && result.docNav)
|
|
487
|
+
yield createJsonObject("doc_nav.json", result.docNav);
|
|
488
|
+
if (!storedPaths.has("full.md") && result.fullMarkdown)
|
|
489
|
+
yield createTextObject("full.md", result.fullMarkdown, markdownContentType);
|
|
490
|
+
if (!storedPaths.has("chunks_slim.json") && result.chunksSlim)
|
|
491
|
+
yield createJsonObject("chunks_slim.json", { chunks: result.chunksSlim });
|
|
492
|
+
if (!storedPaths.has("hierarchy.json") && result.hierarchy)
|
|
493
|
+
yield createJsonObject("hierarchy.json", result.hierarchy);
|
|
494
|
+
if (!storedPaths.has("toc_hierarchies.json") && result.tocHierarchies)
|
|
495
|
+
yield createJsonObject("toc_hierarchies.json", result.tocHierarchies);
|
|
496
|
+
if (!storedPaths.has("kb.csv") && result.kbCsv)
|
|
497
|
+
yield createTextObject("kb.csv", result.kbCsv, csvContentType);
|
|
498
|
+
if (!storedPaths.has("hierarchy_view.html") && result.hierarchyViewHtml)
|
|
499
|
+
yield createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType);
|
|
500
|
+
for (const chunk of result.imageChunks) {
|
|
501
|
+
const path4 = normalizeResultPath(chunk.filePath);
|
|
502
|
+
if (storedPaths.has(path4)) continue;
|
|
503
|
+
yield {
|
|
504
|
+
path: path4,
|
|
505
|
+
body: chunk.data,
|
|
506
|
+
contentType: inferContentTypeFromPath(path4),
|
|
507
|
+
metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
for (const chunk of result.tableChunks) {
|
|
511
|
+
const path4 = normalizeResultPath(chunk.filePath);
|
|
512
|
+
if (storedPaths.has(path4)) continue;
|
|
513
|
+
yield {
|
|
514
|
+
path: path4,
|
|
515
|
+
body: Buffer.from(chunk.html, "utf8"),
|
|
516
|
+
contentType: htmlContentType,
|
|
517
|
+
metadata: { chunkId: chunk.chunkId, chunkType: chunk.type, sourcePath: chunk.filePath }
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
function serializeChunks2(chunks) {
|
|
522
|
+
return {
|
|
523
|
+
chunks: chunks.map((chunk) => {
|
|
524
|
+
const rawChunk = {
|
|
525
|
+
chunkId: chunk.chunkId,
|
|
526
|
+
type: chunk.type,
|
|
527
|
+
contentSource: chunk.contentSource,
|
|
528
|
+
content: chunk.content,
|
|
529
|
+
path: chunk.path,
|
|
530
|
+
metadata: chunk.metadata
|
|
531
|
+
};
|
|
532
|
+
if (chunk.type === "image" || chunk.type === "table") {
|
|
533
|
+
return {
|
|
534
|
+
...rawChunk,
|
|
535
|
+
filePath: chunk.filePath,
|
|
536
|
+
assetUrl: chunk.assetUrl
|
|
537
|
+
};
|
|
538
|
+
}
|
|
539
|
+
return rawChunk;
|
|
540
|
+
})
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
async function writeParsedDocumentJsonObject(storage, params) {
|
|
544
|
+
await storage.writeObject({
|
|
545
|
+
documentId: params.documentId,
|
|
546
|
+
revisionKey: params.revisionKey,
|
|
547
|
+
path: params.path,
|
|
548
|
+
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
549
|
+
contentType: jsonContentType
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
function createJsonObject(path4, value) {
|
|
553
|
+
return {
|
|
554
|
+
path: path4,
|
|
555
|
+
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
556
|
+
contentType: jsonContentType
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
function createTextObject(path4, value, contentType) {
|
|
560
|
+
return {
|
|
561
|
+
path: path4,
|
|
562
|
+
body: Buffer.from(value, "utf8"),
|
|
563
|
+
contentType
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
function inferContentTypeFromPath(filePath) {
|
|
567
|
+
const lower = filePath.toLowerCase();
|
|
568
|
+
if (lower.endsWith(".json")) return jsonContentType;
|
|
569
|
+
if (lower.endsWith(".md")) return markdownContentType;
|
|
570
|
+
if (lower.endsWith(".csv")) return csvContentType;
|
|
571
|
+
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
572
|
+
if (lower.endsWith(".png")) return "image/png";
|
|
573
|
+
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
574
|
+
if (lower.endsWith(".gif")) return "image/gif";
|
|
575
|
+
if (lower.endsWith(".webp")) return "image/webp";
|
|
576
|
+
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
577
|
+
return binaryContentType;
|
|
578
|
+
}
|
|
579
|
+
function normalizeResultPath(value) {
|
|
580
|
+
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
581
|
+
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
582
|
+
throw new Error(`Invalid parsed result path: ${value}`);
|
|
583
|
+
}
|
|
584
|
+
return normalized;
|
|
585
|
+
}
|
|
586
|
+
function isSdkMetadataPath(path4) {
|
|
587
|
+
return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
|
|
588
|
+
}
|
|
589
|
+
var import_jszip3, parsedDocumentCommitPath, jsonContentType, markdownContentType, csvContentType, htmlContentType, binaryContentType;
|
|
590
|
+
var init_parsed_document_storage = __esm({
|
|
591
|
+
"src/storage/parsed-document-storage.ts"() {
|
|
592
|
+
"use strict";
|
|
593
|
+
import_jszip3 = __toESM(require("jszip"));
|
|
594
|
+
init_utils();
|
|
595
|
+
parsedDocumentCommitPath = ".knowhere-sdk/commit.json";
|
|
596
|
+
jsonContentType = "application/json; charset=utf-8";
|
|
597
|
+
markdownContentType = "text/markdown; charset=utf-8";
|
|
598
|
+
csvContentType = "text/csv; charset=utf-8";
|
|
599
|
+
htmlContentType = "text/html; charset=utf-8";
|
|
600
|
+
binaryContentType = "application/octet-stream";
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
// src/storage/archive-cache-transport-error.ts
|
|
605
|
+
var ArchiveCacheTransportError;
|
|
606
|
+
var init_archive_cache_transport_error = __esm({
|
|
607
|
+
"src/storage/archive-cache-transport-error.ts"() {
|
|
608
|
+
"use strict";
|
|
609
|
+
ArchiveCacheTransportError = class extends Error {
|
|
610
|
+
constructor(message) {
|
|
611
|
+
super(message);
|
|
612
|
+
this.name = "ArchiveCacheTransportError";
|
|
613
|
+
}
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
// src/storage/parsed-document-archive-cache.ts
|
|
619
|
+
async function prepareCachedArchive(params) {
|
|
620
|
+
params.signal?.throwIfAborted();
|
|
621
|
+
if (params.archiveCacheBypassed) return createFallback(params);
|
|
622
|
+
if (!params.storage.writeObjectStream || !params.storage.headObject || !params.storage.getObjectUrl) {
|
|
623
|
+
await params.onBypass?.();
|
|
624
|
+
return createFallback(params);
|
|
625
|
+
}
|
|
626
|
+
const controller = new AbortController();
|
|
627
|
+
const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
|
|
628
|
+
const mutations = /* @__PURE__ */ new Set();
|
|
629
|
+
let rejectCancellation;
|
|
630
|
+
const cancellation = new Promise((_resolve, reject) => {
|
|
631
|
+
rejectCancellation = () => {
|
|
632
|
+
const reason = signal.reason;
|
|
633
|
+
reject(
|
|
634
|
+
reason instanceof Error ? reason : new Error("Parsed result archive cache was cancelled.", { cause: reason })
|
|
635
|
+
);
|
|
636
|
+
};
|
|
637
|
+
signal.addEventListener("abort", rejectCancellation, { once: true });
|
|
638
|
+
});
|
|
639
|
+
const timer = setTimeout(() => {
|
|
640
|
+
controller.abort(new Error("Parsed result archive cache exceeded its thirty-second budget."));
|
|
641
|
+
}, ARCHIVE_CACHE_TIMEOUT_MS);
|
|
642
|
+
let origin;
|
|
643
|
+
const preparation = (async () => {
|
|
644
|
+
origin = await probeArchiveOrigin({ ...params, signal });
|
|
645
|
+
signal.throwIfAborted();
|
|
646
|
+
if (!hasStrongEntityTag(origin.entityTag) || origin.contentLength === void 0)
|
|
647
|
+
throw new Error("Parsed result archive cache requires a strong origin version and length.");
|
|
648
|
+
return prepareArchiveCache(params, origin, signal, mutations);
|
|
649
|
+
})();
|
|
650
|
+
try {
|
|
651
|
+
return await Promise.race([preparation, cancellation]);
|
|
652
|
+
} catch (error) {
|
|
653
|
+
controller.abort(error);
|
|
654
|
+
await Promise.allSettled([...mutations]);
|
|
655
|
+
params.signal?.throwIfAborted();
|
|
656
|
+
if (error instanceof ArchiveOriginIntegrityError) throw error;
|
|
657
|
+
await params.onBypass?.();
|
|
658
|
+
return createFallback(params, origin);
|
|
659
|
+
} finally {
|
|
660
|
+
clearTimeout(timer);
|
|
661
|
+
if (rejectCancellation) signal.removeEventListener("abort", rejectCancellation);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
function createFallback(params, origin) {
|
|
665
|
+
return {
|
|
666
|
+
resultUrl: params.resultUrl,
|
|
667
|
+
downloadIdentity: origin?.identity,
|
|
668
|
+
archiveCacheBypassed: true,
|
|
669
|
+
isCached: false,
|
|
670
|
+
validateResponse(response) {
|
|
671
|
+
if (origin && createOriginIdentity(params.jobId, response) !== origin.identity)
|
|
672
|
+
rejectResponse(
|
|
673
|
+
response,
|
|
674
|
+
new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.")
|
|
675
|
+
);
|
|
676
|
+
return response;
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
async function probeArchiveOrigin(params) {
|
|
681
|
+
const response = await fetch(params.resultUrl, {
|
|
682
|
+
headers: { Range: "bytes=0-0" },
|
|
683
|
+
signal: params.signal
|
|
684
|
+
});
|
|
685
|
+
try {
|
|
686
|
+
if (!response.ok) throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
|
|
687
|
+
return {
|
|
688
|
+
identity: createOriginIdentity(params.jobId, response),
|
|
689
|
+
contentLength: readOriginLength(response),
|
|
690
|
+
entityTag: response.headers.get("etag")
|
|
691
|
+
};
|
|
692
|
+
} finally {
|
|
693
|
+
await response.body?.cancel();
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
async function prepareArchiveCache(params, origin, signal, mutations) {
|
|
697
|
+
const object = { ...params, path: "result.zip" };
|
|
698
|
+
const initialHead = await params.storage.headObject(object);
|
|
699
|
+
signal.throwIfAborted();
|
|
700
|
+
let marker = null;
|
|
701
|
+
if (initialHead) {
|
|
702
|
+
const version = readObjectVersion(initialHead);
|
|
703
|
+
const stored = await params.storage.readObject({
|
|
704
|
+
...object,
|
|
705
|
+
path: createMarkerPath(origin.identity, version.entityTag)
|
|
706
|
+
});
|
|
707
|
+
signal.throwIfAborted();
|
|
708
|
+
if (stored) marker = readMarker(stored, params, origin, version);
|
|
709
|
+
}
|
|
710
|
+
if (!marker) marker = await storeArchive(params, origin, signal, mutations);
|
|
711
|
+
signal.throwIfAborted();
|
|
712
|
+
const resultUrl = await params.storage.getObjectUrl(object);
|
|
713
|
+
signal.throwIfAborted();
|
|
714
|
+
if (!resultUrl) throw new Error("Parsed result archive cache has no readable object URL.");
|
|
715
|
+
const validateResponse = (response2) => validateCachedResponse(response2, marker, params.signal);
|
|
716
|
+
const response = await fetch(resultUrl, {
|
|
717
|
+
headers: { Range: "bytes=0-0", "If-Match": marker.storageEntityTag },
|
|
718
|
+
signal
|
|
719
|
+
});
|
|
720
|
+
try {
|
|
721
|
+
if (response.headers.get("content-range") !== `bytes 0-0/${marker.contentLength}` || response.headers.get("content-length") !== "1")
|
|
722
|
+
rejectResponse(
|
|
723
|
+
response,
|
|
724
|
+
new ArchiveCacheTransportError("Parsed result archive cache probe exceeded one byte.")
|
|
725
|
+
);
|
|
726
|
+
const validated = validateResponse(response);
|
|
727
|
+
await validated.arrayBuffer();
|
|
728
|
+
} catch (error) {
|
|
729
|
+
await response.body?.cancel().catch(() => void 0);
|
|
730
|
+
throw error;
|
|
731
|
+
}
|
|
732
|
+
signal.throwIfAborted();
|
|
733
|
+
return {
|
|
734
|
+
resultUrl,
|
|
735
|
+
downloadIdentity: origin.identity,
|
|
736
|
+
archiveCacheBypassed: false,
|
|
737
|
+
isCached: true,
|
|
738
|
+
validateResponse
|
|
739
|
+
};
|
|
740
|
+
}
|
|
741
|
+
async function storeArchive(params, origin, signal, mutations) {
|
|
742
|
+
const response = await fetch(params.resultUrl, {
|
|
743
|
+
headers: { "If-Match": origin.entityTag },
|
|
744
|
+
signal
|
|
745
|
+
});
|
|
746
|
+
if (signal.aborted) {
|
|
747
|
+
await response.body?.cancel();
|
|
748
|
+
signal.throwIfAborted();
|
|
749
|
+
}
|
|
750
|
+
if (!response.ok || !response.body) {
|
|
751
|
+
await response.body?.cancel();
|
|
752
|
+
if (response.status === 412)
|
|
753
|
+
throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
|
|
754
|
+
throw new Error(`Parsed result archive cache download failed (HTTP ${response.status}).`);
|
|
755
|
+
}
|
|
756
|
+
if (createOriginIdentity(params.jobId, response) !== origin.identity) {
|
|
757
|
+
await response.body.cancel();
|
|
758
|
+
throw new ArchiveOriginIntegrityError("Parsed result origin archive identity changed.");
|
|
759
|
+
}
|
|
760
|
+
const hash = (0, import_crypto.createHash)("sha256");
|
|
761
|
+
let contentLength = 0;
|
|
762
|
+
let hasCompleted = false;
|
|
763
|
+
const body = response.body.pipeThrough(
|
|
764
|
+
new TransformStream({
|
|
765
|
+
transform(chunk, controller) {
|
|
766
|
+
signal.throwIfAborted();
|
|
767
|
+
contentLength += chunk.length;
|
|
768
|
+
if (contentLength > origin.contentLength)
|
|
769
|
+
throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
|
|
770
|
+
hash.update(chunk);
|
|
771
|
+
controller.enqueue(chunk);
|
|
772
|
+
},
|
|
773
|
+
flush() {
|
|
774
|
+
if (contentLength !== origin.contentLength)
|
|
775
|
+
throw new ArchiveOriginIntegrityError("Parsed result origin archive length changed.");
|
|
776
|
+
hasCompleted = true;
|
|
777
|
+
}
|
|
778
|
+
})
|
|
779
|
+
);
|
|
780
|
+
const object = { ...params, path: "result.zip" };
|
|
781
|
+
let write;
|
|
782
|
+
try {
|
|
783
|
+
signal.throwIfAborted();
|
|
784
|
+
write = await settleMutation(
|
|
785
|
+
params.storage.writeObjectStream({
|
|
786
|
+
...object,
|
|
787
|
+
body,
|
|
788
|
+
contentType: "application/zip",
|
|
789
|
+
contentLength: origin.contentLength,
|
|
790
|
+
signal
|
|
791
|
+
}),
|
|
792
|
+
mutations
|
|
793
|
+
);
|
|
794
|
+
} catch (error) {
|
|
795
|
+
await body.cancel(error).catch(() => void 0);
|
|
796
|
+
throw error;
|
|
797
|
+
}
|
|
798
|
+
signal.throwIfAborted();
|
|
799
|
+
if (!hasCompleted)
|
|
800
|
+
throw new Error("Parsed result archive cache sink did not consume the stream.");
|
|
801
|
+
if (!hasStrongEntityTag(write.entityTag))
|
|
802
|
+
throw new Error("Parsed result archive cache requires a strong object version.");
|
|
803
|
+
const head = await params.storage.headObject(object);
|
|
804
|
+
signal.throwIfAborted();
|
|
805
|
+
if (!head) throw new Error("Parsed result archive cache was not persisted.");
|
|
806
|
+
const actualVersion = readObjectVersion(head);
|
|
807
|
+
if (write.entityTag !== actualVersion.entityTag || actualVersion.contentLength !== contentLength)
|
|
808
|
+
throw new Error("Parsed result archive cache object version or length changed during upload.");
|
|
809
|
+
const marker = {
|
|
810
|
+
version: 1,
|
|
811
|
+
jobId: params.jobId,
|
|
812
|
+
revisionKey: params.revisionKey,
|
|
813
|
+
originIdentity: origin.identity,
|
|
814
|
+
contentLength,
|
|
815
|
+
sha256: hash.digest("hex"),
|
|
816
|
+
storageEntityTag: actualVersion.entityTag
|
|
817
|
+
};
|
|
818
|
+
signal.throwIfAborted();
|
|
819
|
+
await settleMutation(
|
|
820
|
+
params.storage.writeObject({
|
|
821
|
+
...object,
|
|
822
|
+
path: createMarkerPath(origin.identity, actualVersion.entityTag),
|
|
823
|
+
body: Buffer.from(JSON.stringify(marker)),
|
|
824
|
+
contentType: "application/json; charset=utf-8"
|
|
825
|
+
}),
|
|
826
|
+
mutations
|
|
827
|
+
);
|
|
828
|
+
signal.throwIfAborted();
|
|
829
|
+
return marker;
|
|
830
|
+
}
|
|
831
|
+
async function settleMutation(mutation, mutations) {
|
|
832
|
+
mutations.add(mutation);
|
|
833
|
+
try {
|
|
834
|
+
return await mutation;
|
|
835
|
+
} finally {
|
|
836
|
+
mutations.delete(mutation);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
function readMarker(object, params, origin, version) {
|
|
840
|
+
const value = JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
841
|
+
if (typeof value !== "object" || value === null || !("version" in value) || value.version !== 1 || !("jobId" in value) || value.jobId !== params.jobId || !("revisionKey" in value) || value.revisionKey !== params.revisionKey || !("originIdentity" in value) || value.originIdentity !== origin.identity || !("contentLength" in value) || value.contentLength !== origin.contentLength || value.contentLength !== version.contentLength || !("storageEntityTag" in value) || value.storageEntityTag !== version.entityTag || !("sha256" in value) || typeof value.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(value.sha256))
|
|
842
|
+
throw new Error("Parsed result archive cache completion marker is invalid.");
|
|
843
|
+
return {
|
|
844
|
+
version: 1,
|
|
845
|
+
jobId: params.jobId,
|
|
846
|
+
revisionKey: params.revisionKey,
|
|
847
|
+
originIdentity: origin.identity,
|
|
848
|
+
contentLength: version.contentLength,
|
|
849
|
+
storageEntityTag: version.entityTag,
|
|
850
|
+
sha256: value.sha256
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
function readObjectVersion(object) {
|
|
854
|
+
if (!hasStrongEntityTag(object.entityTag) || !isPositiveInteger(object.contentLength))
|
|
855
|
+
throw new Error("Parsed result archive cache requires a strong object version and length.");
|
|
856
|
+
return { entityTag: object.entityTag, contentLength: object.contentLength };
|
|
857
|
+
}
|
|
858
|
+
function validateCachedResponse(response, marker, signal) {
|
|
859
|
+
if (response.headers.get("etag") !== marker.storageEntityTag)
|
|
860
|
+
rejectResponse(
|
|
861
|
+
response,
|
|
862
|
+
new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
|
|
863
|
+
);
|
|
864
|
+
if (response.status !== 206 || !response.body)
|
|
865
|
+
rejectResponse(
|
|
866
|
+
response,
|
|
867
|
+
new ArchiveCacheTransportError("Parsed result archive cache requires a ranged response.")
|
|
868
|
+
);
|
|
869
|
+
const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
|
|
870
|
+
response.headers.get("content-range") ?? ""
|
|
871
|
+
);
|
|
872
|
+
const contentLength = Number(response.headers.get("content-length"));
|
|
873
|
+
if (!range || Number(range[3]) !== marker.contentLength || Number(range[1]) > Number(range[2]) || Number(range[2]) >= marker.contentLength || contentLength !== Number(range[2]) - Number(range[1]) + 1)
|
|
874
|
+
rejectResponse(
|
|
875
|
+
response,
|
|
876
|
+
new ArchiveCacheTransportError("Parsed result archive cache response length changed.")
|
|
877
|
+
);
|
|
878
|
+
let actualLength = 0;
|
|
879
|
+
return new Response(
|
|
880
|
+
response.body.pipeThrough(
|
|
881
|
+
new TransformStream({
|
|
882
|
+
transform(chunk, controller) {
|
|
883
|
+
signal?.throwIfAborted();
|
|
884
|
+
actualLength += chunk.length;
|
|
885
|
+
if (actualLength > contentLength)
|
|
886
|
+
throw new ArchiveCacheTransportError(
|
|
887
|
+
"Parsed result archive cache response length changed."
|
|
888
|
+
);
|
|
889
|
+
controller.enqueue(chunk);
|
|
890
|
+
},
|
|
891
|
+
flush() {
|
|
892
|
+
if (actualLength !== contentLength)
|
|
893
|
+
throw new ArchiveCacheTransportError(
|
|
894
|
+
"Parsed result archive cache response length changed."
|
|
895
|
+
);
|
|
896
|
+
}
|
|
897
|
+
})
|
|
898
|
+
),
|
|
899
|
+
{ status: response.status, statusText: response.statusText, headers: response.headers }
|
|
900
|
+
);
|
|
901
|
+
}
|
|
902
|
+
function rejectResponse(response, error) {
|
|
903
|
+
void response.body?.cancel(error).catch(() => void 0);
|
|
904
|
+
throw error;
|
|
905
|
+
}
|
|
906
|
+
function createMarkerPath(originIdentity, entityTag) {
|
|
907
|
+
const identity = (0, import_crypto.createHash)("sha256").update(JSON.stringify([originIdentity, entityTag])).digest("hex");
|
|
908
|
+
return `.knowhere-sdk/archive-cache/${identity}.json`;
|
|
909
|
+
}
|
|
910
|
+
function createOriginIdentity(jobId, response) {
|
|
911
|
+
return (0, import_crypto.createHash)("sha256").update(
|
|
912
|
+
JSON.stringify([
|
|
913
|
+
jobId,
|
|
914
|
+
response.headers.get("etag"),
|
|
915
|
+
response.headers.get("last-modified"),
|
|
916
|
+
response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
|
|
917
|
+
])
|
|
918
|
+
).digest("hex");
|
|
919
|
+
}
|
|
920
|
+
function readOriginLength(response) {
|
|
921
|
+
const value = Number(
|
|
922
|
+
response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
|
|
923
|
+
);
|
|
924
|
+
return isPositiveInteger(value) ? value : void 0;
|
|
925
|
+
}
|
|
926
|
+
function hasStrongEntityTag(value) {
|
|
927
|
+
return typeof value === "string" && value.length > 0 && !value.startsWith("W/");
|
|
928
|
+
}
|
|
929
|
+
function isPositiveInteger(value) {
|
|
930
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0;
|
|
931
|
+
}
|
|
932
|
+
var import_crypto, ARCHIVE_CACHE_TIMEOUT_MS, ArchiveOriginIntegrityError;
|
|
933
|
+
var init_parsed_document_archive_cache = __esm({
|
|
934
|
+
"src/storage/parsed-document-archive-cache.ts"() {
|
|
935
|
+
"use strict";
|
|
936
|
+
import_crypto = require("crypto");
|
|
937
|
+
init_archive_cache_transport_error();
|
|
938
|
+
ARCHIVE_CACHE_TIMEOUT_MS = 3e4;
|
|
939
|
+
ArchiveOriginIntegrityError = class extends Error {
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
});
|
|
943
|
+
|
|
944
|
+
// src/storage/sequential-parsed-document-archive.ts
|
|
945
|
+
var sequential_parsed_document_archive_exports = {};
|
|
946
|
+
__export(sequential_parsed_document_archive_exports, {
|
|
947
|
+
openSequentialArchive: () => openSequentialArchive
|
|
948
|
+
});
|
|
949
|
+
async function openSequentialArchive(params, identity, validateResponse) {
|
|
950
|
+
let response = await fetch(params.resultUrl, { signal: params.signal });
|
|
951
|
+
if (!response.ok || !response.body)
|
|
952
|
+
throw new Error(`Parsed result download failed (HTTP ${response.status}).`);
|
|
953
|
+
if (validateResponse) response = validateResponse(response);
|
|
954
|
+
else if (createDownloadIdentity(params.jobId, response) !== identity) {
|
|
955
|
+
await response.body.cancel();
|
|
956
|
+
throw new Error("Parsed result archive identity changed before sequential download.");
|
|
957
|
+
}
|
|
958
|
+
const body = response.body;
|
|
959
|
+
if (!body) throw new Error("Parsed result download returned an empty body.");
|
|
960
|
+
let directory;
|
|
961
|
+
try {
|
|
962
|
+
directory = await (0, import_promises2.mkdtemp)((0, import_path2.join)((0, import_os.tmpdir)(), "knowhere-sync-"));
|
|
963
|
+
} catch (error) {
|
|
964
|
+
await body.cancel();
|
|
965
|
+
throw new ParsedDocumentArchiveError(error);
|
|
966
|
+
}
|
|
967
|
+
let file;
|
|
968
|
+
try {
|
|
969
|
+
file = await (0, import_promises2.open)((0, import_path2.join)(directory, "archive.zip"), "w+");
|
|
970
|
+
} catch (error) {
|
|
971
|
+
await body.cancel();
|
|
972
|
+
await (0, import_promises2.rm)(directory, { recursive: true, force: true });
|
|
973
|
+
throw new ParsedDocumentArchiveError(error);
|
|
974
|
+
}
|
|
975
|
+
const input = import_stream.Readable.fromWeb(body);
|
|
976
|
+
const parser = (0, import_parse.default)({ forceStream: true });
|
|
977
|
+
const checksums = /* @__PURE__ */ new Map();
|
|
978
|
+
let archiveSize = 0;
|
|
979
|
+
const spool = new import_stream.Transform({
|
|
980
|
+
transform(chunk, _encoding, callback) {
|
|
981
|
+
archiveSize += chunk.length;
|
|
982
|
+
void file.writeFile(chunk).then(
|
|
983
|
+
() => {
|
|
984
|
+
callback(null, chunk);
|
|
985
|
+
},
|
|
986
|
+
(error) => {
|
|
987
|
+
callback(new ParsedDocumentArchiveError(error));
|
|
988
|
+
}
|
|
989
|
+
);
|
|
990
|
+
}
|
|
991
|
+
});
|
|
992
|
+
const completed = (0, import_promises.pipeline)(input, spool, parser, { signal: params.signal });
|
|
993
|
+
void completed.catch(() => void 0);
|
|
994
|
+
return {
|
|
995
|
+
identity,
|
|
996
|
+
downloadIdentity: identity,
|
|
997
|
+
canSkipEntries: false,
|
|
998
|
+
entries: (async function* () {
|
|
999
|
+
for await (const value of parser) {
|
|
1000
|
+
const entry = value;
|
|
1001
|
+
if (entry.type === "Directory" || !isSnapshotPath(entry.path)) {
|
|
1002
|
+
await entry.autodrain().promise();
|
|
1003
|
+
continue;
|
|
1004
|
+
}
|
|
1005
|
+
const path4 = normalizeResultPath(entry.path);
|
|
1006
|
+
yield {
|
|
1007
|
+
path: path4,
|
|
1008
|
+
contentLength: !(entry.vars.flags & 8) && "uncompressedSize" in entry.vars && typeof entry.vars.uncompressedSize === "number" ? entry.vars.uncompressedSize : void 0,
|
|
1009
|
+
read: () => {
|
|
1010
|
+
let checksum = 0;
|
|
1011
|
+
let size = 0;
|
|
1012
|
+
return import_stream.Readable.toWeb(entry).pipeThrough(
|
|
1013
|
+
new TransformStream({
|
|
1014
|
+
transform(chunk, controller) {
|
|
1015
|
+
checksum = (0, import_zlib.crc32)(chunk, checksum);
|
|
1016
|
+
size += chunk.length;
|
|
1017
|
+
controller.enqueue(chunk);
|
|
1018
|
+
},
|
|
1019
|
+
flush() {
|
|
1020
|
+
checksums.set(path4, { checksum, size });
|
|
1021
|
+
}
|
|
1022
|
+
})
|
|
1023
|
+
);
|
|
1024
|
+
}
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
await completed;
|
|
1028
|
+
const reader = new import_zip.ZipReader(new DiskArchiveReader(file, archiveSize), {
|
|
1029
|
+
useWebWorkers: false
|
|
1030
|
+
});
|
|
1031
|
+
try {
|
|
1032
|
+
const files = (await reader.getEntries()).filter(
|
|
1033
|
+
(entry) => !entry.directory && isSnapshotPath(entry.filename)
|
|
1034
|
+
);
|
|
1035
|
+
if (files.length !== checksums.size)
|
|
1036
|
+
throw new Error("Parsed result ZIP directory does not match sequential entries.");
|
|
1037
|
+
for (const entry of files) {
|
|
1038
|
+
const actual = checksums.get(normalizeResultPath(entry.filename));
|
|
1039
|
+
if (!actual || actual.checksum !== entry.signature || actual.size !== entry.uncompressedSize)
|
|
1040
|
+
throw new Error(`Invalid parsed result ZIP checksum or size: ${entry.filename}`);
|
|
1041
|
+
}
|
|
1042
|
+
} finally {
|
|
1043
|
+
await reader.close();
|
|
1044
|
+
}
|
|
1045
|
+
})(),
|
|
1046
|
+
close: async () => {
|
|
1047
|
+
input.destroy();
|
|
1048
|
+
spool.destroy();
|
|
1049
|
+
parser.destroy();
|
|
1050
|
+
await completed.catch(() => void 0);
|
|
1051
|
+
try {
|
|
1052
|
+
await file.close();
|
|
1053
|
+
} finally {
|
|
1054
|
+
await (0, import_promises2.rm)(directory, { recursive: true, force: true });
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
var import_stream, import_promises, import_promises2, import_os, import_path2, import_zlib, import_zip, import_parse, DiskArchiveReader;
|
|
1060
|
+
var init_sequential_parsed_document_archive = __esm({
|
|
1061
|
+
"src/storage/sequential-parsed-document-archive.ts"() {
|
|
1062
|
+
"use strict";
|
|
1063
|
+
import_stream = require("stream");
|
|
1064
|
+
import_promises = require("stream/promises");
|
|
1065
|
+
import_promises2 = require("fs/promises");
|
|
1066
|
+
import_os = require("os");
|
|
1067
|
+
import_path2 = require("path");
|
|
1068
|
+
import_zlib = require("zlib");
|
|
1069
|
+
import_zip = require("@zip.js/zip.js");
|
|
1070
|
+
import_parse = __toESM(require("unzipper/lib/parse.js"));
|
|
1071
|
+
init_parsed_document_archive_error();
|
|
1072
|
+
init_parsed_document_storage();
|
|
1073
|
+
init_streamed_parsed_document_sync();
|
|
1074
|
+
DiskArchiveReader = class extends import_zip.Reader {
|
|
1075
|
+
constructor(file, size) {
|
|
1076
|
+
super(file);
|
|
1077
|
+
this.file = file;
|
|
1078
|
+
this.size = size;
|
|
1079
|
+
}
|
|
1080
|
+
file;
|
|
1081
|
+
async readUint8Array(index, length) {
|
|
1082
|
+
const body = new Uint8Array(length);
|
|
1083
|
+
const { bytesRead } = await this.file.read(body, 0, length, index);
|
|
1084
|
+
if (bytesRead !== length) throw new Error("Truncated parsed result ZIP.");
|
|
1085
|
+
return body;
|
|
1086
|
+
}
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
});
|
|
1090
|
+
|
|
1091
|
+
// src/storage/streamed-parsed-document-sync.ts
|
|
1092
|
+
async function syncRemoteResultArchive(params) {
|
|
1093
|
+
params.signal?.throwIfAborted();
|
|
1094
|
+
validateArchiveSyncLimits(params.limits);
|
|
1095
|
+
const startedAt = Date.now();
|
|
1096
|
+
const maxEntries = params.limits?.maxEntriesPerSync ?? 100;
|
|
1097
|
+
const deadlineMs = params.limits?.archiveSyncDeadlineMs ?? 12e4;
|
|
1098
|
+
const checkpointFileInterval = params.limits?.checkpointFileInterval ?? 1;
|
|
1099
|
+
const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
|
|
1100
|
+
const controller = new AbortController();
|
|
1101
|
+
const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
|
|
1102
|
+
const transferParams = { ...params, signal };
|
|
1103
|
+
const previous = await params.storage.readSyncProgress(params);
|
|
1104
|
+
const source = await openArchive(transferParams, previous);
|
|
1105
|
+
const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
|
|
1106
|
+
const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
|
|
1107
|
+
const entryHashes = [];
|
|
1108
|
+
const successfulHashes = /* @__PURE__ */ new Map();
|
|
1109
|
+
const running = /* @__PURE__ */ new Set();
|
|
1110
|
+
const paths = /* @__PURE__ */ new Set();
|
|
1111
|
+
let entryIndex = 0;
|
|
1112
|
+
let startedEntries = 0;
|
|
1113
|
+
let successfulEntries = 0;
|
|
1114
|
+
let checkpointEntry = nextEntry;
|
|
1115
|
+
let checkpointAt = Date.now();
|
|
1116
|
+
let failure;
|
|
1117
|
+
let archiveCacheBypassed = source.archiveCacheBypassed;
|
|
1118
|
+
async function flushProgress(isSegmentBoundary, isRecovering = false) {
|
|
1119
|
+
let contiguousEntry = checkpointEntry;
|
|
1120
|
+
while (successfulHashes.has(contiguousEntry)) {
|
|
1121
|
+
entryHashes[contiguousEntry] = successfulHashes.get(contiguousEntry);
|
|
1122
|
+
contiguousEntry += 1;
|
|
1123
|
+
}
|
|
1124
|
+
if (contiguousEntry === checkpointEntry) return;
|
|
1125
|
+
await params.storage.writeSyncProgress(
|
|
1126
|
+
{
|
|
1127
|
+
documentId: params.documentId,
|
|
1128
|
+
revisionKey: params.revisionKey,
|
|
1129
|
+
pinnedRevisionKey: params.revisionKey,
|
|
1130
|
+
archiveIdentity: source.identity,
|
|
1131
|
+
archiveCacheBypassed,
|
|
1132
|
+
archiveEntryHashes: entryHashes.slice(0, contiguousEntry),
|
|
1133
|
+
nextArchiveEntry: contiguousEntry,
|
|
1134
|
+
status: "running",
|
|
1135
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1136
|
+
},
|
|
1137
|
+
{ signal: isRecovering ? void 0 : signal, isSegmentBoundary }
|
|
1138
|
+
);
|
|
1139
|
+
for (let index = checkpointEntry; index < contiguousEntry; index += 1)
|
|
1140
|
+
successfulHashes.delete(index);
|
|
1141
|
+
checkpointEntry = contiguousEntry;
|
|
1142
|
+
successfulEntries = successfulHashes.size;
|
|
1143
|
+
checkpointAt = Date.now();
|
|
1144
|
+
}
|
|
1145
|
+
function startEntry(entry, index) {
|
|
1146
|
+
const task = copyEntry(transferParams, entry).then(
|
|
1147
|
+
(hash) => {
|
|
1148
|
+
successfulHashes.set(index, hash);
|
|
1149
|
+
successfulEntries += 1;
|
|
1150
|
+
},
|
|
1151
|
+
(error) => {
|
|
1152
|
+
if (failure === void 0)
|
|
1153
|
+
failure = error instanceof Error ? error : new Error(String(error));
|
|
1154
|
+
controller.abort(error);
|
|
1155
|
+
}
|
|
1156
|
+
).finally(() => {
|
|
1157
|
+
running.delete(task);
|
|
1158
|
+
});
|
|
1159
|
+
running.add(task);
|
|
1160
|
+
}
|
|
1161
|
+
async function drainEntries() {
|
|
1162
|
+
await Promise.all(running);
|
|
1163
|
+
if (failure !== void 0) throw failure;
|
|
1164
|
+
signal.throwIfAborted();
|
|
1165
|
+
}
|
|
1166
|
+
try {
|
|
1167
|
+
if (previous?.archiveIdentity && previous.archiveIdentity !== source.identity)
|
|
1168
|
+
throw new Error("Parsed result archive identity changed; start a new revision.");
|
|
1169
|
+
if (previous?.pinnedRevisionKey && previous.pinnedRevisionKey !== params.revisionKey)
|
|
1170
|
+
throw new Error("Parsed result revision changed while resuming archive sync.");
|
|
1171
|
+
for await (const entry of source.entries) {
|
|
1172
|
+
if (failure !== void 0) throw failure;
|
|
1173
|
+
signal.throwIfAborted();
|
|
1174
|
+
if (paths.has(entry.path)) throw new Error(`Duplicate parsed result path: ${entry.path}`);
|
|
1175
|
+
paths.add(entry.path);
|
|
1176
|
+
if (entryIndex < nextEntry) {
|
|
1177
|
+
const previousHash = previous?.archiveEntryHashes?.[entryIndex];
|
|
1178
|
+
if (!previousHash) throw new Error("Parsed archive checkpoint is missing an entry hash.");
|
|
1179
|
+
if (!source.canSkipEntries && await copyEntry(transferParams, entry, true) !== previousHash)
|
|
1180
|
+
throw new Error("Parsed result archive entry changed while resuming.");
|
|
1181
|
+
entryHashes.push(previousHash);
|
|
1182
|
+
entryIndex += 1;
|
|
1183
|
+
continue;
|
|
1184
|
+
}
|
|
1185
|
+
if (startedEntries > 0 && (startedEntries >= maxEntries || Date.now() - startedAt >= deadlineMs)) {
|
|
1186
|
+
await drainEntries();
|
|
1187
|
+
await flushProgress(true);
|
|
1188
|
+
return { documentId: params.documentId, revisionKey: params.revisionKey, completed: false };
|
|
1189
|
+
}
|
|
1190
|
+
if (successfulEntries >= checkpointFileInterval || Date.now() - checkpointAt >= checkpointIntervalMs) {
|
|
1191
|
+
await drainEntries();
|
|
1192
|
+
await flushProgress(false);
|
|
1193
|
+
}
|
|
1194
|
+
startEntry(entry, entryIndex);
|
|
1195
|
+
entryIndex += 1;
|
|
1196
|
+
startedEntries += 1;
|
|
1197
|
+
if (running.size >= concurrency) await Promise.race(running);
|
|
1198
|
+
}
|
|
1199
|
+
await drainEntries();
|
|
1200
|
+
await flushProgress(true);
|
|
1201
|
+
await validateRequiredObjects(params, paths);
|
|
1202
|
+
await params.verifyRevision?.();
|
|
1203
|
+
signal.throwIfAborted();
|
|
1204
|
+
const commit = {
|
|
1205
|
+
version: 1,
|
|
1206
|
+
documentId: params.documentId,
|
|
1207
|
+
revisionKey: params.revisionKey,
|
|
1208
|
+
source: "resultZip",
|
|
1209
|
+
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1210
|
+
};
|
|
1211
|
+
await params.storage.writeObject({
|
|
1212
|
+
documentId: params.documentId,
|
|
1213
|
+
revisionKey: params.revisionKey,
|
|
1214
|
+
path: parsedDocumentCommitPath,
|
|
1215
|
+
body: Buffer.from(JSON.stringify(commit, null, 2)),
|
|
1216
|
+
contentType: "application/json; charset=utf-8"
|
|
1217
|
+
});
|
|
1218
|
+
await params.storage.writeSyncProgress(
|
|
1219
|
+
{
|
|
1220
|
+
documentId: params.documentId,
|
|
1221
|
+
revisionKey: params.revisionKey,
|
|
1222
|
+
pinnedRevisionKey: params.revisionKey,
|
|
1223
|
+
archiveIdentity: source.identity,
|
|
1224
|
+
archiveCacheBypassed,
|
|
1225
|
+
archiveEntryHashes: entryHashes,
|
|
1226
|
+
nextArchiveEntry: entryIndex,
|
|
1227
|
+
status: "completed",
|
|
1228
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1229
|
+
},
|
|
1230
|
+
{ signal, isSegmentBoundary: true }
|
|
1231
|
+
);
|
|
1232
|
+
if (params.retainResultArchive && !source.isCached) {
|
|
1233
|
+
try {
|
|
1234
|
+
await retainArchive(transferParams, source.downloadIdentity);
|
|
1235
|
+
} catch {
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
return {
|
|
1239
|
+
documentId: params.documentId,
|
|
1240
|
+
revisionKey: params.revisionKey,
|
|
1241
|
+
completed: true,
|
|
1242
|
+
commit
|
|
1243
|
+
};
|
|
1244
|
+
} catch (error) {
|
|
1245
|
+
controller.abort(error);
|
|
1246
|
+
await Promise.all(running);
|
|
1247
|
+
const hasCacheTransportFailure = error instanceof ArchiveCacheTransportError || failure instanceof ArchiveCacheTransportError;
|
|
1248
|
+
if (hasCacheTransportFailure) archiveCacheBypassed = true;
|
|
1249
|
+
await flushProgress(true, true).catch(() => void 0);
|
|
1250
|
+
if (hasCacheTransportFailure) {
|
|
1251
|
+
await persistArchiveCacheBypass(
|
|
1252
|
+
{ ...params, signal: void 0 },
|
|
1253
|
+
{
|
|
1254
|
+
documentId: params.documentId,
|
|
1255
|
+
revisionKey: params.revisionKey,
|
|
1256
|
+
pinnedRevisionKey: params.revisionKey,
|
|
1257
|
+
archiveIdentity: source.identity,
|
|
1258
|
+
archiveEntryHashes: entryHashes.slice(0, checkpointEntry),
|
|
1259
|
+
nextArchiveEntry: checkpointEntry,
|
|
1260
|
+
status: "running",
|
|
1261
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1262
|
+
}
|
|
1263
|
+
).catch(() => void 0);
|
|
1264
|
+
}
|
|
1265
|
+
throw failure ?? error;
|
|
1266
|
+
} finally {
|
|
1267
|
+
controller.abort();
|
|
1268
|
+
await Promise.all(running);
|
|
1269
|
+
await source.close();
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
function validateArchiveSyncLimits(limits) {
|
|
1273
|
+
const names = [
|
|
1274
|
+
"maxEntriesPerSync",
|
|
1275
|
+
"archiveSyncDeadlineMs",
|
|
1276
|
+
"maxConcurrentFiles",
|
|
1277
|
+
"checkpointFileInterval",
|
|
1278
|
+
"checkpointIntervalMs"
|
|
1279
|
+
];
|
|
1280
|
+
for (const name of names) {
|
|
1281
|
+
const value = limits?.[name];
|
|
1282
|
+
if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0))
|
|
1283
|
+
throw new ValidationError(`${name} must be a positive safe integer.`);
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
async function copyEntry(params, entry, isVerification = false) {
|
|
1287
|
+
const hash = (0, import_crypto2.createHash)("sha256");
|
|
1288
|
+
const body = entry.read().pipeThrough(
|
|
1289
|
+
new TransformStream({
|
|
1290
|
+
transform(chunk, controller) {
|
|
1291
|
+
params.signal?.throwIfAborted();
|
|
1292
|
+
hash.update(chunk);
|
|
1293
|
+
controller.enqueue(chunk);
|
|
1294
|
+
}
|
|
1295
|
+
})
|
|
1296
|
+
);
|
|
1297
|
+
if (isVerification)
|
|
1298
|
+
await body.pipeTo(new WritableStream(), { signal: params.signal });
|
|
1299
|
+
else
|
|
1300
|
+
await writeStream(params.storage, {
|
|
1301
|
+
documentId: params.documentId,
|
|
1302
|
+
revisionKey: params.revisionKey,
|
|
1303
|
+
path: entry.path,
|
|
1304
|
+
body,
|
|
1305
|
+
contentType: inferContentTypeFromPath(entry.path),
|
|
1306
|
+
contentLength: entry.contentLength,
|
|
1307
|
+
signal: params.signal
|
|
1308
|
+
});
|
|
1309
|
+
return (0, import_crypto2.createHash)("sha256").update(entry.path).update(hash.digest()).digest("hex");
|
|
1310
|
+
}
|
|
1311
|
+
async function openArchive(params, previous) {
|
|
1312
|
+
const prepared = params.cacheResultArchive ? await prepareCachedArchive({
|
|
1313
|
+
...params,
|
|
1314
|
+
archiveCacheBypassed: previous?.archiveCacheBypassed,
|
|
1315
|
+
onBypass: async () => {
|
|
1316
|
+
await persistArchiveCacheBypass(params, previous);
|
|
1317
|
+
}
|
|
1318
|
+
}) : void 0;
|
|
1319
|
+
try {
|
|
1320
|
+
return await openArchiveTransport(params, previous, prepared);
|
|
1321
|
+
} catch (error) {
|
|
1322
|
+
if (!prepared?.isCached) throw error;
|
|
1323
|
+
params.signal?.throwIfAborted();
|
|
1324
|
+
const bypass = await persistArchiveCacheBypass(params, previous);
|
|
1325
|
+
return openArchive({ ...params, cacheResultArchive: false }, bypass);
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
async function persistArchiveCacheBypass(params, previous) {
|
|
1329
|
+
const progress = {
|
|
1330
|
+
...previous?.status === "completed" ? {} : previous,
|
|
1331
|
+
documentId: params.documentId,
|
|
1332
|
+
revisionKey: params.revisionKey,
|
|
1333
|
+
pinnedRevisionKey: params.revisionKey,
|
|
1334
|
+
archiveCacheBypassed: true,
|
|
1335
|
+
status: "running",
|
|
1336
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1337
|
+
};
|
|
1338
|
+
await params.storage.writeSyncProgress(progress, { signal: params.signal });
|
|
1339
|
+
return progress;
|
|
1340
|
+
}
|
|
1341
|
+
async function openArchiveTransport(params, previous, prepared) {
|
|
1342
|
+
const resultUrl = prepared?.resultUrl ?? params.resultUrl;
|
|
1343
|
+
const archiveCacheBypassed = prepared?.archiveCacheBypassed ?? previous?.archiveCacheBypassed;
|
|
1344
|
+
let probe = await fetch(resultUrl, {
|
|
1345
|
+
headers: { Range: "bytes=0-0" },
|
|
1346
|
+
signal: params.signal
|
|
1347
|
+
});
|
|
1348
|
+
if (!probe.ok) {
|
|
1349
|
+
await probe.body?.cancel();
|
|
1350
|
+
throw new Error(`Parsed result download failed (HTTP ${probe.status}).`);
|
|
1351
|
+
}
|
|
1352
|
+
if (prepared) probe = prepared.validateResponse(probe);
|
|
1353
|
+
const identity = prepared?.downloadIdentity ?? createDownloadIdentity(params.jobId, probe);
|
|
1354
|
+
const supportsRange = probe.status === 206 && probe.headers.has("content-range");
|
|
1355
|
+
await probe.body?.cancel();
|
|
1356
|
+
if (!supportsRange) {
|
|
1357
|
+
const { openSequentialArchive: openSequentialArchive2 } = await Promise.resolve().then(() => (init_sequential_parsed_document_archive(), sequential_parsed_document_archive_exports));
|
|
1358
|
+
const source = await openSequentialArchive2(
|
|
1359
|
+
{ ...params, resultUrl },
|
|
1360
|
+
identity,
|
|
1361
|
+
prepared?.isCached ? prepared.validateResponse : void 0
|
|
1362
|
+
);
|
|
1363
|
+
return { ...source, archiveCacheBypassed };
|
|
1364
|
+
}
|
|
1365
|
+
const options = {
|
|
1366
|
+
forceRangeRequests: true,
|
|
1367
|
+
preventHeadRequest: true,
|
|
1368
|
+
checkResourceChanges: true,
|
|
1369
|
+
fetch: async (input, init) => {
|
|
1370
|
+
let response;
|
|
1371
|
+
try {
|
|
1372
|
+
response = await fetch(input, { ...init, signal: params.signal });
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
params.signal?.throwIfAborted();
|
|
1375
|
+
if (prepared?.isCached)
|
|
1376
|
+
throw new ArchiveCacheTransportError(
|
|
1377
|
+
"Parsed result archive cache transport unavailable."
|
|
1378
|
+
);
|
|
1379
|
+
throw error;
|
|
1380
|
+
}
|
|
1381
|
+
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1382
|
+
if (response.status !== 206) {
|
|
1383
|
+
await response.body?.cancel();
|
|
1384
|
+
throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
|
|
1385
|
+
}
|
|
1386
|
+
return prepared ? prepared.validateResponse(response) : response;
|
|
1387
|
+
}
|
|
1388
|
+
};
|
|
1389
|
+
const reader = new import_zip2.ZipReader(new import_zip2.HttpRangeReader(resultUrl, options), {
|
|
1390
|
+
useWebWorkers: false,
|
|
1391
|
+
checkCrc32: true
|
|
1392
|
+
});
|
|
1393
|
+
const producers = /* @__PURE__ */ new Set();
|
|
1394
|
+
try {
|
|
1395
|
+
const entries = await reader.getEntries();
|
|
1396
|
+
const files = entries.filter(
|
|
1397
|
+
(entry) => !entry.directory && isSnapshotPath(entry.filename)
|
|
1398
|
+
);
|
|
1399
|
+
const archiveIdentity = (0, import_crypto2.createHash)("sha256").update(identity).update(
|
|
1400
|
+
JSON.stringify(
|
|
1401
|
+
files.map((entry) => [entry.filename, entry.uncompressedSize, entry.signature])
|
|
1402
|
+
)
|
|
1403
|
+
).digest("hex");
|
|
1404
|
+
return {
|
|
1405
|
+
identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
|
|
1406
|
+
downloadIdentity: identity,
|
|
1407
|
+
archiveCacheBypassed,
|
|
1408
|
+
isCached: prepared?.isCached,
|
|
1409
|
+
canSkipEntries: previous?.archiveIdentity !== identity,
|
|
1410
|
+
entries: (function* () {
|
|
1411
|
+
for (const entry of files)
|
|
1412
|
+
yield {
|
|
1413
|
+
path: normalizeResultPath(entry.filename),
|
|
1414
|
+
contentLength: entry.uncompressedSize,
|
|
1415
|
+
read: () => readRangeEntry(entry, params.signal, producers)
|
|
1416
|
+
};
|
|
1417
|
+
})(),
|
|
1418
|
+
close: async () => {
|
|
1419
|
+
await Promise.all(producers);
|
|
1420
|
+
await reader.close();
|
|
1421
|
+
}
|
|
1422
|
+
};
|
|
1423
|
+
} catch (error) {
|
|
1424
|
+
await reader.close();
|
|
1425
|
+
throw error;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
function readRangeEntry(entry, signal, producers) {
|
|
1429
|
+
const stream = new TransformStream();
|
|
1430
|
+
const producer = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(
|
|
1431
|
+
() => void 0,
|
|
1432
|
+
async (error) => {
|
|
1433
|
+
try {
|
|
1434
|
+
await stream.writable.abort(error);
|
|
1435
|
+
} catch {
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
).finally(() => {
|
|
1439
|
+
producers.delete(producer);
|
|
1440
|
+
});
|
|
1441
|
+
producers.add(producer);
|
|
1442
|
+
return stream.readable;
|
|
1443
|
+
}
|
|
1444
|
+
async function writeStream(storage, params) {
|
|
1445
|
+
if (storage.writeObjectStream) {
|
|
1446
|
+
try {
|
|
1447
|
+
await storage.writeObjectStream(params);
|
|
1448
|
+
} catch (error) {
|
|
1449
|
+
await params.body.cancel(error).catch(() => void 0);
|
|
1450
|
+
throw error;
|
|
1451
|
+
}
|
|
1452
|
+
return;
|
|
1453
|
+
}
|
|
1454
|
+
const parts = [];
|
|
1455
|
+
for await (const part of params.body) {
|
|
1456
|
+
params.signal?.throwIfAborted();
|
|
1457
|
+
parts.push(part);
|
|
1458
|
+
}
|
|
1459
|
+
await storage.writeObject({ ...params, body: Buffer.concat(parts) });
|
|
1460
|
+
}
|
|
1461
|
+
async function validateRequiredObjects(params, paths) {
|
|
1462
|
+
const required = /* @__PURE__ */ new Set(["manifest.json", "chunks.json"]);
|
|
1463
|
+
for (const path4 of required)
|
|
1464
|
+
if (!paths.has(path4)) throw new Error(`Required parsed result file missing: ${path4}`);
|
|
1465
|
+
const manifestObject = await params.storage.readObject({ ...params, path: "manifest.json" });
|
|
1466
|
+
const chunksObject = await params.storage.readObject({ ...params, path: "chunks.json" });
|
|
1467
|
+
if (!manifestObject || !chunksObject)
|
|
1468
|
+
throw new Error("Required parsed result manifest.json or chunks.json was not persisted.");
|
|
1469
|
+
const manifest = JSON.parse(Buffer.from(manifestObject.body).toString("utf8"));
|
|
1470
|
+
const payload = JSON.parse(Buffer.from(chunksObject.body).toString("utf8"));
|
|
1471
|
+
if (!isRecord2(manifest)) throw new Error("Invalid parsed result manifest.json.");
|
|
1472
|
+
const manifestJobId = manifest.job_id ?? manifest.jobId;
|
|
1473
|
+
if (manifestJobId !== params.jobId)
|
|
1474
|
+
throw new Error("Parsed result manifest job identity differs from the pinned job.");
|
|
1475
|
+
const chunks = Array.isArray(payload) ? payload : isRecord2(payload) ? payload.chunks : void 0;
|
|
1476
|
+
if (!Array.isArray(chunks)) throw new Error("Invalid parsed result chunks.json.");
|
|
1477
|
+
if (isRecord2(manifest.files)) {
|
|
1478
|
+
for (const path4 of Object.values(manifest.files))
|
|
1479
|
+
if (typeof path4 === "string") required.add(normalizeResultPath(path4));
|
|
1480
|
+
}
|
|
1481
|
+
for (const chunk of chunks) {
|
|
1482
|
+
if (!isRecord2(chunk)) throw new Error("Invalid parsed result chunk.");
|
|
1483
|
+
const metadata = chunk.metadata;
|
|
1484
|
+
const chunkType = chunk.type ?? chunk.chunk_type ?? chunk.chunkType;
|
|
1485
|
+
const filePath = chunk.file_path ?? chunk.filePath ?? (isRecord2(metadata) ? metadata.file_path ?? metadata.filePath : void 0) ?? (chunkType === "image" || chunkType === "table" ? chunk.path : void 0);
|
|
1486
|
+
if (typeof filePath === "string") required.add(normalizeResultPath(filePath));
|
|
1487
|
+
if (!isRecord2(metadata)) continue;
|
|
1488
|
+
const pageAssets = metadata.page_assets ?? metadata.pageAssets;
|
|
1489
|
+
if (!Array.isArray(pageAssets)) continue;
|
|
1490
|
+
for (const asset of pageAssets)
|
|
1491
|
+
if (isRecord2(asset)) {
|
|
1492
|
+
const ref = asset.artifact_ref ?? asset.artifactRef;
|
|
1493
|
+
if (typeof ref === "string") required.add(normalizeResultPath(ref));
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
for (const path4 of required)
|
|
1497
|
+
if (!paths.has(path4)) throw new Error(`Required parsed result file missing: ${path4}`);
|
|
1498
|
+
}
|
|
1499
|
+
async function retainArchive(params, expectedIdentity) {
|
|
1500
|
+
const response = await fetch(params.resultUrl, { signal: params.signal });
|
|
1501
|
+
if (!response.ok || !response.body)
|
|
1502
|
+
throw new Error(`Optional archive download failed (HTTP ${response.status}).`);
|
|
1503
|
+
if (createDownloadIdentity(params.jobId, response) !== expectedIdentity) {
|
|
1504
|
+
await response.body.cancel();
|
|
1505
|
+
throw new Error("Optional result archive identity changed.");
|
|
1506
|
+
}
|
|
1507
|
+
await writeStream(params.storage, {
|
|
1508
|
+
documentId: params.documentId,
|
|
1509
|
+
revisionKey: params.revisionKey,
|
|
1510
|
+
path: "result.zip",
|
|
1511
|
+
body: response.body,
|
|
1512
|
+
contentType: "application/zip",
|
|
1513
|
+
signal: params.signal
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
function isSnapshotPath(path4) {
|
|
1517
|
+
const normalized = normalizeResultPath(path4.replace(/\/$/, ""));
|
|
1518
|
+
return normalized !== "result.zip" && normalized !== ".knowhere-sdk" && !normalized.startsWith(".knowhere-sdk/");
|
|
1519
|
+
}
|
|
1520
|
+
function isRecord2(value) {
|
|
1521
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1522
|
+
}
|
|
1523
|
+
function createDownloadIdentity(jobId, response) {
|
|
1524
|
+
return (0, import_crypto2.createHash)("sha256").update(
|
|
1525
|
+
JSON.stringify([
|
|
1526
|
+
jobId,
|
|
1527
|
+
response.headers.get("etag"),
|
|
1528
|
+
response.headers.get("last-modified"),
|
|
1529
|
+
response.headers.get("content-range")?.split("/").at(-1) ?? response.headers.get("content-length")
|
|
1530
|
+
])
|
|
1531
|
+
).digest("hex");
|
|
1532
|
+
}
|
|
1533
|
+
var import_crypto2, import_zip2;
|
|
1534
|
+
var init_streamed_parsed_document_sync = __esm({
|
|
1535
|
+
"src/storage/streamed-parsed-document-sync.ts"() {
|
|
1536
|
+
"use strict";
|
|
1537
|
+
import_crypto2 = require("crypto");
|
|
1538
|
+
init_errors();
|
|
1539
|
+
init_archive_cache_transport_error();
|
|
1540
|
+
init_parsed_document_archive_cache();
|
|
1541
|
+
import_zip2 = require("@zip.js/zip.js");
|
|
1542
|
+
init_parsed_document_storage();
|
|
1543
|
+
}
|
|
1544
|
+
});
|
|
1545
|
+
|
|
1546
|
+
// src/index.ts
|
|
1547
|
+
var index_exports = {};
|
|
1548
|
+
__export(index_exports, {
|
|
1549
|
+
APIError: () => APIError,
|
|
1550
|
+
AuthenticationError: () => AuthenticationError,
|
|
1551
|
+
BadRequestError: () => BadRequestError,
|
|
1552
|
+
ChecksumError: () => ChecksumError,
|
|
1553
|
+
ConflictError: () => ConflictError,
|
|
1554
|
+
DiskParsedDocumentStorage: () => DiskParsedDocumentStorage,
|
|
1555
|
+
Documents: () => Documents,
|
|
1556
|
+
GatewayTimeoutError: () => GatewayTimeoutError,
|
|
1557
|
+
InternalServerError: () => InternalServerError,
|
|
1558
|
+
InvalidStateError: () => InvalidStateError,
|
|
1559
|
+
JobFailedError: () => JobFailedError,
|
|
1560
|
+
Jobs: () => Jobs,
|
|
1561
|
+
Knowhere: () => Knowhere,
|
|
1562
|
+
KnowhereError: () => KnowhereError,
|
|
1563
|
+
Knowledge: () => Knowledge,
|
|
1564
|
+
LocalKnowledgeStore: () => LocalKnowledgeStore,
|
|
1565
|
+
NODE_SDK_DOCUMENT_METADATA_DEFAULTS: () => NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
|
|
1566
|
+
NetworkError: () => NetworkError,
|
|
1567
|
+
NotFoundError: () => NotFoundError,
|
|
1568
|
+
ParsedDocumentArchiveError: () => ParsedDocumentArchiveError,
|
|
1569
|
+
PaymentRequiredError: () => PaymentRequiredError,
|
|
1570
|
+
PermissionDeniedError: () => PermissionDeniedError,
|
|
1571
|
+
PollingTimeoutError: () => PollingTimeoutError,
|
|
1572
|
+
RateLimitError: () => RateLimitError,
|
|
1573
|
+
Retrieval: () => Retrieval,
|
|
1574
|
+
ServiceUnavailableError: () => ServiceUnavailableError,
|
|
1575
|
+
TimeoutError: () => TimeoutError,
|
|
1576
|
+
VERSION: () => VERSION,
|
|
1577
|
+
ValidationError: () => ValidationError,
|
|
1578
|
+
default: () => Knowhere,
|
|
1579
|
+
mergeDocumentMetadataDefaults: () => mergeDocumentMetadataDefaults,
|
|
1580
|
+
storeParseResultAssets: () => storeParseResultAssets,
|
|
1581
|
+
syncParseResultToParsedDocumentStorage: () => syncParseResultToParsedDocumentStorage
|
|
1582
|
+
});
|
|
1583
|
+
module.exports = __toCommonJS(index_exports);
|
|
1584
|
+
|
|
1585
|
+
// src/client.ts
|
|
1586
|
+
var import_path4 = __toESM(require("path"));
|
|
1587
|
+
|
|
1588
|
+
// src/lib/http-client.ts
|
|
1589
|
+
var import_axios = __toESM(require("axios"));
|
|
1590
|
+
|
|
1591
|
+
// src/version.ts
|
|
1592
|
+
var VERSION = "2.1.3";
|
|
1593
|
+
|
|
1594
|
+
// src/lib/http-client.ts
|
|
1595
|
+
init_constants();
|
|
1596
|
+
init_errors();
|
|
1597
|
+
init_utils();
|
|
391
1598
|
|
|
392
1599
|
// src/lib/retry.ts
|
|
1600
|
+
init_constants();
|
|
1601
|
+
init_utils();
|
|
393
1602
|
function getErrorCode(error) {
|
|
394
1603
|
const errorWithResponse = error;
|
|
395
1604
|
return errorWithResponse?.response?.data?.error?.code ?? errorWithResponse?.response?.data?.code ?? errorWithResponse?.code;
|
|
@@ -780,6 +1989,7 @@ function mergeDocumentMetadataDefaults(defaults, provided) {
|
|
|
780
1989
|
|
|
781
1990
|
// src/lib/upload.ts
|
|
782
1991
|
var import_fs = require("fs");
|
|
1992
|
+
init_errors();
|
|
783
1993
|
async function uploadFile(httpClient, uploadUrl, file, options) {
|
|
784
1994
|
let data;
|
|
785
1995
|
let contentLength;
|
|
@@ -815,6 +2025,9 @@ function isReadStream(obj) {
|
|
|
815
2025
|
}
|
|
816
2026
|
|
|
817
2027
|
// src/lib/polling.ts
|
|
2028
|
+
init_constants();
|
|
2029
|
+
init_errors();
|
|
2030
|
+
init_utils();
|
|
818
2031
|
async function pollJobStatus(httpClient, jobId, options) {
|
|
819
2032
|
const pollInterval = options?.pollInterval ?? DEFAULT_POLL_INTERVAL;
|
|
820
2033
|
const pollTimeout = options?.pollTimeout ?? DEFAULT_POLL_TIMEOUT;
|
|
@@ -865,6 +2078,8 @@ async function pollJobStatus(httpClient, jobId, options) {
|
|
|
865
2078
|
var import_jszip = __toESM(require("jszip"));
|
|
866
2079
|
var import_fs2 = require("fs");
|
|
867
2080
|
var import_path = require("path");
|
|
2081
|
+
init_errors();
|
|
2082
|
+
init_utils();
|
|
868
2083
|
async function parseResult(httpClient, resultUrl, options) {
|
|
869
2084
|
const zipBuffer = await httpClient.download(resultUrl);
|
|
870
2085
|
if (options?.verifyChecksum !== false) {
|
|
@@ -1367,6 +2582,8 @@ function isMissingFileError(error) {
|
|
|
1367
2582
|
}
|
|
1368
2583
|
|
|
1369
2584
|
// src/resources/jobs.ts
|
|
2585
|
+
init_utils();
|
|
2586
|
+
init_errors();
|
|
1370
2587
|
var Jobs = class extends BaseResource {
|
|
1371
2588
|
pendingUploadJobs = /* @__PURE__ */ new Map();
|
|
1372
2589
|
/**
|
|
@@ -1607,7 +2824,8 @@ var Documents = class extends BaseResource {
|
|
|
1607
2824
|
};
|
|
1608
2825
|
|
|
1609
2826
|
// src/knowledge/knowledge.ts
|
|
1610
|
-
var
|
|
2827
|
+
var import_crypto4 = require("crypto");
|
|
2828
|
+
init_errors();
|
|
1611
2829
|
|
|
1612
2830
|
// src/storage/asset-storage.ts
|
|
1613
2831
|
var import_jszip2 = __toESM(require("jszip"));
|
|
@@ -1871,220 +3089,16 @@ function isRecord(value) {
|
|
|
1871
3089
|
return typeof value === "object" && value !== null;
|
|
1872
3090
|
}
|
|
1873
3091
|
|
|
1874
|
-
// src/
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
var markdownContentType = "text/markdown; charset=utf-8";
|
|
1879
|
-
var csvContentType = "text/csv; charset=utf-8";
|
|
1880
|
-
var htmlContentType = "text/html; charset=utf-8";
|
|
1881
|
-
var binaryContentType = "application/octet-stream";
|
|
1882
|
-
async function syncParseResultToParsedDocumentStorage(params) {
|
|
1883
|
-
const documentId = params.documentId ?? params.result.documentId ?? params.result.jobId;
|
|
1884
|
-
const revisionKey = params.revisionKey ?? params.result.jobId;
|
|
1885
|
-
const objects = await collectResultLayoutObjects(params.result);
|
|
1886
|
-
for (const object of objects) {
|
|
1887
|
-
await params.storage.writeObject({
|
|
1888
|
-
documentId,
|
|
1889
|
-
revisionKey,
|
|
1890
|
-
...object
|
|
1891
|
-
});
|
|
1892
|
-
}
|
|
1893
|
-
const commit = {
|
|
1894
|
-
version: 1,
|
|
1895
|
-
documentId,
|
|
1896
|
-
revisionKey,
|
|
1897
|
-
source: params.source ?? "resultZip",
|
|
1898
|
-
committedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1899
|
-
};
|
|
1900
|
-
await writeParsedDocumentJsonObject(params.storage, {
|
|
1901
|
-
documentId,
|
|
1902
|
-
revisionKey,
|
|
1903
|
-
path: parsedDocumentCommitPath,
|
|
1904
|
-
value: commit
|
|
1905
|
-
});
|
|
1906
|
-
return {
|
|
1907
|
-
result: params.result,
|
|
1908
|
-
documentId,
|
|
1909
|
-
revisionKey,
|
|
1910
|
-
commit
|
|
1911
|
-
};
|
|
1912
|
-
}
|
|
1913
|
-
async function readParsedDocumentCommit(params) {
|
|
1914
|
-
const object = await params.storage.readObject({
|
|
1915
|
-
documentId: params.documentId,
|
|
1916
|
-
revisionKey: params.revisionKey,
|
|
1917
|
-
path: parsedDocumentCommitPath
|
|
1918
|
-
});
|
|
1919
|
-
if (!object) {
|
|
1920
|
-
return null;
|
|
1921
|
-
}
|
|
1922
|
-
const commit = readJsonObject(object);
|
|
1923
|
-
return commit.revisionKey === params.revisionKey && commit.documentId === params.documentId ? commit : null;
|
|
1924
|
-
}
|
|
1925
|
-
function readJsonObject(object) {
|
|
1926
|
-
return JSON.parse(Buffer.from(object.body).toString("utf8"));
|
|
1927
|
-
}
|
|
1928
|
-
async function collectResultLayoutObjects(result) {
|
|
1929
|
-
const objectsByPath = /* @__PURE__ */ new Map();
|
|
1930
|
-
const rawZipObjects = await collectRawZipObjects(result.rawZip);
|
|
1931
|
-
for (const object of rawZipObjects) {
|
|
1932
|
-
objectsByPath.set(object.path, object);
|
|
1933
|
-
}
|
|
1934
|
-
for (const object of collectGeneratedResultObjects(result)) {
|
|
1935
|
-
if (!objectsByPath.has(object.path)) {
|
|
1936
|
-
objectsByPath.set(object.path, object);
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
return [...objectsByPath.values()].filter((object) => !isSdkMetadataPath(object.path));
|
|
1940
|
-
}
|
|
1941
|
-
async function collectRawZipObjects(rawZip) {
|
|
1942
|
-
try {
|
|
1943
|
-
const zip = await import_jszip3.default.loadAsync(rawZip);
|
|
1944
|
-
const objects = [];
|
|
1945
|
-
for (const entry of Object.values(zip.files)) {
|
|
1946
|
-
if (entry.dir || entry.name === "result.zip") {
|
|
1947
|
-
continue;
|
|
1948
|
-
}
|
|
1949
|
-
objects.push({
|
|
1950
|
-
path: normalizeResultPath(entry.name),
|
|
1951
|
-
body: new Uint8Array(await entry.async("nodebuffer")),
|
|
1952
|
-
contentType: inferContentTypeFromPath(entry.name)
|
|
1953
|
-
});
|
|
1954
|
-
}
|
|
1955
|
-
return objects;
|
|
1956
|
-
} catch {
|
|
1957
|
-
return [];
|
|
1958
|
-
}
|
|
1959
|
-
}
|
|
1960
|
-
function collectGeneratedResultObjects(result) {
|
|
1961
|
-
const objects = [
|
|
1962
|
-
createJsonObject("manifest.json", result.manifest),
|
|
1963
|
-
createJsonObject("chunks.json", keysToSnake(serializeChunks2(result.chunks)))
|
|
1964
|
-
];
|
|
1965
|
-
if (result.docNav) {
|
|
1966
|
-
objects.push(createJsonObject("doc_nav.json", result.docNav));
|
|
1967
|
-
}
|
|
1968
|
-
if (result.fullMarkdown) {
|
|
1969
|
-
objects.push(createTextObject("full.md", result.fullMarkdown, markdownContentType));
|
|
1970
|
-
}
|
|
1971
|
-
if (result.chunksSlim) {
|
|
1972
|
-
objects.push(createJsonObject("chunks_slim.json", { chunks: result.chunksSlim }));
|
|
1973
|
-
}
|
|
1974
|
-
if (result.hierarchy) {
|
|
1975
|
-
objects.push(createJsonObject("hierarchy.json", result.hierarchy));
|
|
1976
|
-
}
|
|
1977
|
-
if (result.tocHierarchies) {
|
|
1978
|
-
objects.push(createJsonObject("toc_hierarchies.json", result.tocHierarchies));
|
|
1979
|
-
}
|
|
1980
|
-
if (result.kbCsv) {
|
|
1981
|
-
objects.push(createTextObject("kb.csv", result.kbCsv, csvContentType));
|
|
1982
|
-
}
|
|
1983
|
-
if (result.hierarchyViewHtml) {
|
|
1984
|
-
objects.push(
|
|
1985
|
-
createTextObject("hierarchy_view.html", result.hierarchyViewHtml, htmlContentType)
|
|
1986
|
-
);
|
|
1987
|
-
}
|
|
1988
|
-
for (const chunk of result.imageChunks) {
|
|
1989
|
-
objects.push({
|
|
1990
|
-
path: normalizeResultPath(chunk.filePath),
|
|
1991
|
-
body: new Uint8Array(chunk.data),
|
|
1992
|
-
contentType: inferContentTypeFromPath(chunk.filePath),
|
|
1993
|
-
metadata: {
|
|
1994
|
-
chunkId: chunk.chunkId,
|
|
1995
|
-
chunkType: chunk.type,
|
|
1996
|
-
sourcePath: chunk.filePath
|
|
1997
|
-
}
|
|
1998
|
-
});
|
|
1999
|
-
}
|
|
2000
|
-
for (const chunk of result.tableChunks) {
|
|
2001
|
-
objects.push({
|
|
2002
|
-
path: normalizeResultPath(chunk.filePath),
|
|
2003
|
-
body: Buffer.from(chunk.html, "utf8"),
|
|
2004
|
-
contentType: htmlContentType,
|
|
2005
|
-
metadata: {
|
|
2006
|
-
chunkId: chunk.chunkId,
|
|
2007
|
-
chunkType: chunk.type,
|
|
2008
|
-
sourcePath: chunk.filePath
|
|
2009
|
-
}
|
|
2010
|
-
});
|
|
2011
|
-
}
|
|
2012
|
-
return objects;
|
|
2013
|
-
}
|
|
2014
|
-
function serializeChunks2(chunks) {
|
|
2015
|
-
return {
|
|
2016
|
-
chunks: chunks.map((chunk) => {
|
|
2017
|
-
const rawChunk = {
|
|
2018
|
-
chunkId: chunk.chunkId,
|
|
2019
|
-
type: chunk.type,
|
|
2020
|
-
contentSource: chunk.contentSource,
|
|
2021
|
-
content: chunk.content,
|
|
2022
|
-
path: chunk.path,
|
|
2023
|
-
metadata: chunk.metadata
|
|
2024
|
-
};
|
|
2025
|
-
if (chunk.type === "image" || chunk.type === "table") {
|
|
2026
|
-
return {
|
|
2027
|
-
...rawChunk,
|
|
2028
|
-
filePath: chunk.filePath,
|
|
2029
|
-
assetUrl: chunk.assetUrl
|
|
2030
|
-
};
|
|
2031
|
-
}
|
|
2032
|
-
return rawChunk;
|
|
2033
|
-
})
|
|
2034
|
-
};
|
|
2035
|
-
}
|
|
2036
|
-
async function writeParsedDocumentJsonObject(storage, params) {
|
|
2037
|
-
await storage.writeObject({
|
|
2038
|
-
documentId: params.documentId,
|
|
2039
|
-
revisionKey: params.revisionKey,
|
|
2040
|
-
path: params.path,
|
|
2041
|
-
body: Buffer.from(JSON.stringify(params.value, null, 2), "utf8"),
|
|
2042
|
-
contentType: jsonContentType
|
|
2043
|
-
});
|
|
2044
|
-
}
|
|
2045
|
-
function createJsonObject(path4, value) {
|
|
2046
|
-
return {
|
|
2047
|
-
path: path4,
|
|
2048
|
-
body: Buffer.from(JSON.stringify(value, null, 2), "utf8"),
|
|
2049
|
-
contentType: jsonContentType
|
|
2050
|
-
};
|
|
2051
|
-
}
|
|
2052
|
-
function createTextObject(path4, value, contentType) {
|
|
2053
|
-
return {
|
|
2054
|
-
path: path4,
|
|
2055
|
-
body: Buffer.from(value, "utf8"),
|
|
2056
|
-
contentType
|
|
2057
|
-
};
|
|
2058
|
-
}
|
|
2059
|
-
function inferContentTypeFromPath(filePath) {
|
|
2060
|
-
const lower = filePath.toLowerCase();
|
|
2061
|
-
if (lower.endsWith(".json")) return jsonContentType;
|
|
2062
|
-
if (lower.endsWith(".md")) return markdownContentType;
|
|
2063
|
-
if (lower.endsWith(".csv")) return csvContentType;
|
|
2064
|
-
if (lower.endsWith(".html") || lower.endsWith(".htm")) return htmlContentType;
|
|
2065
|
-
if (lower.endsWith(".png")) return "image/png";
|
|
2066
|
-
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
|
2067
|
-
if (lower.endsWith(".gif")) return "image/gif";
|
|
2068
|
-
if (lower.endsWith(".webp")) return "image/webp";
|
|
2069
|
-
if (lower.endsWith(".svg")) return "image/svg+xml";
|
|
2070
|
-
return binaryContentType;
|
|
2071
|
-
}
|
|
2072
|
-
function normalizeResultPath(value) {
|
|
2073
|
-
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
2074
|
-
if (normalized.length === 0 || normalized.includes("\0") || normalized.startsWith("/") || normalized.split("/").some((part) => part.length === 0 || part === "." || part === "..")) {
|
|
2075
|
-
throw new Error(`Invalid parsed result path: ${value}`);
|
|
2076
|
-
}
|
|
2077
|
-
return normalized;
|
|
2078
|
-
}
|
|
2079
|
-
function isSdkMetadataPath(path4) {
|
|
2080
|
-
return path4 === ".knowhere-sdk" || path4.startsWith(".knowhere-sdk/");
|
|
2081
|
-
}
|
|
3092
|
+
// src/knowledge/knowledge.ts
|
|
3093
|
+
init_parsed_document_storage();
|
|
3094
|
+
init_utils();
|
|
3095
|
+
init_streamed_parsed_document_sync();
|
|
2082
3096
|
|
|
2083
3097
|
// src/knowledge/local-store.ts
|
|
2084
|
-
var
|
|
2085
|
-
var
|
|
3098
|
+
var import_crypto3 = require("crypto");
|
|
3099
|
+
var import_os2 = __toESM(require("os"));
|
|
2086
3100
|
var import_fs3 = require("fs");
|
|
2087
|
-
var
|
|
3101
|
+
var import_path3 = __toESM(require("path"));
|
|
2088
3102
|
var STORE_VERSION = 1;
|
|
2089
3103
|
var LOCAL_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
2090
3104
|
var LocalKnowledgeStore = class {
|
|
@@ -2092,8 +3106,8 @@ var LocalKnowledgeStore = class {
|
|
|
2092
3106
|
indexPath;
|
|
2093
3107
|
resultCache = /* @__PURE__ */ new Map();
|
|
2094
3108
|
constructor(cacheDirectory) {
|
|
2095
|
-
this.cacheDirectory = cacheDirectory ??
|
|
2096
|
-
this.indexPath =
|
|
3109
|
+
this.cacheDirectory = cacheDirectory ?? import_path3.default.join(import_os2.default.homedir(), ".knowhere-node-sdk", "knowledge");
|
|
3110
|
+
this.indexPath = import_path3.default.join(this.cacheDirectory, "index.json");
|
|
2097
3111
|
}
|
|
2098
3112
|
async saveResult(result, options) {
|
|
2099
3113
|
await import_fs3.promises.mkdir(this.cacheDirectory, { recursive: true });
|
|
@@ -2218,8 +3232,8 @@ var LocalKnowledgeStore = class {
|
|
|
2218
3232
|
return { document, result };
|
|
2219
3233
|
}
|
|
2220
3234
|
getResultDirectoryPath(localDocumentId) {
|
|
2221
|
-
const documentsDirectory =
|
|
2222
|
-
const resultDirectoryPath =
|
|
3235
|
+
const documentsDirectory = import_path3.default.resolve(this.cacheDirectory, "documents");
|
|
3236
|
+
const resultDirectoryPath = import_path3.default.resolve(documentsDirectory, localDocumentId);
|
|
2223
3237
|
if (!isPathInsideDirectory(resultDirectoryPath, documentsDirectory)) {
|
|
2224
3238
|
throw new Error(`Local Knowhere document ID resolves outside the cache: ${localDocumentId}`);
|
|
2225
3239
|
}
|
|
@@ -2253,7 +3267,7 @@ var LocalKnowledgeStore = class {
|
|
|
2253
3267
|
}
|
|
2254
3268
|
};
|
|
2255
3269
|
function validateLocalDocumentId(localDocumentId) {
|
|
2256
|
-
if (!LOCAL_DOCUMENT_ID_PATTERN.test(localDocumentId) || localDocumentId.includes("..") ||
|
|
3270
|
+
if (!LOCAL_DOCUMENT_ID_PATTERN.test(localDocumentId) || localDocumentId.includes("..") || import_path3.default.basename(localDocumentId) !== localDocumentId) {
|
|
2257
3271
|
throw new Error(
|
|
2258
3272
|
"Local Knowhere document ID must be a safe slug containing only letters, numbers, dots, underscores, or hyphens"
|
|
2259
3273
|
);
|
|
@@ -2261,11 +3275,11 @@ function validateLocalDocumentId(localDocumentId) {
|
|
|
2261
3275
|
return localDocumentId;
|
|
2262
3276
|
}
|
|
2263
3277
|
function isPathInsideDirectory(targetPath, parentDirectory) {
|
|
2264
|
-
const relativePath =
|
|
2265
|
-
return relativePath.length === 0 || !relativePath.startsWith("..") && !
|
|
3278
|
+
const relativePath = import_path3.default.relative(parentDirectory, targetPath);
|
|
3279
|
+
return relativePath.length === 0 || !relativePath.startsWith("..") && !import_path3.default.isAbsolute(relativePath);
|
|
2266
3280
|
}
|
|
2267
3281
|
function createLocalDocumentId(result) {
|
|
2268
|
-
const hash = (0,
|
|
3282
|
+
const hash = (0, import_crypto3.createHash)("sha256").update(result.jobId).update("\0").update(result.manifest.sourceFileName).digest("hex").slice(0, 16);
|
|
2269
3283
|
return `local_${hash}`;
|
|
2270
3284
|
}
|
|
2271
3285
|
function countChunkTypes(result) {
|
|
@@ -2419,29 +3433,35 @@ var Knowledge = class _Knowledge {
|
|
|
2419
3433
|
return this.importJobResult(params);
|
|
2420
3434
|
}
|
|
2421
3435
|
async syncParsedDocument(params) {
|
|
3436
|
+
params.signal?.throwIfAborted();
|
|
2422
3437
|
const normalized = normalizeDocumentReference(params);
|
|
2423
3438
|
const parsedStorageConfig = this.requireParsedStorageConfig();
|
|
3439
|
+
validateArchiveSyncLimits(parsedStorageConfig.limits);
|
|
2424
3440
|
if (normalized.documentId) {
|
|
2425
3441
|
return this.syncRemoteDocumentToParsedStorage({
|
|
2426
3442
|
documentId: normalized.documentId,
|
|
2427
3443
|
revisionKey: params.revisionKey,
|
|
2428
|
-
parsedStorageConfig
|
|
3444
|
+
parsedStorageConfig,
|
|
3445
|
+
signal: params.signal,
|
|
3446
|
+
retainResultArchive: params.retainResultArchive,
|
|
3447
|
+
cacheResultArchive: params.cacheResultArchive
|
|
2429
3448
|
});
|
|
2430
3449
|
}
|
|
2431
3450
|
if (normalized.jobId) {
|
|
2432
|
-
const
|
|
2433
|
-
|
|
2434
|
-
result
|
|
3451
|
+
const job = await this.client.jobs.get(normalized.jobId);
|
|
3452
|
+
if (!job.isDone || !job.resultUrl)
|
|
3453
|
+
throw new ValidationError("Completed job result archive is unavailable");
|
|
3454
|
+
return syncRemoteResultArchive({
|
|
3455
|
+
documentId: job.documentId ?? normalized.localDocumentId ?? normalized.jobId,
|
|
3456
|
+
revisionKey: params.revisionKey ?? normalized.jobId,
|
|
3457
|
+
jobId: normalized.jobId,
|
|
3458
|
+
resultUrl: job.resultUrl,
|
|
2435
3459
|
storage: parsedStorageConfig.storage,
|
|
2436
|
-
|
|
2437
|
-
|
|
3460
|
+
limits: parsedStorageConfig.limits,
|
|
3461
|
+
signal: params.signal,
|
|
3462
|
+
retainResultArchive: params.retainResultArchive,
|
|
3463
|
+
cacheResultArchive: params.cacheResultArchive
|
|
2438
3464
|
});
|
|
2439
|
-
return {
|
|
2440
|
-
documentId: synced2.documentId,
|
|
2441
|
-
revisionKey: synced2.revisionKey,
|
|
2442
|
-
completed: true,
|
|
2443
|
-
commit: synced2.commit
|
|
2444
|
-
};
|
|
2445
3465
|
}
|
|
2446
3466
|
if (!normalized.localDocumentId) {
|
|
2447
3467
|
throw new ValidationError("localDocumentId, documentId, or jobId is required");
|
|
@@ -3139,10 +4159,14 @@ var Knowledge = class _Knowledge {
|
|
|
3139
4159
|
parsedStorageConfig
|
|
3140
4160
|
});
|
|
3141
4161
|
} catch (error) {
|
|
4162
|
+
const previous = await parsedStorageConfig.storage.readSyncProgress({
|
|
4163
|
+
documentId,
|
|
4164
|
+
revisionKey
|
|
4165
|
+
});
|
|
3142
4166
|
await parsedStorageConfig.storage.writeSyncProgress({
|
|
4167
|
+
...previous,
|
|
3143
4168
|
documentId,
|
|
3144
4169
|
revisionKey,
|
|
3145
|
-
nextChunkPage: 1,
|
|
3146
4170
|
status: "failed",
|
|
3147
4171
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3148
4172
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -3152,6 +4176,7 @@ var Knowledge = class _Knowledge {
|
|
|
3152
4176
|
void Promise.resolve(scheduler.schedule(task)).catch(() => void 0);
|
|
3153
4177
|
}
|
|
3154
4178
|
async syncRemoteDocumentToParsedStorage(params) {
|
|
4179
|
+
params.signal?.throwIfAborted();
|
|
3155
4180
|
const firstPage = await this.client.documents.listChunks(params.documentId, {
|
|
3156
4181
|
page: 1,
|
|
3157
4182
|
pageSize: 1,
|
|
@@ -3159,55 +4184,68 @@ var Knowledge = class _Knowledge {
|
|
|
3159
4184
|
});
|
|
3160
4185
|
const revisionKey = params.revisionKey ?? getRemoteRevisionKey(firstPage);
|
|
3161
4186
|
if (params.revisionKey && getRemoteRevisionKey(firstPage) !== params.revisionKey) {
|
|
3162
|
-
|
|
3163
|
-
documentId: params.documentId,
|
|
3164
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
3165
|
-
});
|
|
4187
|
+
throw new ValidationError("Published document revision changed before parsed snapshot sync");
|
|
3166
4188
|
}
|
|
3167
4189
|
if (firstPage.jobId) {
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
});
|
|
3177
|
-
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
4190
|
+
const job = await this.client.jobs.get(firstPage.jobId).catch((error) => {
|
|
4191
|
+
if (error instanceof NotFoundError) return null;
|
|
4192
|
+
throw error;
|
|
4193
|
+
});
|
|
4194
|
+
if (job?.isDone && job.resultUrl) {
|
|
4195
|
+
if (job.documentId && job.documentId !== params.documentId)
|
|
4196
|
+
throw new ValidationError("Parsed job document identity differs from requested document");
|
|
4197
|
+
return syncRemoteResultArchive({
|
|
3178
4198
|
documentId: params.documentId,
|
|
3179
4199
|
revisionKey,
|
|
3180
|
-
|
|
3181
|
-
|
|
4200
|
+
jobId: firstPage.jobId,
|
|
4201
|
+
resultUrl: job.resultUrl,
|
|
4202
|
+
storage: params.parsedStorageConfig.storage,
|
|
4203
|
+
limits: params.parsedStorageConfig.limits,
|
|
4204
|
+
signal: params.signal,
|
|
4205
|
+
retainResultArchive: params.retainResultArchive,
|
|
4206
|
+
cacheResultArchive: params.cacheResultArchive,
|
|
4207
|
+
verifyRevision: async () => {
|
|
4208
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
4209
|
+
page: 1,
|
|
4210
|
+
pageSize: 1,
|
|
4211
|
+
includeAssetUrls: true
|
|
4212
|
+
});
|
|
4213
|
+
if (getRemoteRevisionKey(current) !== revisionKey || current.jobId !== firstPage.jobId)
|
|
4214
|
+
throw new ValidationError(
|
|
4215
|
+
"Published document revision changed during parsed snapshot sync"
|
|
4216
|
+
);
|
|
4217
|
+
}
|
|
3182
4218
|
});
|
|
3183
|
-
return {
|
|
3184
|
-
documentId: synced.documentId,
|
|
3185
|
-
revisionKey: synced.revisionKey,
|
|
3186
|
-
completed: true,
|
|
3187
|
-
commit: synced.commit
|
|
3188
|
-
};
|
|
3189
|
-
} catch {
|
|
3190
4219
|
}
|
|
3191
4220
|
}
|
|
3192
4221
|
return this.syncRemoteDocumentChunksToParsedStorage({
|
|
3193
4222
|
documentId: params.documentId,
|
|
3194
4223
|
revisionKey,
|
|
3195
4224
|
firstPage,
|
|
3196
|
-
parsedStorageConfig: params.parsedStorageConfig
|
|
4225
|
+
parsedStorageConfig: params.parsedStorageConfig,
|
|
4226
|
+
signal: params.signal
|
|
3197
4227
|
});
|
|
3198
4228
|
}
|
|
3199
4229
|
async syncRemoteDocumentChunksToParsedStorage(params) {
|
|
4230
|
+
params.signal?.throwIfAborted();
|
|
3200
4231
|
const limits = getParsedStorageLimits(params.parsedStorageConfig.limits);
|
|
3201
4232
|
const existingProgress = await params.parsedStorageConfig.storage.readSyncProgress({
|
|
3202
4233
|
documentId: params.documentId,
|
|
3203
4234
|
revisionKey: params.revisionKey
|
|
3204
4235
|
});
|
|
3205
|
-
let nextPage = existingProgress?.status === "running" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
4236
|
+
let nextPage = existingProgress?.status === "running" || existingProgress?.status === "failed" ? existingProgress.nextChunkPage ?? 1 : 1;
|
|
3206
4237
|
let response = nextPage === 1 && params.firstPage.pagination.pageSize === limits.remotePageSize ? params.firstPage : await this.client.documents.listChunks(params.documentId, {
|
|
3207
4238
|
page: nextPage,
|
|
3208
4239
|
pageSize: limits.remotePageSize,
|
|
3209
4240
|
includeAssetUrls: true
|
|
3210
4241
|
});
|
|
4242
|
+
validateRemoteReconstructionRevision(
|
|
4243
|
+
params.documentId,
|
|
4244
|
+
params.revisionKey,
|
|
4245
|
+
params.firstPage.jobId,
|
|
4246
|
+
response
|
|
4247
|
+
);
|
|
4248
|
+
params.signal?.throwIfAborted();
|
|
3211
4249
|
const totalPages = response.pagination.totalPages;
|
|
3212
4250
|
let syncedPages = 0;
|
|
3213
4251
|
const startedAt = Date.now();
|
|
@@ -3219,6 +4257,13 @@ var Knowledge = class _Knowledge {
|
|
|
3219
4257
|
includeAssetUrls: true
|
|
3220
4258
|
});
|
|
3221
4259
|
}
|
|
4260
|
+
params.signal?.throwIfAborted();
|
|
4261
|
+
validateRemoteReconstructionRevision(
|
|
4262
|
+
params.documentId,
|
|
4263
|
+
params.revisionKey,
|
|
4264
|
+
params.firstPage.jobId,
|
|
4265
|
+
response
|
|
4266
|
+
);
|
|
3222
4267
|
await params.parsedStorageConfig.storage.writeObject({
|
|
3223
4268
|
documentId: params.documentId,
|
|
3224
4269
|
revisionKey: params.revisionKey,
|
|
@@ -3266,7 +4311,23 @@ var Knowledge = class _Knowledge {
|
|
|
3266
4311
|
storage: params.parsedStorageConfig.storage,
|
|
3267
4312
|
documentId: params.documentId,
|
|
3268
4313
|
revisionKey: params.revisionKey,
|
|
3269
|
-
source: "remoteReconstruction"
|
|
4314
|
+
source: "remoteReconstruction",
|
|
4315
|
+
signal: params.signal,
|
|
4316
|
+
beforeCommit: async () => {
|
|
4317
|
+
params.signal?.throwIfAborted();
|
|
4318
|
+
const current = await this.client.documents.listChunks(params.documentId, {
|
|
4319
|
+
page: 1,
|
|
4320
|
+
pageSize: 1,
|
|
4321
|
+
includeAssetUrls: true
|
|
4322
|
+
});
|
|
4323
|
+
params.signal?.throwIfAborted();
|
|
4324
|
+
validateRemoteReconstructionRevision(
|
|
4325
|
+
params.documentId,
|
|
4326
|
+
params.revisionKey,
|
|
4327
|
+
params.firstPage.jobId,
|
|
4328
|
+
current
|
|
4329
|
+
);
|
|
4330
|
+
}
|
|
3270
4331
|
});
|
|
3271
4332
|
await params.parsedStorageConfig.storage.writeSyncProgress({
|
|
3272
4333
|
documentId: params.documentId,
|
|
@@ -3338,7 +4399,7 @@ function createLocalDocumentIdForRemote(documentId) {
|
|
|
3338
4399
|
if (SAFE_LOCAL_DOCUMENT_ID_PATTERN.test(documentId) && !documentId.includes("..") && !documentId.includes("/") && !documentId.includes("\\")) {
|
|
3339
4400
|
return documentId;
|
|
3340
4401
|
}
|
|
3341
|
-
const hash = (0,
|
|
4402
|
+
const hash = (0, import_crypto4.createHash)("sha256").update(documentId).digest("hex").slice(0, 16);
|
|
3342
4403
|
return `remote_${hash}`;
|
|
3343
4404
|
}
|
|
3344
4405
|
var defaultParsedStorageScheduler = {
|
|
@@ -3558,10 +4619,10 @@ function parseStoredResultChunks(params) {
|
|
|
3558
4619
|
}
|
|
3559
4620
|
function extractStoredChunkRecords(payload) {
|
|
3560
4621
|
if (Array.isArray(payload)) {
|
|
3561
|
-
return payload.filter(
|
|
4622
|
+
return payload.filter(isRecord3);
|
|
3562
4623
|
}
|
|
3563
|
-
if (
|
|
3564
|
-
return payload.chunks.filter(
|
|
4624
|
+
if (isRecord3(payload) && Array.isArray(payload.chunks)) {
|
|
4625
|
+
return payload.chunks.filter(isRecord3);
|
|
3565
4626
|
}
|
|
3566
4627
|
return [];
|
|
3567
4628
|
}
|
|
@@ -3581,7 +4642,7 @@ function readRecordNumber(record, ...keys) {
|
|
|
3581
4642
|
}
|
|
3582
4643
|
function readRecordMetadata(record) {
|
|
3583
4644
|
const metadata = record.metadata;
|
|
3584
|
-
return
|
|
4645
|
+
return isRecord3(metadata) ? keysToCamel(metadata) : {};
|
|
3585
4646
|
}
|
|
3586
4647
|
function isKnowledgeChunkType(value) {
|
|
3587
4648
|
return value === "text" || value === "image" || value === "table" || value === "page";
|
|
@@ -3680,7 +4741,7 @@ async function applyStoredPageAssetUrls(params) {
|
|
|
3680
4741
|
}
|
|
3681
4742
|
return Promise.all(
|
|
3682
4743
|
pageAssets.map(async (pageAsset) => {
|
|
3683
|
-
if (!
|
|
4744
|
+
if (!isRecord3(pageAsset)) {
|
|
3684
4745
|
return pageAsset;
|
|
3685
4746
|
}
|
|
3686
4747
|
const artifactRef = readRecordString(pageAsset, "artifactRef", "artifact_ref");
|
|
@@ -3709,7 +4770,7 @@ function getStoredPageAssets(metadata) {
|
|
|
3709
4770
|
return pageAssets?.map(normalizeStoredPageAsset);
|
|
3710
4771
|
}
|
|
3711
4772
|
function normalizeStoredPageAsset(pageAsset) {
|
|
3712
|
-
if (!
|
|
4773
|
+
if (!isRecord3(pageAsset)) {
|
|
3713
4774
|
return pageAsset;
|
|
3714
4775
|
}
|
|
3715
4776
|
const normalizedPageAsset = { ...pageAsset };
|
|
@@ -3787,7 +4848,7 @@ function parseContinuationCursor(value) {
|
|
|
3787
4848
|
}
|
|
3788
4849
|
try {
|
|
3789
4850
|
const parsed = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
3790
|
-
if (!
|
|
4851
|
+
if (!isRecord3(parsed)) {
|
|
3791
4852
|
return null;
|
|
3792
4853
|
}
|
|
3793
4854
|
const documentId = parsed.documentId;
|
|
@@ -3825,11 +4886,11 @@ function inferContentTypeFromPath2(filePath) {
|
|
|
3825
4886
|
if (extension === "html" || extension === "htm") return "text/html; charset=utf-8";
|
|
3826
4887
|
return "application/octet-stream";
|
|
3827
4888
|
}
|
|
3828
|
-
function
|
|
4889
|
+
function isRecord3(value) {
|
|
3829
4890
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3830
4891
|
}
|
|
3831
4892
|
function isDocumentChunkArray(value) {
|
|
3832
|
-
return Array.isArray(value) && value.every(
|
|
4893
|
+
return Array.isArray(value) && value.every(isRecord3);
|
|
3833
4894
|
}
|
|
3834
4895
|
function indexChunks(result) {
|
|
3835
4896
|
return result.chunks.map((chunk, index) => {
|
|
@@ -4132,17 +5193,25 @@ function toRemoteSearchResult(result, documentByServerId) {
|
|
|
4132
5193
|
sourceFileName: result.source.sourceFileName ?? void 0
|
|
4133
5194
|
};
|
|
4134
5195
|
}
|
|
5196
|
+
function validateRemoteReconstructionRevision(documentId, revisionKey, jobId, response) {
|
|
5197
|
+
if (response.documentId !== documentId || getRemoteRevisionKey(response) !== revisionKey || (response.jobId ?? void 0) !== (jobId ?? void 0)) {
|
|
5198
|
+
throw new ValidationError("Published document revision changed during remote reconstruction");
|
|
5199
|
+
}
|
|
5200
|
+
}
|
|
4135
5201
|
|
|
4136
5202
|
// src/client.ts
|
|
5203
|
+
init_constants();
|
|
5204
|
+
init_errors();
|
|
5205
|
+
init_utils();
|
|
4137
5206
|
function inferFileName(file, explicitFileName) {
|
|
4138
5207
|
if (explicitFileName) {
|
|
4139
5208
|
return explicitFileName;
|
|
4140
5209
|
}
|
|
4141
5210
|
if (typeof file === "string") {
|
|
4142
|
-
return
|
|
5211
|
+
return import_path4.default.basename(file);
|
|
4143
5212
|
}
|
|
4144
5213
|
if (isReadStream2(file) && typeof file.path === "string") {
|
|
4145
|
-
return
|
|
5214
|
+
return import_path4.default.basename(file.path);
|
|
4146
5215
|
}
|
|
4147
5216
|
return void 0;
|
|
4148
5217
|
}
|
|
@@ -4288,9 +5357,9 @@ var Knowhere = class {
|
|
|
4288
5357
|
};
|
|
4289
5358
|
|
|
4290
5359
|
// src/storage/disk-parsed-document-storage.ts
|
|
4291
|
-
var
|
|
5360
|
+
var import_crypto5 = require("crypto");
|
|
4292
5361
|
var import_fs4 = require("fs");
|
|
4293
|
-
var
|
|
5362
|
+
var import_path5 = __toESM(require("path"));
|
|
4294
5363
|
var syncProgressPath = ".knowhere-sdk/sync-progress.json";
|
|
4295
5364
|
var DiskParsedDocumentStorage = class {
|
|
4296
5365
|
rootDirectory;
|
|
@@ -4319,7 +5388,7 @@ var DiskParsedDocumentStorage = class {
|
|
|
4319
5388
|
}
|
|
4320
5389
|
async writeObject(params) {
|
|
4321
5390
|
const objectPath = this.getObjectPath(params);
|
|
4322
|
-
await import_fs4.promises.mkdir(
|
|
5391
|
+
await import_fs4.promises.mkdir(import_path5.default.dirname(objectPath), { recursive: true });
|
|
4323
5392
|
await import_fs4.promises.writeFile(objectPath, params.body);
|
|
4324
5393
|
await this.writeObjectMetadata(objectPath, {
|
|
4325
5394
|
contentType: params.contentType,
|
|
@@ -4378,14 +5447,14 @@ var DiskParsedDocumentStorage = class {
|
|
|
4378
5447
|
}
|
|
4379
5448
|
getObjectPath(params) {
|
|
4380
5449
|
const revisionDirectory = this.getRevisionDirectory(params.documentId, params.revisionKey);
|
|
4381
|
-
const objectPath =
|
|
5450
|
+
const objectPath = import_path5.default.resolve(revisionDirectory, normalizeRelativeStoragePath(params.path));
|
|
4382
5451
|
if (!isPathInsideDirectory2(objectPath, revisionDirectory)) {
|
|
4383
5452
|
throw new Error(`Parsed document object path resolves outside storage: ${params.path}`);
|
|
4384
5453
|
}
|
|
4385
5454
|
return objectPath;
|
|
4386
5455
|
}
|
|
4387
5456
|
getRevisionDirectory(documentId, revisionKey) {
|
|
4388
|
-
return
|
|
5457
|
+
return import_path5.default.join(this.rootDirectory, hashPathPart(documentId), hashPathPart(revisionKey));
|
|
4389
5458
|
}
|
|
4390
5459
|
async readOptionalJson(params) {
|
|
4391
5460
|
const object = await this.readObject(params);
|
|
@@ -4421,7 +5490,7 @@ function hashPathPart(value) {
|
|
|
4421
5490
|
if (normalized.length === 0) {
|
|
4422
5491
|
throw new Error("Parsed document storage path part must be a non-empty value.");
|
|
4423
5492
|
}
|
|
4424
|
-
return (0,
|
|
5493
|
+
return (0, import_crypto5.createHash)("sha256").update(normalized).digest("hex").slice(0, 24);
|
|
4425
5494
|
}
|
|
4426
5495
|
function normalizeRelativeStoragePath(value) {
|
|
4427
5496
|
const normalized = value.replaceAll("\\", "/").replace(/^\.\/+/, "");
|
|
@@ -4431,8 +5500,8 @@ function normalizeRelativeStoragePath(value) {
|
|
|
4431
5500
|
return normalized;
|
|
4432
5501
|
}
|
|
4433
5502
|
function isPathInsideDirectory2(filePath, directory) {
|
|
4434
|
-
const relative =
|
|
4435
|
-
return relative === "" || !relative.startsWith("..") && !
|
|
5503
|
+
const relative = import_path5.default.relative(directory, filePath);
|
|
5504
|
+
return relative === "" || !relative.startsWith("..") && !import_path5.default.isAbsolute(relative);
|
|
4436
5505
|
}
|
|
4437
5506
|
function toFileUrl(filePath) {
|
|
4438
5507
|
return `file://${filePath}`;
|
|
@@ -4440,6 +5509,10 @@ function toFileUrl(filePath) {
|
|
|
4440
5509
|
function isMissingFileError3(error) {
|
|
4441
5510
|
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
4442
5511
|
}
|
|
5512
|
+
|
|
5513
|
+
// src/index.ts
|
|
5514
|
+
init_parsed_document_storage();
|
|
5515
|
+
init_errors();
|
|
4443
5516
|
// Annotate the CommonJS export names for ESM import in node:
|
|
4444
5517
|
0 && (module.exports = {
|
|
4445
5518
|
APIError,
|
|
@@ -4461,6 +5534,7 @@ function isMissingFileError3(error) {
|
|
|
4461
5534
|
NODE_SDK_DOCUMENT_METADATA_DEFAULTS,
|
|
4462
5535
|
NetworkError,
|
|
4463
5536
|
NotFoundError,
|
|
5537
|
+
ParsedDocumentArchiveError,
|
|
4464
5538
|
PaymentRequiredError,
|
|
4465
5539
|
PermissionDeniedError,
|
|
4466
5540
|
PollingTimeoutError,
|