@ontos-ai/knowhere-sdk 2.6.1-beta.20261027152313 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DIONXJWK.mjs → chunk-OCR7IDYL.mjs} +128 -101
- package/dist/index.d.mts +48 -2
- package/dist/index.d.ts +48 -2
- package/dist/index.js +352 -124
- package/dist/index.mjs +188 -15
- package/dist/{sequential-parsed-document-archive-W7HPYIKA.mjs → sequential-parsed-document-archive-AIRFKO52.mjs} +27 -2
- package/package.json +1 -1
|
@@ -175,9 +175,7 @@ var ParsedDocumentArchiveError = class extends KnowhereError {
|
|
|
175
175
|
code = "ARCHIVE_SPOOL_FAILED";
|
|
176
176
|
cause;
|
|
177
177
|
constructor(cause) {
|
|
178
|
-
super(
|
|
179
|
-
"Parsed result ZIP temporary storage is unavailable. Range-enabled object storage avoids temporary disk usage."
|
|
180
|
-
);
|
|
178
|
+
super("Parsed result ZIP temporary storage is unavailable.");
|
|
181
179
|
this.name = "ParsedDocumentArchiveError";
|
|
182
180
|
this.cause = cause;
|
|
183
181
|
}
|
|
@@ -770,17 +768,15 @@ function readObjectVersion(object) {
|
|
|
770
768
|
return { entityTag: object.entityTag, contentLength: object.contentLength };
|
|
771
769
|
}
|
|
772
770
|
function validateCachedResponse(response, marker, signal) {
|
|
773
|
-
if (response.
|
|
771
|
+
if (response.headers.get("etag") !== marker.storageEntityTag)
|
|
774
772
|
rejectResponse(
|
|
775
773
|
response,
|
|
776
|
-
new ArchiveCacheTransportError(
|
|
777
|
-
`Parsed result archive cache requires a ranged response (HTTP ${response.status}).`
|
|
778
|
-
)
|
|
774
|
+
new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
|
|
779
775
|
);
|
|
780
|
-
if (response.
|
|
776
|
+
if (response.status !== 206 || !response.body)
|
|
781
777
|
rejectResponse(
|
|
782
778
|
response,
|
|
783
|
-
new ArchiveCacheTransportError("Parsed result archive cache
|
|
779
|
+
new ArchiveCacheTransportError("Parsed result archive cache requires a ranged response.")
|
|
784
780
|
);
|
|
785
781
|
const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
|
|
786
782
|
response.headers.get("content-range") ?? ""
|
|
@@ -847,10 +843,7 @@ function isPositiveInteger(value) {
|
|
|
847
843
|
}
|
|
848
844
|
|
|
849
845
|
// src/storage/streamed-parsed-document-sync.ts
|
|
850
|
-
import {
|
|
851
|
-
HttpRangeReader,
|
|
852
|
-
ZipReader
|
|
853
|
-
} from "@zip.js/zip.js";
|
|
846
|
+
import { HttpRangeReader, ZipReader } from "@zip.js/zip.js";
|
|
854
847
|
async function syncRemoteResultArchive(params) {
|
|
855
848
|
params.signal?.throwIfAborted();
|
|
856
849
|
validateArchiveSyncLimits(params.limits);
|
|
@@ -861,14 +854,10 @@ async function syncRemoteResultArchive(params) {
|
|
|
861
854
|
const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
|
|
862
855
|
const controller = new AbortController();
|
|
863
856
|
const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
|
|
864
|
-
const
|
|
857
|
+
const transferParams = { ...params, signal };
|
|
865
858
|
const previous = await params.storage.readSyncProgress(params);
|
|
866
|
-
const openedSource = await openArchive(
|
|
859
|
+
const openedSource = await openArchive(transferParams, previous);
|
|
867
860
|
const source = !openedSource.canSkipEntries && previous?.archiveIdentity && previous.archiveDownloadIdentity === openedSource.downloadIdentity ? { ...openedSource, identity: previous.archiveIdentity } : openedSource;
|
|
868
|
-
const transferParams = {
|
|
869
|
-
...openingParams,
|
|
870
|
-
signal: source.signal ?? signal
|
|
871
|
-
};
|
|
872
861
|
const archiveDownloadIdentity = source.isCached ? source.downloadIdentity : previous?.archiveDownloadIdentity;
|
|
873
862
|
const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
|
|
874
863
|
const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
|
|
@@ -895,6 +884,8 @@ async function syncRemoteResultArchive(params) {
|
|
|
895
884
|
documentId: params.documentId,
|
|
896
885
|
revisionKey: params.revisionKey,
|
|
897
886
|
pinnedRevisionKey: params.revisionKey,
|
|
887
|
+
snapshotChecksum: params.snapshot?.checksum,
|
|
888
|
+
upstreamJobResultId: params.snapshot?.jobResultId,
|
|
898
889
|
archiveIdentity: source.identity,
|
|
899
890
|
archiveDownloadIdentity,
|
|
900
891
|
archiveCacheBypassed,
|
|
@@ -933,10 +924,38 @@ async function syncRemoteResultArchive(params) {
|
|
|
933
924
|
signal.throwIfAborted();
|
|
934
925
|
}
|
|
935
926
|
try {
|
|
936
|
-
if (previous?.
|
|
927
|
+
if (previous?.snapshotChecksum && previous.snapshotChecksum !== params.snapshot?.checksum)
|
|
928
|
+
throw new Error("Parsed snapshot checksum changed within one generation.");
|
|
929
|
+
if (previous?.upstreamJobResultId && previous.upstreamJobResultId !== params.snapshot?.jobResultId)
|
|
930
|
+
throw new Error("Parsed snapshot upstream revision changed within one generation.");
|
|
931
|
+
if (previous?.archiveIdentity && previous.archiveIdentity !== source.identity) {
|
|
937
932
|
throw new Error("Parsed result archive identity changed; start a new revision.");
|
|
938
|
-
|
|
933
|
+
}
|
|
934
|
+
if (previous?.pinnedRevisionKey && previous.pinnedRevisionKey !== params.revisionKey) {
|
|
939
935
|
throw new Error("Parsed result revision changed while resuming archive sync.");
|
|
936
|
+
}
|
|
937
|
+
if (params.snapshot && previous?.status === "completed") {
|
|
938
|
+
const committed = await readParsedDocumentCommit(params);
|
|
939
|
+
if (committed) {
|
|
940
|
+
if (source.verifyArchiveChecksum) {
|
|
941
|
+
await source.verifyArchiveChecksum(previous.archiveEntryHashes ?? []);
|
|
942
|
+
} else {
|
|
943
|
+
await verifyArchiveChecksum(
|
|
944
|
+
params,
|
|
945
|
+
source.downloadIdentity,
|
|
946
|
+
previous.archiveEntryHashes ?? [],
|
|
947
|
+
params.retainResultArchive
|
|
948
|
+
);
|
|
949
|
+
}
|
|
950
|
+
return {
|
|
951
|
+
documentId: params.documentId,
|
|
952
|
+
revisionKey: params.revisionKey,
|
|
953
|
+
snapshot: params.snapshot,
|
|
954
|
+
completed: true,
|
|
955
|
+
commit: committed
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
}
|
|
940
959
|
for await (const entry of source.entries) {
|
|
941
960
|
if (failure !== void 0) throw failure;
|
|
942
961
|
signal.throwIfAborted();
|
|
@@ -954,7 +973,12 @@ async function syncRemoteResultArchive(params) {
|
|
|
954
973
|
if (startedEntries > 0 && (startedEntries >= maxEntries || Date.now() - startedAt >= deadlineMs)) {
|
|
955
974
|
await drainEntries();
|
|
956
975
|
await flushProgress(true);
|
|
957
|
-
return {
|
|
976
|
+
return {
|
|
977
|
+
documentId: params.documentId,
|
|
978
|
+
revisionKey: params.revisionKey,
|
|
979
|
+
snapshot: params.snapshot,
|
|
980
|
+
completed: false
|
|
981
|
+
};
|
|
958
982
|
}
|
|
959
983
|
if (successfulEntries >= checkpointFileInterval || Date.now() - checkpointAt >= checkpointIntervalMs) {
|
|
960
984
|
await drainEntries();
|
|
@@ -968,9 +992,16 @@ async function syncRemoteResultArchive(params) {
|
|
|
968
992
|
await drainEntries();
|
|
969
993
|
await flushProgress(true);
|
|
970
994
|
await validateRequiredObjects(params, paths);
|
|
995
|
+
if (params.snapshot && source.canSkipEntries)
|
|
996
|
+
if (source.verifyArchiveChecksum) await source.verifyArchiveChecksum(entryHashes);
|
|
997
|
+
else await verifyArchiveChecksum(params, source.downloadIdentity, entryHashes);
|
|
971
998
|
await params.verifyRevision?.();
|
|
972
999
|
signal.throwIfAborted();
|
|
973
1000
|
const commit = {
|
|
1001
|
+
snapshotId: params.snapshot?.snapshotId,
|
|
1002
|
+
jobResultId: params.snapshot?.jobResultId,
|
|
1003
|
+
checksum: params.snapshot?.checksum,
|
|
1004
|
+
stage: params.snapshot?.stage,
|
|
974
1005
|
version: 1,
|
|
975
1006
|
documentId: params.documentId,
|
|
976
1007
|
revisionKey: params.revisionKey,
|
|
@@ -989,6 +1020,8 @@ async function syncRemoteResultArchive(params) {
|
|
|
989
1020
|
documentId: params.documentId,
|
|
990
1021
|
revisionKey: params.revisionKey,
|
|
991
1022
|
pinnedRevisionKey: params.revisionKey,
|
|
1023
|
+
snapshotChecksum: params.snapshot?.checksum,
|
|
1024
|
+
upstreamJobResultId: params.snapshot?.jobResultId,
|
|
992
1025
|
archiveIdentity: source.identity,
|
|
993
1026
|
archiveDownloadIdentity,
|
|
994
1027
|
archiveCacheBypassed,
|
|
@@ -1009,6 +1042,7 @@ async function syncRemoteResultArchive(params) {
|
|
|
1009
1042
|
documentId: params.documentId,
|
|
1010
1043
|
revisionKey: params.revisionKey,
|
|
1011
1044
|
completed: true,
|
|
1045
|
+
snapshot: params.snapshot,
|
|
1012
1046
|
commit
|
|
1013
1047
|
};
|
|
1014
1048
|
} catch (error) {
|
|
@@ -1024,6 +1058,8 @@ async function syncRemoteResultArchive(params) {
|
|
|
1024
1058
|
documentId: params.documentId,
|
|
1025
1059
|
revisionKey: params.revisionKey,
|
|
1026
1060
|
pinnedRevisionKey: params.revisionKey,
|
|
1061
|
+
snapshotChecksum: params.snapshot?.checksum,
|
|
1062
|
+
upstreamJobResultId: params.snapshot?.jobResultId,
|
|
1027
1063
|
archiveIdentity: source.identity,
|
|
1028
1064
|
archiveDownloadIdentity,
|
|
1029
1065
|
archiveEntryHashes: entryHashes.slice(0, checkpointEntry),
|
|
@@ -1113,6 +1149,8 @@ async function persistArchiveCacheBypass(params, previous) {
|
|
|
1113
1149
|
documentId: params.documentId,
|
|
1114
1150
|
revisionKey: params.revisionKey,
|
|
1115
1151
|
pinnedRevisionKey: params.revisionKey,
|
|
1152
|
+
snapshotChecksum: params.snapshot?.checksum,
|
|
1153
|
+
upstreamJobResultId: params.snapshot?.jobResultId,
|
|
1116
1154
|
archiveDownloadIdentity: previous?.archiveDownloadIdentity,
|
|
1117
1155
|
archiveCacheBypassed: true,
|
|
1118
1156
|
status: "running",
|
|
@@ -1121,50 +1159,6 @@ async function persistArchiveCacheBypass(params, previous) {
|
|
|
1121
1159
|
await params.storage.writeSyncProgress(progress, { signal: params.signal });
|
|
1122
1160
|
return progress;
|
|
1123
1161
|
}
|
|
1124
|
-
var AbortableHttpRangeReader = class extends HttpRangeReader {
|
|
1125
|
-
constructor(url, options, controller) {
|
|
1126
|
-
super(url, options);
|
|
1127
|
-
this.controller = controller;
|
|
1128
|
-
}
|
|
1129
|
-
controller;
|
|
1130
|
-
createReadable(options) {
|
|
1131
|
-
const reader = super.createReadable(options).getReader();
|
|
1132
|
-
const cancellation = this.controller;
|
|
1133
|
-
return new ReadableStream(
|
|
1134
|
-
{
|
|
1135
|
-
async pull(output) {
|
|
1136
|
-
try {
|
|
1137
|
-
const chunk = await reader.read();
|
|
1138
|
-
if (chunk.done) {
|
|
1139
|
-
reader.releaseLock();
|
|
1140
|
-
output.close();
|
|
1141
|
-
} else output.enqueue(chunk.value);
|
|
1142
|
-
} catch (error) {
|
|
1143
|
-
cancellation.abort(error);
|
|
1144
|
-
reader.releaseLock();
|
|
1145
|
-
output.error(error);
|
|
1146
|
-
}
|
|
1147
|
-
},
|
|
1148
|
-
async cancel(reason) {
|
|
1149
|
-
try {
|
|
1150
|
-
await reader.cancel(reason);
|
|
1151
|
-
} finally {
|
|
1152
|
-
reader.releaseLock();
|
|
1153
|
-
}
|
|
1154
|
-
}
|
|
1155
|
-
},
|
|
1156
|
-
{ highWaterMark: 0 }
|
|
1157
|
-
);
|
|
1158
|
-
}
|
|
1159
|
-
async readUint8Array(index, length) {
|
|
1160
|
-
try {
|
|
1161
|
-
return await super.readUint8Array(index, length);
|
|
1162
|
-
} catch (error) {
|
|
1163
|
-
this.controller.abort(error);
|
|
1164
|
-
throw error;
|
|
1165
|
-
}
|
|
1166
|
-
}
|
|
1167
|
-
};
|
|
1168
1162
|
async function openArchiveTransport(params, previous, prepared) {
|
|
1169
1163
|
const resultUrl = prepared?.resultUrl ?? params.resultUrl;
|
|
1170
1164
|
const archiveCacheBypassed = prepared?.archiveCacheBypassed ?? previous?.archiveCacheBypassed;
|
|
@@ -1181,7 +1175,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1181
1175
|
const supportsRange = probe.status === 206 && probe.headers.has("content-range");
|
|
1182
1176
|
await probe.body?.cancel();
|
|
1183
1177
|
if (!supportsRange) {
|
|
1184
|
-
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-
|
|
1178
|
+
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-AIRFKO52.mjs");
|
|
1185
1179
|
const source = await openSequentialArchive(
|
|
1186
1180
|
{ ...params, resultUrl },
|
|
1187
1181
|
identity,
|
|
@@ -1189,43 +1183,32 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1189
1183
|
);
|
|
1190
1184
|
return { ...source, archiveCacheBypassed };
|
|
1191
1185
|
}
|
|
1192
|
-
const rangeController = new AbortController();
|
|
1193
|
-
const rangeSignal = params.signal ? AbortSignal.any([params.signal, rangeController.signal]) : rangeController.signal;
|
|
1194
1186
|
const options = {
|
|
1195
1187
|
forceRangeRequests: true,
|
|
1196
1188
|
preventHeadRequest: true,
|
|
1197
1189
|
// Cached reads already pin a strong ETag and exact lengths; CDN timestamps can drift.
|
|
1198
1190
|
checkResourceChanges: !prepared?.isCached,
|
|
1199
1191
|
fetch: async (input, init) => {
|
|
1192
|
+
let response;
|
|
1200
1193
|
try {
|
|
1201
|
-
|
|
1202
|
-
try {
|
|
1203
|
-
response = await fetch(input, { ...init, signal: rangeSignal });
|
|
1204
|
-
} catch (error) {
|
|
1205
|
-
rangeSignal.throwIfAborted();
|
|
1206
|
-
if (prepared?.isCached)
|
|
1207
|
-
throw new ArchiveCacheTransportError(
|
|
1208
|
-
"Parsed result archive cache transport unavailable."
|
|
1209
|
-
);
|
|
1210
|
-
throw error;
|
|
1211
|
-
}
|
|
1212
|
-
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1213
|
-
if (response.status !== 206) {
|
|
1214
|
-
const error = new Error(
|
|
1215
|
-
`Parsed result Range request failed (HTTP ${response.status}).`
|
|
1216
|
-
);
|
|
1217
|
-
rangeController.abort(error);
|
|
1218
|
-
await response.body?.cancel(error).catch(() => void 0);
|
|
1219
|
-
throw error;
|
|
1220
|
-
}
|
|
1221
|
-
return prepared ? prepared.validateResponse(response) : response;
|
|
1194
|
+
response = await fetch(input, { ...init, signal: params.signal });
|
|
1222
1195
|
} catch (error) {
|
|
1223
|
-
|
|
1196
|
+
params.signal?.throwIfAborted();
|
|
1197
|
+
if (prepared?.isCached)
|
|
1198
|
+
throw new ArchiveCacheTransportError(
|
|
1199
|
+
"Parsed result archive cache transport unavailable."
|
|
1200
|
+
);
|
|
1224
1201
|
throw error;
|
|
1225
1202
|
}
|
|
1203
|
+
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1204
|
+
if (response.status !== 206) {
|
|
1205
|
+
await response.body?.cancel();
|
|
1206
|
+
throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
|
|
1207
|
+
}
|
|
1208
|
+
return prepared ? prepared.validateResponse(response) : response;
|
|
1226
1209
|
}
|
|
1227
1210
|
};
|
|
1228
|
-
const reader = new ZipReader(new
|
|
1211
|
+
const reader = new ZipReader(new HttpRangeReader(resultUrl, options), {
|
|
1229
1212
|
useWebWorkers: false,
|
|
1230
1213
|
checkCrc32: true
|
|
1231
1214
|
});
|
|
@@ -1241,11 +1224,17 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1241
1224
|
)
|
|
1242
1225
|
).digest("hex");
|
|
1243
1226
|
return {
|
|
1244
|
-
signal: rangeSignal,
|
|
1245
1227
|
identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
|
|
1246
1228
|
downloadIdentity: identity,
|
|
1247
1229
|
archiveCacheBypassed,
|
|
1248
1230
|
isCached: prepared?.isCached,
|
|
1231
|
+
verifyArchiveChecksum: prepared?.isCached ? (entryHashes) => verifyArchiveChecksum(
|
|
1232
|
+
{ ...params, resultUrl },
|
|
1233
|
+
identity,
|
|
1234
|
+
entryHashes,
|
|
1235
|
+
false,
|
|
1236
|
+
prepared.validateResponse
|
|
1237
|
+
) : void 0,
|
|
1249
1238
|
canSkipEntries: previous?.archiveIdentity !== identity,
|
|
1250
1239
|
entries: (function* () {
|
|
1251
1240
|
for (const entry of files) {
|
|
@@ -1254,7 +1243,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1254
1243
|
path: normalizeResultPath(entry.filename),
|
|
1255
1244
|
contentLength: entry.uncompressedSize,
|
|
1256
1245
|
read: () => {
|
|
1257
|
-
entryRead = readRangeEntry(entry,
|
|
1246
|
+
entryRead = readRangeEntry(entry, params.signal, producers);
|
|
1258
1247
|
return entryRead.body;
|
|
1259
1248
|
},
|
|
1260
1249
|
waitForRead: () => entryRead?.completed ?? Promise.resolve()
|
|
@@ -1267,7 +1256,6 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1267
1256
|
}
|
|
1268
1257
|
};
|
|
1269
1258
|
} catch (error) {
|
|
1270
|
-
rangeController.abort(error);
|
|
1271
1259
|
await reader.close();
|
|
1272
1260
|
throw error;
|
|
1273
1261
|
}
|
|
@@ -1279,16 +1267,10 @@ function readRangeEntry(entry, signal, producers) {
|
|
|
1279
1267
|
streamController = controller;
|
|
1280
1268
|
}
|
|
1281
1269
|
});
|
|
1282
|
-
const cancelOutput = () => {
|
|
1283
|
-
streamController?.error(signal?.reason);
|
|
1284
|
-
};
|
|
1285
|
-
signal?.addEventListener("abort", cancelOutput, { once: true });
|
|
1286
|
-
if (signal?.aborted) cancelOutput();
|
|
1287
1270
|
const completed = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(() => void 0);
|
|
1288
1271
|
const producer = completed.catch((error) => {
|
|
1289
1272
|
streamController?.error(error);
|
|
1290
1273
|
}).finally(() => {
|
|
1291
|
-
signal?.removeEventListener("abort", cancelOutput);
|
|
1292
1274
|
producers.delete(producer);
|
|
1293
1275
|
});
|
|
1294
1276
|
producers.add(producer);
|
|
@@ -1325,6 +1307,9 @@ async function validateRequiredObjects(params, paths) {
|
|
|
1325
1307
|
const manifestJobId = manifest.job_id ?? manifest.jobId;
|
|
1326
1308
|
if (manifestJobId !== params.jobId)
|
|
1327
1309
|
throw new Error("Parsed result manifest job identity differs from the pinned job.");
|
|
1310
|
+
const manifestRevision = manifest.job_result_id ?? manifest.jobResultId;
|
|
1311
|
+
if (params.snapshot && manifestRevision !== void 0 && manifestRevision !== params.snapshot.jobResultId)
|
|
1312
|
+
throw new Error("Parsed result manifest revision differs from the snapshot upstream revision.");
|
|
1328
1313
|
const chunks = Array.isArray(payload) ? payload : isRecord(payload) ? payload.chunks : void 0;
|
|
1329
1314
|
if (!Array.isArray(chunks)) throw new Error("Invalid parsed result chunks.json.");
|
|
1330
1315
|
if (isRecord(manifest.files)) {
|
|
@@ -1349,7 +1334,49 @@ async function validateRequiredObjects(params, paths) {
|
|
|
1349
1334
|
for (const path of required)
|
|
1350
1335
|
if (!paths.has(path)) throw new Error(`Required parsed result file missing: ${path}`);
|
|
1351
1336
|
}
|
|
1337
|
+
async function verifyArchiveChecksum(params, identity, expectedEntryHashes, retainResultArchive = false, validateResponse) {
|
|
1338
|
+
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-AIRFKO52.mjs");
|
|
1339
|
+
const source = await openSequentialArchive(params, identity, validateResponse);
|
|
1340
|
+
let entryIndex = 0;
|
|
1341
|
+
try {
|
|
1342
|
+
for await (const entry of source.entries) {
|
|
1343
|
+
const hash = createHash2("sha256");
|
|
1344
|
+
for await (const part of entry.read()) {
|
|
1345
|
+
params.signal?.throwIfAborted();
|
|
1346
|
+
hash.update(part);
|
|
1347
|
+
}
|
|
1348
|
+
const entryHash = createHash2("sha256").update(entry.path).update(hash.digest()).digest("hex");
|
|
1349
|
+
if (expectedEntryHashes[entryIndex] !== entryHash)
|
|
1350
|
+
throw new Error(
|
|
1351
|
+
"Parsed snapshot archive entry changed between extraction and checksum verification."
|
|
1352
|
+
);
|
|
1353
|
+
entryIndex += 1;
|
|
1354
|
+
}
|
|
1355
|
+
if (entryIndex !== expectedEntryHashes.length)
|
|
1356
|
+
throw new Error("Parsed snapshot archive entry count changed during checksum verification.");
|
|
1357
|
+
if (retainResultArchive && source.readArchive) {
|
|
1358
|
+
try {
|
|
1359
|
+
await writeStream(params.storage, {
|
|
1360
|
+
documentId: params.documentId,
|
|
1361
|
+
revisionKey: params.revisionKey,
|
|
1362
|
+
path: "result.zip",
|
|
1363
|
+
body: source.readArchive(),
|
|
1364
|
+
contentType: "application/zip",
|
|
1365
|
+
signal: params.signal
|
|
1366
|
+
});
|
|
1367
|
+
} catch {
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
} finally {
|
|
1371
|
+
await source.close();
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1352
1374
|
async function retainArchive(params, expectedIdentity) {
|
|
1375
|
+
if (params.snapshot) {
|
|
1376
|
+
const progress = await params.storage.readSyncProgress(params);
|
|
1377
|
+
await verifyArchiveChecksum(params, expectedIdentity, progress?.archiveEntryHashes ?? [], true);
|
|
1378
|
+
return;
|
|
1379
|
+
}
|
|
1353
1380
|
const response = await fetch(params.resultUrl, { signal: params.signal });
|
|
1354
1381
|
if (!response.ok || !response.body)
|
|
1355
1382
|
throw new Error(`Optional archive download failed (HTTP ${response.status}).`);
|
package/dist/index.d.mts
CHANGED
|
@@ -74,6 +74,9 @@ interface JobError {
|
|
|
74
74
|
* Job status response with full details
|
|
75
75
|
*/
|
|
76
76
|
interface JobResult {
|
|
77
|
+
enhancement?: JobEnhancement;
|
|
78
|
+
snapshots?: readonly JobSnapshot[];
|
|
79
|
+
currentSnapshotId?: string;
|
|
77
80
|
/** Unique job identifier */
|
|
78
81
|
jobId: string;
|
|
79
82
|
/** Current job status */
|
|
@@ -117,6 +120,28 @@ interface JobResult {
|
|
|
117
120
|
/** Whether the job failed */
|
|
118
121
|
readonly isFailed: boolean;
|
|
119
122
|
}
|
|
123
|
+
/** Background work does not change an already completed ingestion job. */
|
|
124
|
+
interface JobEnhancement {
|
|
125
|
+
readonly status: 'pending' | 'running' | 'complete' | 'failed';
|
|
126
|
+
readonly canRetry: boolean;
|
|
127
|
+
readonly attempt: number;
|
|
128
|
+
readonly error?: string | null;
|
|
129
|
+
}
|
|
130
|
+
/** Immutable archive generation; signed URL capabilities may be refreshed. */
|
|
131
|
+
interface JobSnapshot {
|
|
132
|
+
readonly snapshotId: string;
|
|
133
|
+
readonly stage: 'baseline' | 'final';
|
|
134
|
+
readonly jobId: string;
|
|
135
|
+
readonly jobResultId: string;
|
|
136
|
+
readonly documentId: string;
|
|
137
|
+
readonly checksum: string;
|
|
138
|
+
readonly resultUrl: string;
|
|
139
|
+
readonly resultUrlExpiresAt: Date;
|
|
140
|
+
}
|
|
141
|
+
interface JobEnhancementRetryResult {
|
|
142
|
+
readonly jobId: string;
|
|
143
|
+
readonly enhancement: JobEnhancement;
|
|
144
|
+
}
|
|
120
145
|
|
|
121
146
|
/**
|
|
122
147
|
* Statistics about the parsed document
|
|
@@ -455,6 +480,10 @@ interface ParsedDocumentWriteObjectResult extends ParsedDocumentObjectParams {
|
|
|
455
480
|
}
|
|
456
481
|
type ParsedDocumentCommitSource = 'resultZip' | 'remoteReconstruction';
|
|
457
482
|
interface ParsedDocumentCommit {
|
|
483
|
+
readonly snapshotId?: string;
|
|
484
|
+
readonly jobResultId?: string;
|
|
485
|
+
readonly checksum?: string;
|
|
486
|
+
readonly stage?: 'baseline' | 'final';
|
|
458
487
|
readonly version: 1;
|
|
459
488
|
readonly documentId: string;
|
|
460
489
|
readonly revisionKey: string;
|
|
@@ -464,6 +493,8 @@ interface ParsedDocumentCommit {
|
|
|
464
493
|
interface ParsedDocumentSyncProgress {
|
|
465
494
|
readonly documentId: string;
|
|
466
495
|
readonly revisionKey: string;
|
|
496
|
+
readonly snapshotChecksum?: string;
|
|
497
|
+
readonly upstreamJobResultId?: string;
|
|
467
498
|
readonly nextChunkPage?: number;
|
|
468
499
|
readonly pinnedRevisionKey?: string;
|
|
469
500
|
readonly archiveIdentity?: string;
|
|
@@ -830,6 +861,8 @@ declare class Jobs extends BaseResource {
|
|
|
830
861
|
* Get job status
|
|
831
862
|
*/
|
|
832
863
|
get(jobId: string): Promise<JobResult>;
|
|
864
|
+
/** Renew unfinished enhancement without creating another ingestion. */
|
|
865
|
+
retryEnhancement(jobId: string): Promise<JobEnhancementRetryResult>;
|
|
833
866
|
/**
|
|
834
867
|
* Upload file for a job
|
|
835
868
|
*/
|
|
@@ -924,6 +957,10 @@ interface RetrievalSource {
|
|
|
924
957
|
sourceFileName?: string | null;
|
|
925
958
|
/** Human-readable section path */
|
|
926
959
|
sectionPath?: string | null;
|
|
960
|
+
/** Physical source pages, stable across progressive parsing snapshots. */
|
|
961
|
+
pageNums?: number[];
|
|
962
|
+
/** Original ingestion revision, shared by its baseline and final snapshots. */
|
|
963
|
+
jobResultId?: string | null;
|
|
927
964
|
}
|
|
928
965
|
/**
|
|
929
966
|
* One composed evidence part. Tables stay in text as HTML; images are inline bytes.
|
|
@@ -981,6 +1018,10 @@ interface RetrievalReferencedChunk {
|
|
|
981
1018
|
filePath?: string | null;
|
|
982
1019
|
/** Published job identifier for the referenced chunk */
|
|
983
1020
|
jobId?: string | null;
|
|
1021
|
+
/** Original ingestion revision, shared by its baseline and final snapshots. */
|
|
1022
|
+
jobResultId?: string | null;
|
|
1023
|
+
/** All physical source pages supporting this evidence. */
|
|
1024
|
+
pageNums?: number[];
|
|
984
1025
|
/** Presigned asset URL for media chunks when available */
|
|
985
1026
|
assetUrl?: string | null;
|
|
986
1027
|
/** Chunk metadata returned by the API */
|
|
@@ -1332,7 +1373,7 @@ interface KnowledgeDocumentReference {
|
|
|
1332
1373
|
documentId?: string;
|
|
1333
1374
|
/** Server parse job identifier whose completed result should be cached before reading. */
|
|
1334
1375
|
jobId?: string;
|
|
1335
|
-
/**
|
|
1376
|
+
/** Opaque snapshot ID for progressive archives; upstream Job Result ID for legacy results. */
|
|
1336
1377
|
revisionKey?: string;
|
|
1337
1378
|
}
|
|
1338
1379
|
type KnowledgeParsedStorageOptions = ParsedDocumentStorageConfig;
|
|
@@ -1344,6 +1385,8 @@ interface KnowledgeSyncParsedDocumentParams extends KnowledgeDocumentReference {
|
|
|
1344
1385
|
readonly cacheResultArchive?: boolean;
|
|
1345
1386
|
}
|
|
1346
1387
|
interface KnowledgeSyncParsedDocumentResponse {
|
|
1388
|
+
/** Selected immutable descriptor; reuse revisionKey for each segmented continuation. */
|
|
1389
|
+
readonly snapshot?: JobSnapshot;
|
|
1347
1390
|
documentId: string;
|
|
1348
1391
|
revisionKey: string;
|
|
1349
1392
|
completed: boolean;
|
|
@@ -1491,6 +1534,9 @@ declare class Knowledge {
|
|
|
1491
1534
|
/** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */
|
|
1492
1535
|
cacheJobResult(params: KnowledgeCacheJobResultParams): Promise<LocalKnowledgeParseResponse>;
|
|
1493
1536
|
syncParsedDocument(params: KnowledgeSyncParsedDocumentParams): Promise<KnowledgeSyncParsedDocumentResponse>;
|
|
1537
|
+
private performParsedDocumentSync;
|
|
1538
|
+
private syncJobSnapshot;
|
|
1539
|
+
private resolveStoredSnapshotKey;
|
|
1494
1540
|
private syncParseResultToConfiguredStorage;
|
|
1495
1541
|
private parseWithoutClientStorageAdapter;
|
|
1496
1542
|
cacheDocument(params: KnowledgeCacheDocumentParams): Promise<LocalKnowledgeParseResponse>;
|
|
@@ -1794,7 +1840,7 @@ declare class JobFailedError extends KnowhereError {
|
|
|
1794
1840
|
constructor(message: string, code: string, jobResult: JobResult);
|
|
1795
1841
|
}
|
|
1796
1842
|
|
|
1797
|
-
/**
|
|
1843
|
+
/** Sequential ZIP extraction or snapshot verification could not access temporary storage. */
|
|
1798
1844
|
declare class ParsedDocumentArchiveError extends KnowhereError {
|
|
1799
1845
|
readonly code: "ARCHIVE_SPOOL_FAILED";
|
|
1800
1846
|
readonly cause: unknown;
|
package/dist/index.d.ts
CHANGED
|
@@ -74,6 +74,9 @@ interface JobError {
|
|
|
74
74
|
* Job status response with full details
|
|
75
75
|
*/
|
|
76
76
|
interface JobResult {
|
|
77
|
+
enhancement?: JobEnhancement;
|
|
78
|
+
snapshots?: readonly JobSnapshot[];
|
|
79
|
+
currentSnapshotId?: string;
|
|
77
80
|
/** Unique job identifier */
|
|
78
81
|
jobId: string;
|
|
79
82
|
/** Current job status */
|
|
@@ -117,6 +120,28 @@ interface JobResult {
|
|
|
117
120
|
/** Whether the job failed */
|
|
118
121
|
readonly isFailed: boolean;
|
|
119
122
|
}
|
|
123
|
+
/** Background work does not change an already completed ingestion job. */
|
|
124
|
+
interface JobEnhancement {
|
|
125
|
+
readonly status: 'pending' | 'running' | 'complete' | 'failed';
|
|
126
|
+
readonly canRetry: boolean;
|
|
127
|
+
readonly attempt: number;
|
|
128
|
+
readonly error?: string | null;
|
|
129
|
+
}
|
|
130
|
+
/** Immutable archive generation; signed URL capabilities may be refreshed. */
|
|
131
|
+
interface JobSnapshot {
|
|
132
|
+
readonly snapshotId: string;
|
|
133
|
+
readonly stage: 'baseline' | 'final';
|
|
134
|
+
readonly jobId: string;
|
|
135
|
+
readonly jobResultId: string;
|
|
136
|
+
readonly documentId: string;
|
|
137
|
+
readonly checksum: string;
|
|
138
|
+
readonly resultUrl: string;
|
|
139
|
+
readonly resultUrlExpiresAt: Date;
|
|
140
|
+
}
|
|
141
|
+
interface JobEnhancementRetryResult {
|
|
142
|
+
readonly jobId: string;
|
|
143
|
+
readonly enhancement: JobEnhancement;
|
|
144
|
+
}
|
|
120
145
|
|
|
121
146
|
/**
|
|
122
147
|
* Statistics about the parsed document
|
|
@@ -455,6 +480,10 @@ interface ParsedDocumentWriteObjectResult extends ParsedDocumentObjectParams {
|
|
|
455
480
|
}
|
|
456
481
|
type ParsedDocumentCommitSource = 'resultZip' | 'remoteReconstruction';
|
|
457
482
|
interface ParsedDocumentCommit {
|
|
483
|
+
readonly snapshotId?: string;
|
|
484
|
+
readonly jobResultId?: string;
|
|
485
|
+
readonly checksum?: string;
|
|
486
|
+
readonly stage?: 'baseline' | 'final';
|
|
458
487
|
readonly version: 1;
|
|
459
488
|
readonly documentId: string;
|
|
460
489
|
readonly revisionKey: string;
|
|
@@ -464,6 +493,8 @@ interface ParsedDocumentCommit {
|
|
|
464
493
|
interface ParsedDocumentSyncProgress {
|
|
465
494
|
readonly documentId: string;
|
|
466
495
|
readonly revisionKey: string;
|
|
496
|
+
readonly snapshotChecksum?: string;
|
|
497
|
+
readonly upstreamJobResultId?: string;
|
|
467
498
|
readonly nextChunkPage?: number;
|
|
468
499
|
readonly pinnedRevisionKey?: string;
|
|
469
500
|
readonly archiveIdentity?: string;
|
|
@@ -830,6 +861,8 @@ declare class Jobs extends BaseResource {
|
|
|
830
861
|
* Get job status
|
|
831
862
|
*/
|
|
832
863
|
get(jobId: string): Promise<JobResult>;
|
|
864
|
+
/** Renew unfinished enhancement without creating another ingestion. */
|
|
865
|
+
retryEnhancement(jobId: string): Promise<JobEnhancementRetryResult>;
|
|
833
866
|
/**
|
|
834
867
|
* Upload file for a job
|
|
835
868
|
*/
|
|
@@ -924,6 +957,10 @@ interface RetrievalSource {
|
|
|
924
957
|
sourceFileName?: string | null;
|
|
925
958
|
/** Human-readable section path */
|
|
926
959
|
sectionPath?: string | null;
|
|
960
|
+
/** Physical source pages, stable across progressive parsing snapshots. */
|
|
961
|
+
pageNums?: number[];
|
|
962
|
+
/** Original ingestion revision, shared by its baseline and final snapshots. */
|
|
963
|
+
jobResultId?: string | null;
|
|
927
964
|
}
|
|
928
965
|
/**
|
|
929
966
|
* One composed evidence part. Tables stay in text as HTML; images are inline bytes.
|
|
@@ -981,6 +1018,10 @@ interface RetrievalReferencedChunk {
|
|
|
981
1018
|
filePath?: string | null;
|
|
982
1019
|
/** Published job identifier for the referenced chunk */
|
|
983
1020
|
jobId?: string | null;
|
|
1021
|
+
/** Original ingestion revision, shared by its baseline and final snapshots. */
|
|
1022
|
+
jobResultId?: string | null;
|
|
1023
|
+
/** All physical source pages supporting this evidence. */
|
|
1024
|
+
pageNums?: number[];
|
|
984
1025
|
/** Presigned asset URL for media chunks when available */
|
|
985
1026
|
assetUrl?: string | null;
|
|
986
1027
|
/** Chunk metadata returned by the API */
|
|
@@ -1332,7 +1373,7 @@ interface KnowledgeDocumentReference {
|
|
|
1332
1373
|
documentId?: string;
|
|
1333
1374
|
/** Server parse job identifier whose completed result should be cached before reading. */
|
|
1334
1375
|
jobId?: string;
|
|
1335
|
-
/**
|
|
1376
|
+
/** Opaque snapshot ID for progressive archives; upstream Job Result ID for legacy results. */
|
|
1336
1377
|
revisionKey?: string;
|
|
1337
1378
|
}
|
|
1338
1379
|
type KnowledgeParsedStorageOptions = ParsedDocumentStorageConfig;
|
|
@@ -1344,6 +1385,8 @@ interface KnowledgeSyncParsedDocumentParams extends KnowledgeDocumentReference {
|
|
|
1344
1385
|
readonly cacheResultArchive?: boolean;
|
|
1345
1386
|
}
|
|
1346
1387
|
interface KnowledgeSyncParsedDocumentResponse {
|
|
1388
|
+
/** Selected immutable descriptor; reuse revisionKey for each segmented continuation. */
|
|
1389
|
+
readonly snapshot?: JobSnapshot;
|
|
1347
1390
|
documentId: string;
|
|
1348
1391
|
revisionKey: string;
|
|
1349
1392
|
completed: boolean;
|
|
@@ -1491,6 +1534,9 @@ declare class Knowledge {
|
|
|
1491
1534
|
/** @deprecated Use importJobResult for local cache imports or loadJobResult for server-safe loads. */
|
|
1492
1535
|
cacheJobResult(params: KnowledgeCacheJobResultParams): Promise<LocalKnowledgeParseResponse>;
|
|
1493
1536
|
syncParsedDocument(params: KnowledgeSyncParsedDocumentParams): Promise<KnowledgeSyncParsedDocumentResponse>;
|
|
1537
|
+
private performParsedDocumentSync;
|
|
1538
|
+
private syncJobSnapshot;
|
|
1539
|
+
private resolveStoredSnapshotKey;
|
|
1494
1540
|
private syncParseResultToConfiguredStorage;
|
|
1495
1541
|
private parseWithoutClientStorageAdapter;
|
|
1496
1542
|
cacheDocument(params: KnowledgeCacheDocumentParams): Promise<LocalKnowledgeParseResponse>;
|
|
@@ -1794,7 +1840,7 @@ declare class JobFailedError extends KnowhereError {
|
|
|
1794
1840
|
constructor(message: string, code: string, jobResult: JobResult);
|
|
1795
1841
|
}
|
|
1796
1842
|
|
|
1797
|
-
/**
|
|
1843
|
+
/** Sequential ZIP extraction or snapshot verification could not access temporary storage. */
|
|
1798
1844
|
declare class ParsedDocumentArchiveError extends KnowhereError {
|
|
1799
1845
|
readonly code: "ARCHIVE_SPOOL_FAILED";
|
|
1800
1846
|
readonly cause: unknown;
|