@ontos-ai/knowhere-sdk 2.6.0 → 2.6.1-beta.20261027152313
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -770,15 +770,17 @@ function readObjectVersion(object) {
|
|
|
770
770
|
return { entityTag: object.entityTag, contentLength: object.contentLength };
|
|
771
771
|
}
|
|
772
772
|
function validateCachedResponse(response, marker, signal) {
|
|
773
|
-
if (response.
|
|
773
|
+
if (response.status !== 206 || !response.body)
|
|
774
774
|
rejectResponse(
|
|
775
775
|
response,
|
|
776
|
-
new ArchiveCacheTransportError(
|
|
776
|
+
new ArchiveCacheTransportError(
|
|
777
|
+
`Parsed result archive cache requires a ranged response (HTTP ${response.status}).`
|
|
778
|
+
)
|
|
777
779
|
);
|
|
778
|
-
if (response.
|
|
780
|
+
if (response.headers.get("etag") !== marker.storageEntityTag)
|
|
779
781
|
rejectResponse(
|
|
780
782
|
response,
|
|
781
|
-
new ArchiveCacheTransportError("Parsed result archive cache
|
|
783
|
+
new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
|
|
782
784
|
);
|
|
783
785
|
const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
|
|
784
786
|
response.headers.get("content-range") ?? ""
|
|
@@ -845,7 +847,10 @@ function isPositiveInteger(value) {
|
|
|
845
847
|
}
|
|
846
848
|
|
|
847
849
|
// src/storage/streamed-parsed-document-sync.ts
|
|
848
|
-
import {
|
|
850
|
+
import {
|
|
851
|
+
HttpRangeReader,
|
|
852
|
+
ZipReader
|
|
853
|
+
} from "@zip.js/zip.js";
|
|
849
854
|
async function syncRemoteResultArchive(params) {
|
|
850
855
|
params.signal?.throwIfAborted();
|
|
851
856
|
validateArchiveSyncLimits(params.limits);
|
|
@@ -856,10 +861,14 @@ async function syncRemoteResultArchive(params) {
|
|
|
856
861
|
const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
|
|
857
862
|
const controller = new AbortController();
|
|
858
863
|
const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
|
|
859
|
-
const
|
|
864
|
+
const openingParams = { ...params, signal };
|
|
860
865
|
const previous = await params.storage.readSyncProgress(params);
|
|
861
|
-
const openedSource = await openArchive(
|
|
866
|
+
const openedSource = await openArchive(openingParams, previous);
|
|
862
867
|
const source = !openedSource.canSkipEntries && previous?.archiveIdentity && previous.archiveDownloadIdentity === openedSource.downloadIdentity ? { ...openedSource, identity: previous.archiveIdentity } : openedSource;
|
|
868
|
+
const transferParams = {
|
|
869
|
+
...openingParams,
|
|
870
|
+
signal: source.signal ?? signal
|
|
871
|
+
};
|
|
863
872
|
const archiveDownloadIdentity = source.isCached ? source.downloadIdentity : previous?.archiveDownloadIdentity;
|
|
864
873
|
const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
|
|
865
874
|
const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
|
|
@@ -1112,6 +1121,50 @@ async function persistArchiveCacheBypass(params, previous) {
|
|
|
1112
1121
|
await params.storage.writeSyncProgress(progress, { signal: params.signal });
|
|
1113
1122
|
return progress;
|
|
1114
1123
|
}
|
|
1124
|
+
var AbortableHttpRangeReader = class extends HttpRangeReader {
|
|
1125
|
+
constructor(url, options, controller) {
|
|
1126
|
+
super(url, options);
|
|
1127
|
+
this.controller = controller;
|
|
1128
|
+
}
|
|
1129
|
+
controller;
|
|
1130
|
+
createReadable(options) {
|
|
1131
|
+
const reader = super.createReadable(options).getReader();
|
|
1132
|
+
const cancellation = this.controller;
|
|
1133
|
+
return new ReadableStream(
|
|
1134
|
+
{
|
|
1135
|
+
async pull(output) {
|
|
1136
|
+
try {
|
|
1137
|
+
const chunk = await reader.read();
|
|
1138
|
+
if (chunk.done) {
|
|
1139
|
+
reader.releaseLock();
|
|
1140
|
+
output.close();
|
|
1141
|
+
} else output.enqueue(chunk.value);
|
|
1142
|
+
} catch (error) {
|
|
1143
|
+
cancellation.abort(error);
|
|
1144
|
+
reader.releaseLock();
|
|
1145
|
+
output.error(error);
|
|
1146
|
+
}
|
|
1147
|
+
},
|
|
1148
|
+
async cancel(reason) {
|
|
1149
|
+
try {
|
|
1150
|
+
await reader.cancel(reason);
|
|
1151
|
+
} finally {
|
|
1152
|
+
reader.releaseLock();
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
},
|
|
1156
|
+
{ highWaterMark: 0 }
|
|
1157
|
+
);
|
|
1158
|
+
}
|
|
1159
|
+
async readUint8Array(index, length) {
|
|
1160
|
+
try {
|
|
1161
|
+
return await super.readUint8Array(index, length);
|
|
1162
|
+
} catch (error) {
|
|
1163
|
+
this.controller.abort(error);
|
|
1164
|
+
throw error;
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1115
1168
|
async function openArchiveTransport(params, previous, prepared) {
|
|
1116
1169
|
const resultUrl = prepared?.resultUrl ?? params.resultUrl;
|
|
1117
1170
|
const archiveCacheBypassed = prepared?.archiveCacheBypassed ?? previous?.archiveCacheBypassed;
|
|
@@ -1128,7 +1181,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1128
1181
|
const supportsRange = probe.status === 206 && probe.headers.has("content-range");
|
|
1129
1182
|
await probe.body?.cancel();
|
|
1130
1183
|
if (!supportsRange) {
|
|
1131
|
-
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-
|
|
1184
|
+
const { openSequentialArchive } = await import("./sequential-parsed-document-archive-W7HPYIKA.mjs");
|
|
1132
1185
|
const source = await openSequentialArchive(
|
|
1133
1186
|
{ ...params, resultUrl },
|
|
1134
1187
|
identity,
|
|
@@ -1136,32 +1189,43 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1136
1189
|
);
|
|
1137
1190
|
return { ...source, archiveCacheBypassed };
|
|
1138
1191
|
}
|
|
1192
|
+
const rangeController = new AbortController();
|
|
1193
|
+
const rangeSignal = params.signal ? AbortSignal.any([params.signal, rangeController.signal]) : rangeController.signal;
|
|
1139
1194
|
const options = {
|
|
1140
1195
|
forceRangeRequests: true,
|
|
1141
1196
|
preventHeadRequest: true,
|
|
1142
1197
|
// Cached reads already pin a strong ETag and exact lengths; CDN timestamps can drift.
|
|
1143
1198
|
checkResourceChanges: !prepared?.isCached,
|
|
1144
1199
|
fetch: async (input, init) => {
|
|
1145
|
-
let response;
|
|
1146
1200
|
try {
|
|
1147
|
-
response
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1201
|
+
let response;
|
|
1202
|
+
try {
|
|
1203
|
+
response = await fetch(input, { ...init, signal: rangeSignal });
|
|
1204
|
+
} catch (error) {
|
|
1205
|
+
rangeSignal.throwIfAborted();
|
|
1206
|
+
if (prepared?.isCached)
|
|
1207
|
+
throw new ArchiveCacheTransportError(
|
|
1208
|
+
"Parsed result archive cache transport unavailable."
|
|
1209
|
+
);
|
|
1210
|
+
throw error;
|
|
1211
|
+
}
|
|
1212
|
+
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1213
|
+
if (response.status !== 206) {
|
|
1214
|
+
const error = new Error(
|
|
1215
|
+
`Parsed result Range request failed (HTTP ${response.status}).`
|
|
1153
1216
|
);
|
|
1217
|
+
rangeController.abort(error);
|
|
1218
|
+
await response.body?.cancel(error).catch(() => void 0);
|
|
1219
|
+
throw error;
|
|
1220
|
+
}
|
|
1221
|
+
return prepared ? prepared.validateResponse(response) : response;
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
rangeController.abort(error);
|
|
1154
1224
|
throw error;
|
|
1155
1225
|
}
|
|
1156
|
-
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1157
|
-
if (response.status !== 206) {
|
|
1158
|
-
await response.body?.cancel();
|
|
1159
|
-
throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
|
|
1160
|
-
}
|
|
1161
|
-
return prepared ? prepared.validateResponse(response) : response;
|
|
1162
1226
|
}
|
|
1163
1227
|
};
|
|
1164
|
-
const reader = new ZipReader(new
|
|
1228
|
+
const reader = new ZipReader(new AbortableHttpRangeReader(resultUrl, options, rangeController), {
|
|
1165
1229
|
useWebWorkers: false,
|
|
1166
1230
|
checkCrc32: true
|
|
1167
1231
|
});
|
|
@@ -1177,6 +1241,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1177
1241
|
)
|
|
1178
1242
|
).digest("hex");
|
|
1179
1243
|
return {
|
|
1244
|
+
signal: rangeSignal,
|
|
1180
1245
|
identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
|
|
1181
1246
|
downloadIdentity: identity,
|
|
1182
1247
|
archiveCacheBypassed,
|
|
@@ -1189,7 +1254,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1189
1254
|
path: normalizeResultPath(entry.filename),
|
|
1190
1255
|
contentLength: entry.uncompressedSize,
|
|
1191
1256
|
read: () => {
|
|
1192
|
-
entryRead = readRangeEntry(entry,
|
|
1257
|
+
entryRead = readRangeEntry(entry, rangeSignal, producers);
|
|
1193
1258
|
return entryRead.body;
|
|
1194
1259
|
},
|
|
1195
1260
|
waitForRead: () => entryRead?.completed ?? Promise.resolve()
|
|
@@ -1202,6 +1267,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1202
1267
|
}
|
|
1203
1268
|
};
|
|
1204
1269
|
} catch (error) {
|
|
1270
|
+
rangeController.abort(error);
|
|
1205
1271
|
await reader.close();
|
|
1206
1272
|
throw error;
|
|
1207
1273
|
}
|
|
@@ -1213,10 +1279,16 @@ function readRangeEntry(entry, signal, producers) {
|
|
|
1213
1279
|
streamController = controller;
|
|
1214
1280
|
}
|
|
1215
1281
|
});
|
|
1282
|
+
const cancelOutput = () => {
|
|
1283
|
+
streamController?.error(signal?.reason);
|
|
1284
|
+
};
|
|
1285
|
+
signal?.addEventListener("abort", cancelOutput, { once: true });
|
|
1286
|
+
if (signal?.aborted) cancelOutput();
|
|
1216
1287
|
const completed = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(() => void 0);
|
|
1217
1288
|
const producer = completed.catch((error) => {
|
|
1218
1289
|
streamController?.error(error);
|
|
1219
1290
|
}).finally(() => {
|
|
1291
|
+
signal?.removeEventListener("abort", cancelOutput);
|
|
1220
1292
|
producers.delete(producer);
|
|
1221
1293
|
});
|
|
1222
1294
|
producers.add(producer);
|
package/dist/index.js
CHANGED
|
@@ -856,15 +856,17 @@ function readObjectVersion(object) {
|
|
|
856
856
|
return { entityTag: object.entityTag, contentLength: object.contentLength };
|
|
857
857
|
}
|
|
858
858
|
function validateCachedResponse(response, marker, signal) {
|
|
859
|
-
if (response.
|
|
859
|
+
if (response.status !== 206 || !response.body)
|
|
860
860
|
rejectResponse(
|
|
861
861
|
response,
|
|
862
|
-
new ArchiveCacheTransportError(
|
|
862
|
+
new ArchiveCacheTransportError(
|
|
863
|
+
`Parsed result archive cache requires a ranged response (HTTP ${response.status}).`
|
|
864
|
+
)
|
|
863
865
|
);
|
|
864
|
-
if (response.
|
|
866
|
+
if (response.headers.get("etag") !== marker.storageEntityTag)
|
|
865
867
|
rejectResponse(
|
|
866
868
|
response,
|
|
867
|
-
new ArchiveCacheTransportError("Parsed result archive cache
|
|
869
|
+
new ArchiveCacheTransportError("Parsed result archive cache response version changed.")
|
|
868
870
|
);
|
|
869
871
|
const range = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(
|
|
870
872
|
response.headers.get("content-range") ?? ""
|
|
@@ -1099,10 +1101,14 @@ async function syncRemoteResultArchive(params) {
|
|
|
1099
1101
|
const checkpointIntervalMs = params.limits?.checkpointIntervalMs ?? 2e3;
|
|
1100
1102
|
const controller = new AbortController();
|
|
1101
1103
|
const signal = params.signal ? AbortSignal.any([params.signal, controller.signal]) : controller.signal;
|
|
1102
|
-
const
|
|
1104
|
+
const openingParams = { ...params, signal };
|
|
1103
1105
|
const previous = await params.storage.readSyncProgress(params);
|
|
1104
|
-
const openedSource = await openArchive(
|
|
1106
|
+
const openedSource = await openArchive(openingParams, previous);
|
|
1105
1107
|
const source = !openedSource.canSkipEntries && previous?.archiveIdentity && previous.archiveDownloadIdentity === openedSource.downloadIdentity ? { ...openedSource, identity: previous.archiveIdentity } : openedSource;
|
|
1108
|
+
const transferParams = {
|
|
1109
|
+
...openingParams,
|
|
1110
|
+
signal: source.signal ?? signal
|
|
1111
|
+
};
|
|
1106
1112
|
const archiveDownloadIdentity = source.isCached ? source.downloadIdentity : previous?.archiveDownloadIdentity;
|
|
1107
1113
|
const concurrency = source.canSkipEntries ? params.limits?.maxConcurrentFiles ?? 1 : 1;
|
|
1108
1114
|
const nextEntry = previous?.status === "completed" ? 0 : previous?.nextArchiveEntry ?? 0;
|
|
@@ -1379,32 +1385,43 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1379
1385
|
);
|
|
1380
1386
|
return { ...source, archiveCacheBypassed };
|
|
1381
1387
|
}
|
|
1388
|
+
const rangeController = new AbortController();
|
|
1389
|
+
const rangeSignal = params.signal ? AbortSignal.any([params.signal, rangeController.signal]) : rangeController.signal;
|
|
1382
1390
|
const options = {
|
|
1383
1391
|
forceRangeRequests: true,
|
|
1384
1392
|
preventHeadRequest: true,
|
|
1385
1393
|
// Cached reads already pin a strong ETag and exact lengths; CDN timestamps can drift.
|
|
1386
1394
|
checkResourceChanges: !prepared?.isCached,
|
|
1387
1395
|
fetch: async (input, init) => {
|
|
1388
|
-
let response;
|
|
1389
1396
|
try {
|
|
1390
|
-
response
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1397
|
+
let response;
|
|
1398
|
+
try {
|
|
1399
|
+
response = await fetch(input, { ...init, signal: rangeSignal });
|
|
1400
|
+
} catch (error) {
|
|
1401
|
+
rangeSignal.throwIfAborted();
|
|
1402
|
+
if (prepared?.isCached)
|
|
1403
|
+
throw new ArchiveCacheTransportError(
|
|
1404
|
+
"Parsed result archive cache transport unavailable."
|
|
1405
|
+
);
|
|
1406
|
+
throw error;
|
|
1407
|
+
}
|
|
1408
|
+
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1409
|
+
if (response.status !== 206) {
|
|
1410
|
+
const error = new Error(
|
|
1411
|
+
`Parsed result Range request failed (HTTP ${response.status}).`
|
|
1396
1412
|
);
|
|
1413
|
+
rangeController.abort(error);
|
|
1414
|
+
await response.body?.cancel(error).catch(() => void 0);
|
|
1415
|
+
throw error;
|
|
1416
|
+
}
|
|
1417
|
+
return prepared ? prepared.validateResponse(response) : response;
|
|
1418
|
+
} catch (error) {
|
|
1419
|
+
rangeController.abort(error);
|
|
1397
1420
|
throw error;
|
|
1398
1421
|
}
|
|
1399
|
-
if (prepared?.isCached) return prepared.validateResponse(response);
|
|
1400
|
-
if (response.status !== 206) {
|
|
1401
|
-
await response.body?.cancel();
|
|
1402
|
-
throw new Error(`Parsed result Range request failed (HTTP ${response.status}).`);
|
|
1403
|
-
}
|
|
1404
|
-
return prepared ? prepared.validateResponse(response) : response;
|
|
1405
1422
|
}
|
|
1406
1423
|
};
|
|
1407
|
-
const reader = new import_zip2.ZipReader(new
|
|
1424
|
+
const reader = new import_zip2.ZipReader(new AbortableHttpRangeReader(resultUrl, options, rangeController), {
|
|
1408
1425
|
useWebWorkers: false,
|
|
1409
1426
|
checkCrc32: true
|
|
1410
1427
|
});
|
|
@@ -1420,6 +1437,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1420
1437
|
)
|
|
1421
1438
|
).digest("hex");
|
|
1422
1439
|
return {
|
|
1440
|
+
signal: rangeSignal,
|
|
1423
1441
|
identity: previous?.archiveIdentity === identity ? identity : archiveIdentity,
|
|
1424
1442
|
downloadIdentity: identity,
|
|
1425
1443
|
archiveCacheBypassed,
|
|
@@ -1432,7 +1450,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1432
1450
|
path: normalizeResultPath(entry.filename),
|
|
1433
1451
|
contentLength: entry.uncompressedSize,
|
|
1434
1452
|
read: () => {
|
|
1435
|
-
entryRead = readRangeEntry(entry,
|
|
1453
|
+
entryRead = readRangeEntry(entry, rangeSignal, producers);
|
|
1436
1454
|
return entryRead.body;
|
|
1437
1455
|
},
|
|
1438
1456
|
waitForRead: () => entryRead?.completed ?? Promise.resolve()
|
|
@@ -1445,6 +1463,7 @@ async function openArchiveTransport(params, previous, prepared) {
|
|
|
1445
1463
|
}
|
|
1446
1464
|
};
|
|
1447
1465
|
} catch (error) {
|
|
1466
|
+
rangeController.abort(error);
|
|
1448
1467
|
await reader.close();
|
|
1449
1468
|
throw error;
|
|
1450
1469
|
}
|
|
@@ -1456,10 +1475,16 @@ function readRangeEntry(entry, signal, producers) {
|
|
|
1456
1475
|
streamController = controller;
|
|
1457
1476
|
}
|
|
1458
1477
|
});
|
|
1478
|
+
const cancelOutput = () => {
|
|
1479
|
+
streamController?.error(signal?.reason);
|
|
1480
|
+
};
|
|
1481
|
+
signal?.addEventListener("abort", cancelOutput, { once: true });
|
|
1482
|
+
if (signal?.aborted) cancelOutput();
|
|
1459
1483
|
const completed = entry.getData(stream.writable, { signal, checkCrc32: true, useWebWorkers: false }).then(() => void 0);
|
|
1460
1484
|
const producer = completed.catch((error) => {
|
|
1461
1485
|
streamController?.error(error);
|
|
1462
1486
|
}).finally(() => {
|
|
1487
|
+
signal?.removeEventListener("abort", cancelOutput);
|
|
1463
1488
|
producers.delete(producer);
|
|
1464
1489
|
});
|
|
1465
1490
|
producers.add(producer);
|
|
@@ -1554,7 +1579,7 @@ function createDownloadIdentity(jobId, response) {
|
|
|
1554
1579
|
])
|
|
1555
1580
|
).digest("hex");
|
|
1556
1581
|
}
|
|
1557
|
-
var import_crypto2, import_zip2;
|
|
1582
|
+
var import_crypto2, import_zip2, AbortableHttpRangeReader;
|
|
1558
1583
|
var init_streamed_parsed_document_sync = __esm({
|
|
1559
1584
|
"src/storage/streamed-parsed-document-sync.ts"() {
|
|
1560
1585
|
"use strict";
|
|
@@ -1564,6 +1589,50 @@ var init_streamed_parsed_document_sync = __esm({
|
|
|
1564
1589
|
init_parsed_document_archive_cache();
|
|
1565
1590
|
import_zip2 = require("@zip.js/zip.js");
|
|
1566
1591
|
init_parsed_document_storage();
|
|
1592
|
+
AbortableHttpRangeReader = class extends import_zip2.HttpRangeReader {
|
|
1593
|
+
constructor(url, options, controller) {
|
|
1594
|
+
super(url, options);
|
|
1595
|
+
this.controller = controller;
|
|
1596
|
+
}
|
|
1597
|
+
controller;
|
|
1598
|
+
createReadable(options) {
|
|
1599
|
+
const reader = super.createReadable(options).getReader();
|
|
1600
|
+
const cancellation = this.controller;
|
|
1601
|
+
return new ReadableStream(
|
|
1602
|
+
{
|
|
1603
|
+
async pull(output) {
|
|
1604
|
+
try {
|
|
1605
|
+
const chunk = await reader.read();
|
|
1606
|
+
if (chunk.done) {
|
|
1607
|
+
reader.releaseLock();
|
|
1608
|
+
output.close();
|
|
1609
|
+
} else output.enqueue(chunk.value);
|
|
1610
|
+
} catch (error) {
|
|
1611
|
+
cancellation.abort(error);
|
|
1612
|
+
reader.releaseLock();
|
|
1613
|
+
output.error(error);
|
|
1614
|
+
}
|
|
1615
|
+
},
|
|
1616
|
+
async cancel(reason) {
|
|
1617
|
+
try {
|
|
1618
|
+
await reader.cancel(reason);
|
|
1619
|
+
} finally {
|
|
1620
|
+
reader.releaseLock();
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
},
|
|
1624
|
+
{ highWaterMark: 0 }
|
|
1625
|
+
);
|
|
1626
|
+
}
|
|
1627
|
+
async readUint8Array(index, length) {
|
|
1628
|
+
try {
|
|
1629
|
+
return await super.readUint8Array(index, length);
|
|
1630
|
+
} catch (error) {
|
|
1631
|
+
this.controller.abort(error);
|
|
1632
|
+
throw error;
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
};
|
|
1567
1636
|
}
|
|
1568
1637
|
});
|
|
1569
1638
|
|
package/dist/index.mjs
CHANGED