@delali/narsil 0.3.1-next.145 → 0.3.1-next.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/analysis-change-ZCJNYBJM.mjs +39 -0
- package/dist/{analysis-change-entry-V4SQQSXQ.mjs → analysis-change-entry-YN3ELL6C.mjs} +1 -1
- package/dist/browser/{analysis-change-QTCQ5ETU.mjs → analysis-change-M65AZQQG.mjs} +8 -5
- package/dist/browser/{analysis-change-entry-KYEQSP3I.mjs → analysis-change-entry-H3YIOLXZ.mjs} +1 -1
- package/dist/browser/{chunk-GP4QLUQM.mjs → chunk-2JABMB4S.mjs} +10 -6
- package/dist/browser/{chunk-5WYWLTM5.mjs → chunk-4DYLCICG.mjs} +2 -2
- package/dist/browser/{chunk-75PMIFQA.mjs → chunk-4RD3AFSB.mjs} +1 -1
- package/dist/browser/{chunk-XWMEGKDC.mjs → chunk-5V74FFZE.mjs} +2 -2
- package/dist/browser/{chunk-PMBSBSCN.mjs → chunk-6OWX3OP2.mjs} +2 -2
- package/dist/browser/chunk-BRWY3JCN.mjs +678 -0
- package/dist/browser/{chunk-ZALEKEUH.mjs → chunk-BVX2MEHA.mjs} +306 -607
- package/dist/browser/{chunk-FXPMF65P.mjs → chunk-D6UFGBF5.mjs} +15 -45
- package/dist/browser/{chunk-QNDWMPV2.mjs → chunk-IAXZAVYL.mjs} +25 -66
- package/dist/browser/chunk-JNOAKDUM.mjs +31 -0
- package/dist/browser/{chunk-BTMRV5MY.mjs → chunk-KAEVGLYN.mjs} +2 -2
- package/dist/browser/{chunk-FHO5N7FD.mjs → chunk-LURXVCHG.mjs} +1 -1
- package/dist/browser/chunk-MPK76GVS.mjs +63 -0
- package/dist/browser/{chunk-6OFS7RUN.mjs → chunk-NVEJJP4R.mjs} +1 -1
- package/dist/browser/{chunk-U77XQJOY.mjs → chunk-ODQP7P3H.mjs} +455 -66
- package/dist/browser/{chunk-IFGTRRPU.mjs → chunk-PIS7ZAP7.mjs} +3 -25
- package/dist/browser/{chunk-XIHLIMFX.mjs → chunk-QMNZFFD3.mjs} +4 -3
- package/dist/browser/{chunk-AOKLYVTT.mjs → chunk-SRL47USS.mjs} +2 -2
- package/dist/browser/chunk-TY7L6OXO.mjs +69 -0
- package/dist/browser/{chunk-LGJ6XMH5.mjs → chunk-W643QV3L.mjs} +6 -5
- package/dist/browser/chunk-WXLBZJMT.mjs +25 -0
- package/dist/browser/{chunk-ZKVDQ5ZH.mjs → chunk-XILIT5MT.mjs} +8 -8
- package/dist/browser/{cold-module-Y2IJAP7S.mjs → cold-module-A5S6JFDN.mjs} +20 -20
- package/dist/browser/{durability-wiring-MLXZ5QQF.mjs → durability-wiring-W5AVV3RF.mjs} +53 -30
- package/dist/browser/gzip-5S7ROJFJ.mjs +1 -0
- package/dist/browser/import-PTT63AGK.mjs +12 -0
- package/dist/browser/{module-5LMDGXQH.mjs → module-CEKA7MQH.mjs} +13 -10
- package/dist/browser/module-X5PGOE47.mjs +4288 -0
- package/dist/browser/{phrase-module-KXNRTWCS.mjs → phrase-module-CNKMUY4U.mjs} +2 -1
- package/dist/browser/pool-DTYSPBMQ.mjs +296 -0
- package/dist/{restore-queue-UAVGBSJG.mjs → browser/restore-queue-DD4LWZK7.mjs} +2 -2
- package/dist/browser/{runtime-ZNEGK33G.mjs → runtime-D2ULZGCN.mjs} +3 -2
- package/dist/browser/{snapshot-2EGS2FYU.mjs → snapshot-DQ6A4PQP.mjs} +18 -14
- package/dist/browser/{write-WMZA6NSF.mjs → write-MAGRAIEH.mjs} +14 -11
- package/dist/{capabilities-CNnzGtqP.d.ts → capabilities-BT-HLDC8.d.ts} +2 -2
- package/dist/{chunk-FRJ5RGX7.mjs → chunk-2HJWQPR6.mjs} +2 -2
- package/dist/{chunk-DKMZ7JFW.mjs → chunk-3BVGJ4JR.mjs} +10 -4
- package/dist/{chunk-QYMAYU7M.mjs → chunk-3HV7M5OJ.mjs} +155 -149
- package/dist/chunk-3MXV3VHR.mjs +46 -0
- package/dist/{chunk-6FOJKNUR.mjs → chunk-3SBJVQLL.mjs} +2 -2
- package/dist/{chunk-PRID7CA6.mjs → chunk-3TVDTACC.mjs} +1 -1
- package/dist/{chunk-RTVAWNUL.mjs → chunk-5WB2A3XU.mjs} +9 -6
- package/dist/{chunk-LWFK3TC2.mjs → chunk-65WCV5JO.mjs} +1 -1
- package/dist/{chunk-G6Q5DEF5.mjs → chunk-6BWLDLS3.mjs} +3 -46
- package/dist/{chunk-W6M4RRAT.mjs → chunk-6C5EX2QR.mjs} +3 -3
- package/dist/{chunk-HOC5JP2H.mjs → chunk-6F6USYII.mjs} +119 -11
- package/dist/{chunk-7523VCM4.mjs → chunk-7HKWO4BJ.mjs} +7 -7
- package/dist/{chunk-ONDEEBTA.mjs → chunk-7N223GWV.mjs} +2 -2
- package/dist/{chunk-UGQULMSU.mjs → chunk-ADBS5XRI.mjs} +10 -10
- package/dist/{chunk-MJTEU2XK.mjs → chunk-AWNHDBKV.mjs} +429 -73
- package/dist/{chunk-2E2ZFOPT.mjs → chunk-B2FTZNNF.mjs} +2 -2
- package/dist/{chunk-NWTEJ2T7.mjs → chunk-B7XIOAMK.mjs} +3 -3
- package/dist/{chunk-W55X2BYN.mjs → chunk-BEZH7VEH.mjs} +1 -1
- package/dist/chunk-BTPGQC6Z.mjs +27 -0
- package/dist/{chunk-KG5N7LE6.mjs → chunk-CYYSZPVV.mjs} +11 -11
- package/dist/{chunk-HTMUGEFU.mjs → chunk-DOY6EG76.mjs} +6 -6
- package/dist/{chunk-UHVTEVF5.mjs → chunk-F22IOCR2.mjs} +10 -6
- package/dist/{chunk-FTQLF5JI.mjs → chunk-IRWP524D.mjs} +1 -1
- package/dist/{chunk-XLAI4OIO.mjs → chunk-IZT3G24Q.mjs} +2 -2
- package/dist/{chunk-SE7MKJVK.mjs → chunk-J7M73VY7.mjs} +1 -1
- package/dist/chunk-KAKQ74LS.mjs +53 -0
- package/dist/{chunk-GVPYMSLO.mjs → chunk-KYVRZAYP.mjs} +1 -1
- package/dist/{chunk-F442VZUL.mjs → chunk-KZ5GQE4Z.mjs} +5 -5
- package/dist/{chunk-WDEWU46C.mjs → chunk-KZQIVGW7.mjs} +2 -2
- package/dist/{chunk-7AJAW6NN.mjs → chunk-L7TIAYPX.mjs} +8 -8
- package/dist/{chunk-GRFXSEFF.mjs → chunk-LDGGR7YT.mjs} +4 -81
- package/dist/{chunk-WMIV7PJ7.mjs → chunk-LJ4ENXOD.mjs} +15 -14
- package/dist/chunk-LQG5G2LY.mjs +39 -0
- package/dist/{chunk-IQHZMCLX.mjs → chunk-MDF6PDIA.mjs} +2 -0
- package/dist/{chunk-QXH2CS4V.mjs → chunk-NVTFQFUL.mjs} +7 -26
- package/dist/{chunk-5K2JHKM3.mjs → chunk-NYSONUB7.mjs} +1 -1
- package/dist/{chunk-OH7SX5MY.mjs → chunk-O4KUNDVF.mjs} +12 -10
- package/dist/{chunk-EBIHRL7B.mjs → chunk-Q2UL2TPC.mjs} +1 -1
- package/dist/{chunk-RNUP6WZ6.mjs → chunk-QEYF2SQ6.mjs} +2 -2
- package/dist/{chunk-RVBUQITB.mjs → chunk-QN5WVV5K.mjs} +7 -7
- package/dist/{chunk-RIIUZJQ3.mjs → chunk-QYRNLLLI.mjs} +1 -1
- package/dist/{chunk-BTARFTEN.mjs → chunk-SOUZMPME.mjs} +4 -4
- package/dist/{chunk-ISUOSACK.mjs → chunk-T4CJDBBE.mjs} +1 -1
- package/dist/{chunk-DHEJAGVW.mjs → chunk-TTCKT2RT.mjs} +5 -5
- package/dist/{chunk-QU2XF7VI.mjs → chunk-U2726TL5.mjs} +5 -4
- package/dist/{chunk-LYKUJNAV.mjs → chunk-U46IZNCW.mjs} +2 -2
- package/dist/{chunk-SNB4DYL3.mjs → chunk-U6NE6RCV.mjs} +4 -4
- package/dist/{chunk-SA52N7G3.mjs → chunk-VP6XRMWR.mjs} +4 -4
- package/dist/{chunk-YUCGFPMS.mjs → chunk-VQDZVWN3.mjs} +449 -405
- package/dist/{chunk-IWVEG5CX.mjs → chunk-XIHUI4KB.mjs} +2 -2
- package/dist/{chunk-IV2XTSS2.mjs → chunk-XKPNKR3V.mjs} +16 -46
- package/dist/{chunk-UQS5PAS5.mjs → chunk-YGQOPB2Z.mjs} +1 -1
- package/dist/{chunk-SVO6NV4D.mjs → chunk-YKEHQUSZ.mjs} +4 -4
- package/dist/{chunk-6Z22L5RN.mjs → chunk-ZAYWOHVG.mjs} +1 -1
- package/dist/{chunk-QLDLQW36.mjs → chunk-ZEKDO7YX.mjs} +2 -2
- package/dist/chunk-ZXNNR73R.mjs +31 -0
- package/dist/{chunk-VJS4JFBQ.mjs → chunk-ZXX5GZEX.mjs} +3 -1
- package/dist/{chunk-X5SBWARI.mjs → chunk-ZYKUD5NX.mjs} +9 -7
- package/dist/{client-DoYPHcUn.d.ts → client-Cn3gNxUU.d.ts} +3 -3
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +2 -0
- package/dist/{cold-module-NUD332YD.mjs → cold-module-2WWGBFJC.mjs} +36 -33
- package/dist/{config-CuUfls_H.d.ts → config-CfhH1IWQ.d.ts} +1 -1
- package/dist/{config-Dzs46im6.d.ts → config-aIsy7mcL.d.ts} +52 -1
- package/dist/direct-executor-QLVHMELJ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +150 -255
- package/dist/{durability-wiring-K7RDXWL6.mjs → durability-wiring-GNWQSZI3.mjs} +67 -42
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-C0ifUEcm.d.ts → engine-CJRUHXgf.d.ts} +1 -1
- package/dist/gzip-F4HMIEEQ.mjs +1 -0
- package/dist/import-DX55PN4G.mjs +41 -0
- package/dist/index.browser.mjs +7 -4
- package/dist/index.d.ts +539 -5
- package/dist/index.mjs +22 -16
- package/dist/index.react-native.mjs +19 -0
- package/dist/{long-query-thread-FKVHRCHF.mjs → long-query-thread-SCUE62TO.mjs} +20 -17
- package/dist/{module-GJUKW5VU.mjs → module-HW6IXUM6.mjs} +601 -108
- package/dist/{module-TMFIWOD2.mjs → module-V5K4ZP23.mjs} +29 -23
- package/dist/{paged-partition-archive-WNUEMYWM.mjs → paged-partition-archive-FDIIWZNR.mjs} +31 -25
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -22
- package/dist/{phrase-module-OMY3QQV7.mjs → phrase-module-OUQIRUQU.mjs} +1 -1
- package/dist/pool-UPV5PVPY.mjs +298 -0
- package/dist/react-native/chunks/analysis-change-HGW3L6NY.mjs +893 -0
- package/dist/react-native/chunks/analysis-change-entry-BNRU6STO.mjs +1 -0
- package/dist/react-native/chunks/chunk-2J2727V4.mjs +495 -0
- package/dist/react-native/chunks/chunk-3CTWSQRW.mjs +1251 -0
- package/dist/react-native/chunks/chunk-5FV4ZEVA.mjs +2619 -0
- package/dist/react-native/chunks/chunk-7VWQLEFV.mjs +642 -0
- package/dist/react-native/chunks/chunk-AAAETEJK.mjs +27 -0
- package/dist/react-native/chunks/chunk-AM7RSBPF.mjs +32 -0
- package/dist/react-native/chunks/chunk-B6OD63Z3.mjs +997 -0
- package/dist/react-native/chunks/chunk-BRNAVOKP.mjs +177 -0
- package/dist/react-native/chunks/chunk-DKJTZ625.mjs +207 -0
- package/dist/react-native/chunks/chunk-FKGG4UOF.mjs +177 -0
- package/dist/{browser/chunk-K7UFXEJS.mjs → react-native/chunks/chunk-GV5KFQ6M.mjs} +12 -6
- package/dist/react-native/chunks/chunk-HURV3ASY.mjs +4416 -0
- package/dist/react-native/chunks/chunk-IO2SMUQD.mjs +10 -0
- package/dist/react-native/chunks/chunk-JV6VQY2S.mjs +17 -0
- package/dist/react-native/chunks/chunk-KUJOBRKS.mjs +217 -0
- package/dist/react-native/chunks/chunk-KWRW5U65.mjs +226 -0
- package/dist/react-native/chunks/chunk-L4M7QJNP.mjs +110 -0
- package/dist/react-native/chunks/chunk-LQ6Q4L5B.mjs +360 -0
- package/dist/react-native/chunks/chunk-LQAKO5QV.mjs +523 -0
- package/dist/react-native/chunks/chunk-LRGWK367.mjs +11 -0
- package/dist/react-native/chunks/chunk-LVIKKAA6.mjs +93 -0
- package/dist/react-native/chunks/chunk-NFGBOI46.mjs +14 -0
- package/dist/react-native/chunks/chunk-NNE4VDMM.mjs +8 -0
- package/dist/react-native/chunks/chunk-PPEPTUVV.mjs +56 -0
- package/dist/react-native/chunks/chunk-PSWKQVVT.mjs +9 -0
- package/dist/react-native/chunks/chunk-PU55IOG4.mjs +910 -0
- package/dist/react-native/chunks/chunk-QC4ZRBSX.mjs +270 -0
- package/dist/react-native/chunks/chunk-R4EEMYM6.mjs +21 -0
- package/dist/react-native/chunks/chunk-R4OPUZQ5.mjs +86 -0
- package/dist/react-native/chunks/chunk-SLGIAD6B.mjs +2634 -0
- package/dist/react-native/chunks/chunk-TJMSKBEM.mjs +25 -0
- package/dist/react-native/chunks/chunk-TLYD7IJ2.mjs +23 -0
- package/dist/react-native/chunks/chunk-UFNCLEQM.mjs +73 -0
- package/dist/react-native/chunks/chunk-YH7W556M.mjs +179 -0
- package/dist/react-native/chunks/chunk-YJSMEKRE.mjs +22385 -0
- package/dist/react-native/chunks/chunk-ZCKNBPQ4.mjs +79 -0
- package/dist/react-native/chunks/cold-module-TSFWLYWC.mjs +755 -0
- package/dist/react-native/chunks/durability-wiring-PZR6KOQK.mjs +4509 -0
- package/dist/react-native/chunks/gzip.react-native-7JWCLVOB.mjs +3 -0
- package/dist/react-native/chunks/import-VTODLPLO.mjs +22 -0
- package/dist/{browser/module-Q6VFGLMG.mjs → react-native/chunks/module-JBBH6PZH.mjs} +609 -144
- package/dist/react-native/chunks/module-VEK2KCKU.mjs +25 -0
- package/dist/react-native/chunks/phrase-module-76UAPPSG.mjs +329 -0
- package/dist/react-native/chunks/pool-CD4ANYUI.mjs +296 -0
- package/dist/react-native/chunks/release-D7ZTIANM.mjs +33 -0
- package/dist/react-native/chunks/replace-IMZUVUOE.mjs +57 -0
- package/dist/{browser/restore-queue-SOAHKZSQ.mjs → react-native/chunks/restore-queue-OI5EH7R3.mjs} +2 -2
- package/dist/react-native/chunks/runtime-3MCDLJAF.mjs +559 -0
- package/dist/react-native/chunks/snapshot-AUZMEV36.mjs +1205 -0
- package/dist/react-native/chunks/write-DDC2QBRJ.mjs +197 -0
- package/dist/react-native/partial-words.mjs +733 -0
- package/dist/react-native/pattern.mjs +2442 -0
- package/dist/react-native/synonyms.mjs +460 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-ZZ5CHKLW.mjs → rerank-3W54N6P2.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/restore-queue-BQWWNNQI.mjs +41 -0
- package/dist/{runtime-RAOCKSYA.mjs → runtime-L2DZYY6N.mjs} +5 -5
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +32 -28
- package/dist/{snapshot-DB5ZVCDN.mjs → snapshot-EHXTOTW5.mjs} +39 -33
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-ChWp0wrF.d.ts → tasks-C4ElXr4P.d.ts} +2 -2
- package/dist/thread-I7DVHHW7.mjs +31 -0
- package/dist/vector/search-worker.mjs +7 -5
- package/dist/workers/entry.mjs +9 -9
- package/dist/{write-KZVTLGCI.mjs → write-SDJXXQ2G.mjs} +29 -23
- package/package.json +88 -7
- package/dist/analysis-change-D5APH4Z3.mjs +0 -34
- package/dist/browser/import-FTMLTPGQ.mjs +0 -9
- package/dist/direct-executor-FDQYQEJQ.mjs +0 -22
- package/dist/import-6KXFY6XI.mjs +0 -35
- package/dist/thread-OV6T555H.mjs +0 -28
- /package/dist/{chunk-22YPZVMG.mjs → chunk-ZXNQEW43.mjs} +0 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-KUJOBRKS.mjs';
|
|
2
|
+
import './chunk-QC4ZRBSX.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LQ6Q4L5B.mjs';
|
|
4
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-R4OPUZQ5.mjs';
|
|
5
|
+
import './chunk-BRNAVOKP.mjs';
|
|
6
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-7VWQLEFV.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-YJSMEKRE.mjs';
|
|
8
|
+
import './chunk-LQAKO5QV.mjs';
|
|
9
|
+
import './chunk-NNE4VDMM.mjs';
|
|
10
|
+
import './chunk-L4M7QJNP.mjs';
|
|
11
|
+
import './chunk-IO2SMUQD.mjs';
|
|
12
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-AM7RSBPF.mjs';
|
|
13
|
+
import './chunk-ZCKNBPQ4.mjs';
|
|
14
|
+
import './chunk-PU55IOG4.mjs';
|
|
15
|
+
import './chunk-TJMSKBEM.mjs';
|
|
16
|
+
import './chunk-DKJTZ625.mjs';
|
|
17
|
+
import './chunk-3CTWSQRW.mjs';
|
|
18
|
+
import './chunk-2J2727V4.mjs';
|
|
19
|
+
import './chunk-KWRW5U65.mjs';
|
|
20
|
+
import './chunk-PPEPTUVV.mjs';
|
|
21
|
+
import './chunk-LVIKKAA6.mjs';
|
|
22
|
+
import './chunk-AAAETEJK.mjs';
|
|
23
|
+
import './chunk-JV6VQY2S.mjs';
|
|
24
|
+
import './chunk-NFGBOI46.mjs';
|
|
25
|
+
import './chunk-YH7W556M.mjs';
|
|
26
|
+
import { decode } from '@msgpack/msgpack';
|
|
27
|
+
|
|
28
|
+
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
29
|
+
var FLOAT32_BYTES = 4;
|
|
30
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
31
|
+
async function locateVectors(checkpoint) {
|
|
32
|
+
const located = /* @__PURE__ */ new Map();
|
|
33
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
34
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
35
|
+
for (const ref of field.files) {
|
|
36
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
37
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
38
|
+
what: `vector file "${ref.key}"`,
|
|
39
|
+
allowsCompression: false
|
|
40
|
+
});
|
|
41
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
42
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
43
|
+
if (isDead(ref.dead, position)) continue;
|
|
44
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
located.set(field.fieldPath, byDocId);
|
|
48
|
+
}
|
|
49
|
+
return located;
|
|
50
|
+
}
|
|
51
|
+
async function readVector(checkpoint, location) {
|
|
52
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
53
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
54
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
55
|
+
}
|
|
56
|
+
async function readSegment(checkpoint, ref) {
|
|
57
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
58
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
59
|
+
what: `segment "${ref.key}"`,
|
|
60
|
+
allowsCompression: true
|
|
61
|
+
});
|
|
62
|
+
return decodeKeywordSegment(payload, false);
|
|
63
|
+
}
|
|
64
|
+
var decoder = new TextDecoder();
|
|
65
|
+
function idAt(contents, ordinal) {
|
|
66
|
+
const ids = contents.arrays.idTable;
|
|
67
|
+
return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
68
|
+
}
|
|
69
|
+
async function winnersOf(checkpoint, partition) {
|
|
70
|
+
const winners = /* @__PURE__ */ new Map();
|
|
71
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
72
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
73
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
74
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
75
|
+
}
|
|
76
|
+
return winners;
|
|
77
|
+
}
|
|
78
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
79
|
+
const contents = await readSegment(checkpoint, ref);
|
|
80
|
+
const table = contents.arrays.documentTable;
|
|
81
|
+
const ids = /* @__PURE__ */ new Set();
|
|
82
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
83
|
+
return {
|
|
84
|
+
count: contents.arrays.documentCount,
|
|
85
|
+
schema: contents.schema,
|
|
86
|
+
tombstones: new Set(contents.tombstones),
|
|
87
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
88
|
+
holds: (docId) => ids.has(docId),
|
|
89
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
90
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
91
|
+
table.discardedFields
|
|
92
|
+
),
|
|
93
|
+
close: () => void 0
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
97
|
+
const located = await locateVectors(checkpoint);
|
|
98
|
+
return {
|
|
99
|
+
async vectorOf(fieldPath, docId) {
|
|
100
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
101
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
102
|
+
},
|
|
103
|
+
close: () => located.clear()
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
107
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-VEK2KCKU.mjs');
|
|
108
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
109
|
+
try {
|
|
110
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
111
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
112
|
+
}
|
|
113
|
+
} finally {
|
|
114
|
+
vectors.close();
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
118
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
119
|
+
for (const ref of partition.segments) {
|
|
120
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
121
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
async function* documentLines(checkpoint) {
|
|
127
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
128
|
+
yield* pagedDocumentLines(checkpoint);
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
const vectors = await locateVectors(checkpoint);
|
|
132
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
133
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
134
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
135
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
136
|
+
const table = contents.arrays.documentTable;
|
|
137
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
138
|
+
const docId = idAt(contents, ordinal);
|
|
139
|
+
if (winners.get(docId) !== at) continue;
|
|
140
|
+
const stored = decodeStoredDocument(
|
|
141
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
142
|
+
table.discardedFields
|
|
143
|
+
);
|
|
144
|
+
const document = { id: docId, ...stored };
|
|
145
|
+
document.id = docId;
|
|
146
|
+
for (const [fieldPath, located] of vectors) {
|
|
147
|
+
const location = located.get(docId);
|
|
148
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
149
|
+
}
|
|
150
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
151
|
+
`;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
async function* exportLines(header, checkpoint) {
|
|
157
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
158
|
+
`;
|
|
159
|
+
yield* documentLines(checkpoint);
|
|
160
|
+
}
|
|
161
|
+
function documentExportStream(header, checkpoint) {
|
|
162
|
+
const lines = exportLines(header, checkpoint);
|
|
163
|
+
const encoder = new TextEncoder();
|
|
164
|
+
let released = false;
|
|
165
|
+
const release = () => {
|
|
166
|
+
if (released) return;
|
|
167
|
+
released = true;
|
|
168
|
+
checkpoint.release();
|
|
169
|
+
};
|
|
170
|
+
return new ReadableStream({
|
|
171
|
+
async pull(controller) {
|
|
172
|
+
try {
|
|
173
|
+
let text = "";
|
|
174
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
175
|
+
const step = await lines.next();
|
|
176
|
+
if (step.done) {
|
|
177
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
178
|
+
release();
|
|
179
|
+
controller.close();
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
text += step.value;
|
|
183
|
+
}
|
|
184
|
+
controller.enqueue(encoder.encode(text));
|
|
185
|
+
} catch (error) {
|
|
186
|
+
release();
|
|
187
|
+
controller.error(error);
|
|
188
|
+
}
|
|
189
|
+
},
|
|
190
|
+
async cancel() {
|
|
191
|
+
release();
|
|
192
|
+
await lines.return(void 0);
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export { documentExportStream };
|