@delali/narsil 0.3.1-next.143 → 0.3.1-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
- package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
- package/dist/browser/import-FTMLTPGQ.mjs +9 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
- package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
- package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
- package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
- package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
- package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
- package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
- package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
- package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
- package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
- package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
- package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
- package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
- package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
- package/dist/chunk-NWTEJ2T7.mjs +100 -0
- package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
- package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/chunk-PRID7CA6.mjs +104 -0
- package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
- package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
- package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
- package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
- package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
- package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
- package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
- package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
- package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
- package/dist/chunk-SVO6NV4D.mjs +248 -0
- package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
- package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
- package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
- package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
- package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
- package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
- package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +44 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
- package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
- package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16219 -14535
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
- package/dist/import-6KXFY6XI.mjs +35 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -30
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
- package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
- package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +9 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
- package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +153 -36
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
- package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +5 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
- package/openapi.json +349 -2
- package/package.json +27 -8
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-KGOXHLLZ.mjs +0 -693
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-QSK35UC2.mjs +0 -35
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { requireBucket, objectKeyPrefix, objectStoreFailure, requireObjectKey, refuseOversizedObject, requireStreamLength, countedChunks, readWhole, partsOf } from '../chunk-PRID7CA6.mjs';
|
|
2
|
+
import '../chunk-U476IXKQ.mjs';
|
|
3
|
+
import { OBJECT_STORE_LIST_PAGE_KEYS, OBJECT_STORE_CONFLICT_ATTEMPTS, OBJECT_STORE_CONFLICT_RETRY_MS, OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
|
|
4
|
+
import { ErrorCodes, NarsilError } from '../chunk-IQHZMCLX.mjs';
|
|
5
|
+
|
|
6
|
+
// src/persistence/s3.ts
|
|
7
|
+
var S3_NOT_FOUND_STATUS = 404;
|
|
8
|
+
var S3_PRECONDITION_FAILED_STATUS = 412;
|
|
9
|
+
var S3_CONFLICT_STATUS = 409;
|
|
10
|
+
async function loadS3Module() {
|
|
11
|
+
try {
|
|
12
|
+
return await import('@aws-sdk/client-s3');
|
|
13
|
+
} catch (err) {
|
|
14
|
+
throw new NarsilError(
|
|
15
|
+
ErrorCodes.PERSISTENCE_DEPENDENCY_MISSING,
|
|
16
|
+
"The `@aws-sdk/client-s3` package is not installed. Install it with `pnpm add @aws-sdk/client-s3` (or the npm or yarn equivalent) to use createS3Persistence.",
|
|
17
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function isNotFound(error) {
|
|
22
|
+
if (typeof error !== "object" || error === null) return false;
|
|
23
|
+
const named = error;
|
|
24
|
+
return named.name === "NoSuchKey" || named.name === "NotFound" || named.$metadata?.httpStatusCode === S3_NOT_FOUND_STATUS;
|
|
25
|
+
}
|
|
26
|
+
function statusCodeOf(error) {
|
|
27
|
+
if (typeof error !== "object" || error === null) return void 0;
|
|
28
|
+
return error.$metadata?.httpStatusCode;
|
|
29
|
+
}
|
|
30
|
+
function isPreconditionFailed(error) {
|
|
31
|
+
return statusCodeOf(error) === S3_PRECONDITION_FAILED_STATUS;
|
|
32
|
+
}
|
|
33
|
+
function discardBody(body) {
|
|
34
|
+
if (typeof body === "object" && body !== null && "destroy" in body && typeof body.destroy === "function") {
|
|
35
|
+
body.destroy();
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async function uploadParts(target, parts, first) {
|
|
39
|
+
const { sdk, client, bucket, objectKey } = target;
|
|
40
|
+
const created = await client.send(new sdk.CreateMultipartUploadCommand({ Bucket: bucket, Key: objectKey }));
|
|
41
|
+
const uploadId = created.UploadId;
|
|
42
|
+
if (uploadId === void 0) {
|
|
43
|
+
throw new NarsilError(
|
|
44
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
45
|
+
`The object store started a multipart upload of "${objectKey}" without an upload ID`,
|
|
46
|
+
{ key: objectKey }
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
const completed = [];
|
|
51
|
+
let part = first;
|
|
52
|
+
while (part !== void 0) {
|
|
53
|
+
const PartNumber = completed.length + 1;
|
|
54
|
+
const sent = await client.send(
|
|
55
|
+
new sdk.UploadPartCommand({
|
|
56
|
+
Bucket: bucket,
|
|
57
|
+
Key: objectKey,
|
|
58
|
+
UploadId: uploadId,
|
|
59
|
+
PartNumber,
|
|
60
|
+
Body: part,
|
|
61
|
+
ContentLength: part.byteLength
|
|
62
|
+
})
|
|
63
|
+
);
|
|
64
|
+
completed.push({ ETag: sent.ETag, PartNumber });
|
|
65
|
+
const next = await parts.next();
|
|
66
|
+
part = next.done === true ? void 0 : next.value;
|
|
67
|
+
}
|
|
68
|
+
await client.send(
|
|
69
|
+
new sdk.CompleteMultipartUploadCommand({
|
|
70
|
+
Bucket: bucket,
|
|
71
|
+
Key: objectKey,
|
|
72
|
+
UploadId: uploadId,
|
|
73
|
+
MultipartUpload: { Parts: completed }
|
|
74
|
+
})
|
|
75
|
+
);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
await client.send(new sdk.AbortMultipartUploadCommand({ Bucket: bucket, Key: objectKey, UploadId: uploadId })).catch(() => void 0);
|
|
78
|
+
throw error;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async function saveObjectStream(target, key, data, length) {
|
|
82
|
+
requireStreamLength(key, length);
|
|
83
|
+
const counted = countedChunks(key, data, length);
|
|
84
|
+
if (length <= OBJECT_STORE_PART_BYTES) {
|
|
85
|
+
const body = await readWhole(counted, length);
|
|
86
|
+
await target.client.send(
|
|
87
|
+
new target.sdk.PutObjectCommand({
|
|
88
|
+
Bucket: target.bucket,
|
|
89
|
+
Key: target.objectKey,
|
|
90
|
+
Body: body,
|
|
91
|
+
ContentLength: length
|
|
92
|
+
})
|
|
93
|
+
);
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
const parts = partsOf(counted, OBJECT_STORE_PART_BYTES)[Symbol.asyncIterator]();
|
|
97
|
+
const first = await parts.next();
|
|
98
|
+
if (first.done === true) {
|
|
99
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The stream for "${key}" ended before its first part`, {
|
|
100
|
+
key
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
await uploadParts(target, parts, first.value);
|
|
104
|
+
}
|
|
105
|
+
async function* bodyChunks(body) {
|
|
106
|
+
const reader = body.transformToWebStream().getReader();
|
|
107
|
+
try {
|
|
108
|
+
for (; ; ) {
|
|
109
|
+
const step = await reader.read();
|
|
110
|
+
if (step.done) return;
|
|
111
|
+
yield step.value;
|
|
112
|
+
}
|
|
113
|
+
} finally {
|
|
114
|
+
await reader.cancel().catch(() => void 0);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
async function* failingAs(key, chunks) {
|
|
118
|
+
try {
|
|
119
|
+
yield* chunks;
|
|
120
|
+
} catch (error) {
|
|
121
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async function createS3Persistence(config) {
|
|
125
|
+
const bucket = requireBucket(config.bucket, "S3");
|
|
126
|
+
const root = objectKeyPrefix(config.keyPrefix, "S3");
|
|
127
|
+
const sdk = await loadS3Module();
|
|
128
|
+
const client = new sdk.S3Client({
|
|
129
|
+
...config.region === void 0 ? {} : { region: config.region },
|
|
130
|
+
...config.endpoint === void 0 ? {} : { endpoint: config.endpoint },
|
|
131
|
+
...config.forcePathStyle === void 0 ? {} : { forcePathStyle: config.forcePathStyle },
|
|
132
|
+
...config.credentials === void 0 ? {} : { credentials: config.credentials },
|
|
133
|
+
...config.maxAttempts === void 0 ? {} : { maxAttempts: config.maxAttempts }
|
|
134
|
+
});
|
|
135
|
+
return {
|
|
136
|
+
async save(key, data) {
|
|
137
|
+
requireObjectKey(key);
|
|
138
|
+
try {
|
|
139
|
+
await client.send(
|
|
140
|
+
new sdk.PutObjectCommand({
|
|
141
|
+
Bucket: bucket,
|
|
142
|
+
Key: `${root}${key}`,
|
|
143
|
+
Body: data,
|
|
144
|
+
ContentLength: data.byteLength
|
|
145
|
+
})
|
|
146
|
+
);
|
|
147
|
+
} catch (error) {
|
|
148
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
async saveIfAbsent(key, data) {
|
|
152
|
+
requireObjectKey(key);
|
|
153
|
+
for (let attempt = 1; ; attempt++) {
|
|
154
|
+
try {
|
|
155
|
+
await client.send(
|
|
156
|
+
new sdk.PutObjectCommand({
|
|
157
|
+
Bucket: bucket,
|
|
158
|
+
Key: `${root}${key}`,
|
|
159
|
+
Body: data,
|
|
160
|
+
ContentLength: data.byteLength,
|
|
161
|
+
IfNoneMatch: "*"
|
|
162
|
+
})
|
|
163
|
+
);
|
|
164
|
+
return true;
|
|
165
|
+
} catch (error) {
|
|
166
|
+
if (isPreconditionFailed(error)) return false;
|
|
167
|
+
if (statusCodeOf(error) !== S3_CONFLICT_STATUS || attempt >= OBJECT_STORE_CONFLICT_ATTEMPTS) {
|
|
168
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
await new Promise((resolve) => setTimeout(resolve, OBJECT_STORE_CONFLICT_RETRY_MS * attempt));
|
|
172
|
+
}
|
|
173
|
+
},
|
|
174
|
+
async load(key) {
|
|
175
|
+
requireObjectKey(key);
|
|
176
|
+
try {
|
|
177
|
+
const object = await client.send(new sdk.GetObjectCommand({ Bucket: bucket, Key: `${root}${key}` }));
|
|
178
|
+
try {
|
|
179
|
+
refuseOversizedObject(key, object.ContentLength ?? 0);
|
|
180
|
+
} catch (error) {
|
|
181
|
+
discardBody(object.Body);
|
|
182
|
+
throw error;
|
|
183
|
+
}
|
|
184
|
+
if (object.Body === void 0) return new Uint8Array(0);
|
|
185
|
+
const bytes = await object.Body.transformToByteArray();
|
|
186
|
+
refuseOversizedObject(key, bytes.byteLength);
|
|
187
|
+
return bytes;
|
|
188
|
+
} catch (error) {
|
|
189
|
+
if (isNotFound(error)) return null;
|
|
190
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async saveStream(key, data, length) {
|
|
194
|
+
requireObjectKey(key);
|
|
195
|
+
try {
|
|
196
|
+
await saveObjectStream({ sdk, client, bucket, objectKey: `${root}${key}` }, key, data, length);
|
|
197
|
+
} catch (error) {
|
|
198
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
async loadStream(key) {
|
|
202
|
+
requireObjectKey(key);
|
|
203
|
+
try {
|
|
204
|
+
const object = await client.send(new sdk.GetObjectCommand({ Bucket: bucket, Key: `${root}${key}` }));
|
|
205
|
+
try {
|
|
206
|
+
refuseOversizedObject(key, object.ContentLength ?? 0);
|
|
207
|
+
} catch (error) {
|
|
208
|
+
discardBody(object.Body);
|
|
209
|
+
throw error;
|
|
210
|
+
}
|
|
211
|
+
const body = object.Body;
|
|
212
|
+
if (body === void 0) return failingAs(key, []);
|
|
213
|
+
return failingAs(key, bodyChunks(body));
|
|
214
|
+
} catch (error) {
|
|
215
|
+
if (isNotFound(error)) return null;
|
|
216
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
async delete(key) {
|
|
220
|
+
requireObjectKey(key);
|
|
221
|
+
const objectKey = `${root}${key}`;
|
|
222
|
+
try {
|
|
223
|
+
const head = await client.send(new sdk.HeadObjectCommand({ Bucket: bucket, Key: objectKey }));
|
|
224
|
+
if (head.ETag === void 0) {
|
|
225
|
+
throw new NarsilError(
|
|
226
|
+
ErrorCodes.PERSISTENCE_DELETE_FAILED,
|
|
227
|
+
`The store reported no ETag for "${key}", so the adapter deletes nothing`,
|
|
228
|
+
{ key }
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
await client.send(new sdk.DeleteObjectCommand({ Bucket: bucket, Key: objectKey, IfMatch: head.ETag }));
|
|
232
|
+
} catch (error) {
|
|
233
|
+
if (isNotFound(error) || isPreconditionFailed(error)) return;
|
|
234
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key, error);
|
|
235
|
+
}
|
|
236
|
+
},
|
|
237
|
+
async list(prefix) {
|
|
238
|
+
const keys = [];
|
|
239
|
+
let continuationToken;
|
|
240
|
+
try {
|
|
241
|
+
do {
|
|
242
|
+
const page = await client.send(
|
|
243
|
+
new sdk.ListObjectsV2Command({
|
|
244
|
+
Bucket: bucket,
|
|
245
|
+
Prefix: `${root}${prefix}`,
|
|
246
|
+
MaxKeys: OBJECT_STORE_LIST_PAGE_KEYS,
|
|
247
|
+
...continuationToken === void 0 ? {} : { ContinuationToken: continuationToken }
|
|
248
|
+
})
|
|
249
|
+
);
|
|
250
|
+
for (const object of page.Contents ?? []) {
|
|
251
|
+
if (object.Key?.startsWith(root) === true) keys.push(object.Key.slice(root.length));
|
|
252
|
+
}
|
|
253
|
+
continuationToken = page.IsTruncated === true ? page.NextContinuationToken : void 0;
|
|
254
|
+
} while (continuationToken !== void 0);
|
|
255
|
+
} catch (error) {
|
|
256
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "list the keys under", prefix, error);
|
|
257
|
+
}
|
|
258
|
+
return keys;
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export { createS3Persistence };
|
|
@@ -36,6 +36,39 @@ interface PersistenceAdapter {
|
|
|
36
36
|
* @returns Every matching key, in any order.
|
|
37
37
|
*/
|
|
38
38
|
list(prefix: string): Promise<string[]>;
|
|
39
|
+
/**
|
|
40
|
+
* Writes the bytes that `data` yields under `key` without holding the whole file in memory, replacing whatever the
|
|
41
|
+
* key held. A failed or short stream leaves the previous value in place. Offer it for a store that holds large
|
|
42
|
+
* files, because a cluster backup streams each file through it.
|
|
43
|
+
*
|
|
44
|
+
* @param key - The key that a later {@link PersistenceAdapter.load} reads.
|
|
45
|
+
* @param data - The bytes to store, in order.
|
|
46
|
+
* @param length - The number of bytes that `data` yields.
|
|
47
|
+
* @throws A `NarsilError` with `PERSISTENCE_SAVE_FAILED` when `data` yields more or fewer than `length` bytes, or
|
|
48
|
+
* when the store refuses the write.
|
|
49
|
+
*/
|
|
50
|
+
saveStream?(key: string, data: AsyncIterable<Uint8Array>, length: number): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Reads the bytes stored under `key` in order, without holding the whole file in memory.
|
|
53
|
+
*
|
|
54
|
+
* @param key - The key that a matching save wrote.
|
|
55
|
+
* @returns The stored bytes in chunks, or `null` when the key holds nothing.
|
|
56
|
+
* @throws A `NarsilError` with `PERSISTENCE_LOAD_FAILED` when the store fails the read.
|
|
57
|
+
*/
|
|
58
|
+
loadStream?(key: string): Promise<AsyncIterable<Uint8Array> | null>;
|
|
59
|
+
/**
|
|
60
|
+
* Stores `data` under `key` only where the key holds nothing, as one atomic step, so that where several calls race on
|
|
61
|
+
* an absent key, exactly one returns `true`. A cluster backup store must offer it, because the cluster claims its key
|
|
62
|
+
* prefix with it.
|
|
63
|
+
*
|
|
64
|
+
* @param key - The key to fill.
|
|
65
|
+
* @param data - The bytes to store.
|
|
66
|
+
* @returns `true` where this call stored `data`, and `false` where the key already held a value. A retry after a lost
|
|
67
|
+
* response can return `false` for bytes that the earlier attempt stored, so load the key and compare where you need to
|
|
68
|
+
* know whose bytes it holds.
|
|
69
|
+
* @throws A `NarsilError` with `PERSISTENCE_SAVE_FAILED` when the store refuses the write.
|
|
70
|
+
*/
|
|
71
|
+
saveIfAbsent?(key: string, data: Uint8Array): Promise<boolean>;
|
|
39
72
|
/** This adapter writes to this directory, when a filesystem backs it. */
|
|
40
73
|
readonly directory?: string;
|
|
41
74
|
}
|
|
@@ -1,43 +1,34 @@
|
|
|
1
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
1
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-HTMUGEFU.mjs';
|
|
2
|
+
import './chunk-SA52N7G3.mjs';
|
|
3
|
+
import './chunk-RIIUZJQ3.mjs';
|
|
4
|
+
import './chunk-YUCGFPMS.mjs';
|
|
5
|
+
import './chunk-HOC5JP2H.mjs';
|
|
6
6
|
import './chunk-HJSN7UYG.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-AV4IJKD2.mjs';
|
|
7
|
+
import './chunk-G6Q5DEF5.mjs';
|
|
9
8
|
import './chunk-EEBCDP36.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-N37VMC62.mjs';
|
|
13
|
-
import './chunk-ZATRDAYU.mjs';
|
|
9
|
+
import './chunk-OH7SX5MY.mjs';
|
|
10
|
+
import './chunk-QYMAYU7M.mjs';
|
|
14
11
|
import './chunk-EBIHRL7B.mjs';
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-QSK35UC2.mjs';
|
|
17
|
-
import './chunk-74XB2SXI.mjs';
|
|
18
|
-
import './chunk-RKAVHNRW.mjs';
|
|
19
|
-
import './chunk-WQBCBKY3.mjs';
|
|
20
|
-
import './chunk-JUKYYXSP.mjs';
|
|
21
|
-
import './chunk-DQNKVOJ2.mjs';
|
|
22
|
-
import './chunk-KGOXHLLZ.mjs';
|
|
23
|
-
import './chunk-U476IXKQ.mjs';
|
|
24
|
-
import './chunk-FQBFZAWG.mjs';
|
|
25
|
-
import './chunk-VJS4JFBQ.mjs';
|
|
26
|
-
import './chunk-GES7U2PM.mjs';
|
|
27
|
-
import './chunk-ZSWHPKGM.mjs';
|
|
28
|
-
import './chunk-O4LJQI4G.mjs';
|
|
12
|
+
import './chunk-FRJ5RGX7.mjs';
|
|
29
13
|
import './chunk-X5CV2E2B.mjs';
|
|
30
|
-
import './chunk-WNBRW4Q2.mjs';
|
|
31
14
|
import './chunk-TGP5M5R7.mjs';
|
|
32
15
|
import './chunk-U2BKYEC7.mjs';
|
|
33
|
-
import './chunk-NPNK57CA.mjs';
|
|
34
|
-
import './chunk-4UMZ7KQL.mjs';
|
|
35
|
-
import './chunk-R6A773WK.mjs';
|
|
36
|
-
import './chunk-YTJCGKM4.mjs';
|
|
37
|
-
import './chunk-RIIUZJQ3.mjs';
|
|
38
16
|
import './chunk-QHPYS427.mjs';
|
|
39
|
-
import './chunk-
|
|
17
|
+
import './chunk-6Z22L5RN.mjs';
|
|
40
18
|
import './chunk-22YPZVMG.mjs';
|
|
19
|
+
import './chunk-KOKM4PQN.mjs';
|
|
20
|
+
import './chunk-QXH2CS4V.mjs';
|
|
41
21
|
import './chunk-RFBLQLLZ.mjs';
|
|
22
|
+
import './chunk-LYKUJNAV.mjs';
|
|
23
|
+
import './chunk-JUKYYXSP.mjs';
|
|
24
|
+
import './chunk-BTARFTEN.mjs';
|
|
25
|
+
import './chunk-FQBFZAWG.mjs';
|
|
26
|
+
import './chunk-LWFK3TC2.mjs';
|
|
27
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
28
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
29
|
+
import './chunk-GVPYMSLO.mjs';
|
|
42
30
|
import './chunk-L5ORED7V.mjs';
|
|
43
|
-
import './chunk-
|
|
31
|
+
import './chunk-U476IXKQ.mjs';
|
|
32
|
+
import './chunk-IQHZMCLX.mjs';
|
|
33
|
+
import './chunk-R6A773WK.mjs';
|
|
34
|
+
import './chunk-YTJCGKM4.mjs';
|
package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-WDEWU46C.mjs';
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
2
|
-
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-AOKLYVTT.mjs';
|
|
2
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-ZALEKEUH.mjs';
|
|
3
3
|
import './chunk-JPTQZWJQ.mjs';
|
|
4
4
|
import './chunk-26A5JFT2.mjs';
|
|
5
5
|
import './chunk-OIZS5KIX.mjs';
|
|
6
|
-
import { compareCodePoints } from './chunk-
|
|
7
|
-
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { compareCodePoints } from './chunk-QNDWMPV2.mjs';
|
|
7
|
+
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-75PMIFQA.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-IFGTRRPU.mjs';
|
|
9
9
|
|
|
10
10
|
// src/engine/analysis-change/pending-target.ts
|
|
11
11
|
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-5WYWLTM5.mjs';
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-QTCQ5ETU.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-QTCQ5ETU.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { extractVectorFieldsFromSchema, readHeapStatistics, readResidentMemory } from './chunk-
|
|
1
|
+
import { extractVectorFieldsFromSchema, readHeapStatistics, readResidentMemory } from './chunk-ZALEKEUH.mjs';
|
|
2
2
|
import { collectAndReturnPages } from './chunk-OIZS5KIX.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-IFGTRRPU.mjs';
|
|
4
4
|
|
|
5
5
|
// src/engine/snapshot/constants.ts
|
|
6
6
|
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-QNDWMPV2.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-IFGTRRPU.mjs';
|
|
3
3
|
|
|
4
4
|
// src/vector/vector-index/dead-bits.ts
|
|
5
5
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-
|
|
2
|
-
import { restoreVectorFields } from './chunk-
|
|
3
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-
|
|
5
|
-
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, encodeSegmentManifest, manifestKey, manifestReferencedKeys, segmentsPrefix, snapshotBundleKey, readListedFile, requireListedChecksum, vectorFileKey, DEFAULT_COMPACTION_THRESHOLD, writeSegmentsOf, readSegmentFile, graphPartKey, requireListedSize, SEGMENT_PAGE_BYTES, STABLE_HEAD_ATTEMPTS, RECORD_PAGE_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, COMPACTION_LARGEST_MERGED_DOCUMENT_COUNT, SEGMENT_FILE_FRAMING_SLACK_BYTES } from './chunk-LGJ6XMH5.mjs';
|
|
2
|
+
import { restoreVectorFields } from './chunk-XIHLIMFX.mjs';
|
|
3
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-BTMRV5MY.mjs';
|
|
4
|
+
import { VERSION, DEFAULT_BM25_B, DEFAULT_BM25_K1, rulesOfStoredSynonyms, flattenSchema, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, decodeMapWithoutFields, prepareDocumentVectors, resolveIndexAnalysis } from './chunk-ZALEKEUH.mjs';
|
|
5
|
+
import { compareCodePoints, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-QNDWMPV2.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-IFGTRRPU.mjs';
|
|
7
7
|
import { decode, encode } from '@msgpack/msgpack';
|
|
8
8
|
|
|
9
9
|
// src/engine/durability-metadata.ts
|
|
@@ -1388,7 +1388,7 @@ function holdsEveryHead(ref) {
|
|
|
1388
1388
|
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
1389
1389
|
const { pathOf } = store;
|
|
1390
1390
|
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
1391
|
-
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-
|
|
1391
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-Y2IJAP7S.mjs');
|
|
1392
1392
|
await Promise.all([
|
|
1393
1393
|
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
1394
1394
|
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
@@ -2149,7 +2149,7 @@ async function compactPartitionSegments(input) {
|
|
|
2149
2149
|
if (paged) {
|
|
2150
2150
|
const { directory } = input.store;
|
|
2151
2151
|
if (directory === void 0) return { segments, nextSegmentId: input.nextSegmentId };
|
|
2152
|
-
const { mergePagedWindow } = await import('./module-
|
|
2152
|
+
const { mergePagedWindow } = await import('./module-5LMDGXQH.mjs');
|
|
2153
2153
|
return replacing(
|
|
2154
2154
|
await mergePagedWindow({
|
|
2155
2155
|
store: input.store,
|
|
@@ -2237,7 +2237,7 @@ function pagedSettingsOf(context) {
|
|
|
2237
2237
|
async function tombstonesOfDelta(context, older, selections, deleted) {
|
|
2238
2238
|
const { directory } = context.input.store;
|
|
2239
2239
|
if (context.input.metadata.textStorage !== "disk" || directory === void 0) return deleted;
|
|
2240
|
-
const { pagedTombstonesFor } = await import('./module-
|
|
2240
|
+
const { pagedTombstonesFor } = await import('./module-5LMDGXQH.mjs');
|
|
2241
2241
|
return pagedTombstonesFor(
|
|
2242
2242
|
directory,
|
|
2243
2243
|
older,
|
|
@@ -2438,7 +2438,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
2438
2438
|
}
|
|
2439
2439
|
}
|
|
2440
2440
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
2441
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-
|
|
2441
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-5LMDGXQH.mjs');
|
|
2442
2442
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
2443
2443
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
2444
2444
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
@@ -2452,7 +2452,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
2452
2452
|
const { directory } = store;
|
|
2453
2453
|
for (const ref of partition.segments) {
|
|
2454
2454
|
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
2455
|
-
const { isPagedSegmentFile } = await import('./module-
|
|
2455
|
+
const { isPagedSegmentFile } = await import('./module-5LMDGXQH.mjs');
|
|
2456
2456
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
2457
2457
|
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
2458
2458
|
await yieldToEventLoop();
|
|
@@ -2508,7 +2508,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
2508
2508
|
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
2509
2509
|
);
|
|
2510
2510
|
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
2511
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-
|
|
2511
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-5LMDGXQH.mjs');
|
|
2512
2512
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
2513
2513
|
}
|
|
2514
2514
|
for (const partition of manifest.partitions) {
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { chunksOf, restoredConfigFields, restoredEmbedding } from './chunk-
|
|
2
|
-
import { untagDocumentLine } from './chunk-
|
|
3
|
-
import { flattenSchema, requireConfigurationShape } from './chunk-
|
|
4
|
-
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { chunksOf, restoredConfigFields, restoredEmbedding } from './chunk-6OFS7RUN.mjs';
|
|
2
|
+
import { untagDocumentLine } from './chunk-XWMEGKDC.mjs';
|
|
3
|
+
import { flattenSchema, requireConfigurationShape } from './chunk-ZALEKEUH.mjs';
|
|
4
|
+
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-75PMIFQA.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-IFGTRRPU.mjs';
|
|
6
6
|
|
|
7
7
|
// src/engine/export/lines.ts
|
|
8
8
|
var NEWLINE = 10;
|
|
@@ -25,6 +25,15 @@ var ErrorCodes = {
|
|
|
25
25
|
PERSISTENCE_CRC_MISMATCH: "PERSISTENCE_CRC_MISMATCH",
|
|
26
26
|
PERSISTENCE_WAL_CORRUPT: "PERSISTENCE_WAL_CORRUPT",
|
|
27
27
|
PERSISTENCE_FSYNC_FAILED: "PERSISTENCE_FSYNC_FAILED",
|
|
28
|
+
PERSISTENCE_DEPENDENCY_MISSING: "PERSISTENCE_DEPENDENCY_MISSING",
|
|
29
|
+
BACKUP_NOT_FOUND: "BACKUP_NOT_FOUND",
|
|
30
|
+
BACKUP_ALREADY_EXISTS: "BACKUP_ALREADY_EXISTS",
|
|
31
|
+
BACKUP_DAMAGED: "BACKUP_DAMAGED",
|
|
32
|
+
BACKUP_DURABILITY_REQUIRED: "BACKUP_DURABILITY_REQUIRED",
|
|
33
|
+
BACKUP_IN_PROGRESS: "BACKUP_IN_PROGRESS",
|
|
34
|
+
BACKUP_STORE_CLAIMED: "BACKUP_STORE_CLAIMED",
|
|
35
|
+
BACKUP_STORE_NOT_CONFIGURED: "BACKUP_STORE_NOT_CONFIGURED",
|
|
36
|
+
BACKUP_NOT_READY: "BACKUP_NOT_READY",
|
|
28
37
|
SEARCH_INVALID_FIELD: "SEARCH_INVALID_FIELD",
|
|
29
38
|
VECTOR_DIMENSION_MISMATCH: "VECTOR_DIMENSION_MISMATCH",
|
|
30
39
|
SEARCH_INVALID_FILTER: "SEARCH_INVALID_FILTER",
|