@delali/narsil 0.3.1-next.143 → 0.3.1-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
- package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
- package/dist/browser/import-FTMLTPGQ.mjs +9 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
- package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
- package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
- package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
- package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
- package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
- package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
- package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
- package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
- package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
- package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
- package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
- package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
- package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
- package/dist/chunk-NWTEJ2T7.mjs +100 -0
- package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
- package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/chunk-PRID7CA6.mjs +104 -0
- package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
- package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
- package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
- package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
- package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
- package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
- package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
- package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
- package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
- package/dist/chunk-SVO6NV4D.mjs +248 -0
- package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
- package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
- package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
- package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
- package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
- package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
- package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +44 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
- package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
- package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16219 -14535
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
- package/dist/import-6KXFY6XI.mjs +35 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -30
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
- package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
- package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +9 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
- package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +153 -36
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
- package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +5 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
- package/openapi.json +349 -2
- package/package.json +27 -8
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-KGOXHLLZ.mjs +0 -693
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-QSK35UC2.mjs +0 -35
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import { isValidInteger, isRecord } from './chunk-SNB4DYL3.mjs';
|
|
2
|
+
import { MAX_REPLICATION_FACTOR, MAX_PARTITION_COUNT } from './chunk-JDDXV6UF.mjs';
|
|
3
|
+
import { openChecksummedEnvelope, packChecksummedEnvelope } from './chunk-YUCGFPMS.mjs';
|
|
4
|
+
import { joinChunks } from './chunk-PRID7CA6.mjs';
|
|
5
|
+
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
7
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
8
|
+
|
|
9
|
+
// src/narsil/write-blocks.ts
|
|
10
|
+
var writeBlockLookups = /* @__PURE__ */ new WeakMap();
|
|
11
|
+
function bindWriteBlockLookup(owner, lookup) {
|
|
12
|
+
writeBlockLookups.set(owner, lookup);
|
|
13
|
+
}
|
|
14
|
+
function writeBlockLookupOf(owner) {
|
|
15
|
+
return writeBlockLookups.get(owner);
|
|
16
|
+
}
|
|
17
|
+
function bindEngineWriteBlock(engine, owner) {
|
|
18
|
+
const lookup = writeBlockLookups.get(owner);
|
|
19
|
+
if (lookup !== void 0) writeBlockLookups.set(engine, lookup);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// src/narsil/index-reads.ts
|
|
23
|
+
var boundReads = /* @__PURE__ */ new WeakMap();
|
|
24
|
+
function bindIndexReads(engine, reads) {
|
|
25
|
+
boundReads.set(engine, reads);
|
|
26
|
+
}
|
|
27
|
+
function indexReadsOf(engine) {
|
|
28
|
+
const bound = boundReads.get(engine);
|
|
29
|
+
if (bound !== void 0) return bound;
|
|
30
|
+
return {
|
|
31
|
+
listIndexes: async () => engine.listIndexes(),
|
|
32
|
+
getStats: async (indexName) => engine.getStats(indexName),
|
|
33
|
+
getPartitionStats: async (indexName) => engine.getPartitionStats(indexName)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// src/distribution/backups/constants.ts
|
|
38
|
+
var BACKUP_JOBS_KEY = "_narsil/backups/jobs";
|
|
39
|
+
var CLUSTER_UUID_KEY = "_narsil/cluster/uuid";
|
|
40
|
+
var BACKUP_JOB_CAS_ATTEMPTS = 20;
|
|
41
|
+
var BACKUP_JOB_CAS_RETRY_MS = 25;
|
|
42
|
+
var MAX_FINISHED_BACKUP_JOBS = 100;
|
|
43
|
+
var BACKUP_JOB_POLL_INTERVAL_MS = 250;
|
|
44
|
+
var MAX_BACKUP_NAME_LENGTH = 128;
|
|
45
|
+
var MAX_BACKUP_KEY_PREFIX_LENGTH = 512;
|
|
46
|
+
var BACKUP_PARTITION_POLL_INTERVAL_MS = 500;
|
|
47
|
+
var BACKUP_PARTITION_RESULT_RETENTION_MS = 6e5;
|
|
48
|
+
var BACKUP_PRIMARY_WAIT_MS = 12e4;
|
|
49
|
+
var BACKUP_PARTITIONS_AT_ONCE = 4;
|
|
50
|
+
var BACKUP_FILES_AT_ONCE = 4;
|
|
51
|
+
var BACKUP_FILE_CHUNK_BYTES = 1048576;
|
|
52
|
+
var BACKUP_DELETES_AT_ONCE = 8;
|
|
53
|
+
var BACKUP_PROGRESS_REPORT_INTERVAL_MS = 1e3;
|
|
54
|
+
var BACKUP_SWEEP_INTERVAL_MS = 3e4;
|
|
55
|
+
var BACKUP_PUBLISH_RETRY_MS = 1e3;
|
|
56
|
+
var MAX_BACKUP_FILES_PER_PARTITION = 1e6;
|
|
57
|
+
var MAX_BACKUP_FILE_PATH_LENGTH = 1024;
|
|
58
|
+
var BACKUP_MANIFEST_READS_AT_ONCE = 8;
|
|
59
|
+
var MAX_CLUSTER_UUID_BYTES = 128;
|
|
60
|
+
|
|
61
|
+
// src/distribution/backups/keys.ts
|
|
62
|
+
var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
63
|
+
var PREFIX_SEGMENT_PATTERN = /^[A-Za-z0-9._-]+$/;
|
|
64
|
+
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/;
|
|
65
|
+
var CLAIM_KEY = "cluster";
|
|
66
|
+
var MANIFESTS_DIRECTORY = "backups/";
|
|
67
|
+
var FILES_DIRECTORY = "files/";
|
|
68
|
+
function validateBackupName(backupName) {
|
|
69
|
+
if (typeof backupName !== "string" || backupName.length > MAX_BACKUP_NAME_LENGTH || !NAME_PATTERN.test(backupName)) {
|
|
70
|
+
throw new NarsilError(
|
|
71
|
+
ErrorCodes.CONFIG_INVALID,
|
|
72
|
+
`A backup name holds from 1 to ${MAX_BACKUP_NAME_LENGTH} letters, digits, '.', '_', and '-', and starts with a letter or a digit`,
|
|
73
|
+
{ backupName: typeof backupName === "string" ? backupName.slice(0, MAX_BACKUP_NAME_LENGTH) : typeof backupName }
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
return backupName;
|
|
77
|
+
}
|
|
78
|
+
function validateBackupKeyPrefix(keyPrefix) {
|
|
79
|
+
const valid = typeof keyPrefix === "string" && keyPrefix.length <= MAX_BACKUP_KEY_PREFIX_LENGTH && keyPrefix.split("/").every((segment) => PREFIX_SEGMENT_PATTERN.test(segment) && segment !== "." && segment !== "..");
|
|
80
|
+
if (!valid) {
|
|
81
|
+
throw new NarsilError(
|
|
82
|
+
ErrorCodes.CONFIG_INVALID,
|
|
83
|
+
`A backup key prefix holds up to ${MAX_BACKUP_KEY_PREFIX_LENGTH} characters in segments joined by '/', where each segment holds letters, digits, '.', '_', and '-', and no segment is '.' or '..'`,
|
|
84
|
+
{
|
|
85
|
+
keyPrefix: typeof keyPrefix === "string" ? keyPrefix.slice(0, MAX_BACKUP_KEY_PREFIX_LENGTH) : typeof keyPrefix
|
|
86
|
+
}
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
return keyPrefix;
|
|
90
|
+
}
|
|
91
|
+
function storeWithoutSaveIfAbsent() {
|
|
92
|
+
return new NarsilError(
|
|
93
|
+
ErrorCodes.CONFIG_INVALID,
|
|
94
|
+
"A backup store must offer saveIfAbsent, because a save followed by a read can't stop two clusters from both claiming an empty key prefix",
|
|
95
|
+
{}
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
function validateBackupConfig(backups) {
|
|
99
|
+
const store = typeof backups === "object" && backups !== null ? backups.store : void 0;
|
|
100
|
+
if (typeof store !== "object" || store === null) {
|
|
101
|
+
throw new NarsilError(
|
|
102
|
+
ErrorCodes.CONFIG_INVALID,
|
|
103
|
+
"ClusterNodeConfig.backups.store must be a persistence adapter",
|
|
104
|
+
{}
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
if (typeof store.saveIfAbsent !== "function") throw storeWithoutSaveIfAbsent();
|
|
108
|
+
validateBackupKeyPrefix(backups.keyPrefix);
|
|
109
|
+
}
|
|
110
|
+
function isSha256Hex(value) {
|
|
111
|
+
return typeof value === "string" && SHA256_HEX_PATTERN.test(value);
|
|
112
|
+
}
|
|
113
|
+
function backupKeys(keyPrefix) {
|
|
114
|
+
const prefix = `${validateBackupKeyPrefix(keyPrefix)}/`;
|
|
115
|
+
return {
|
|
116
|
+
prefix,
|
|
117
|
+
claim: `${prefix}${CLAIM_KEY}`,
|
|
118
|
+
manifests: `${prefix}${MANIFESTS_DIRECTORY}`,
|
|
119
|
+
files: `${prefix}${FILES_DIRECTORY}`,
|
|
120
|
+
manifest: (backupName) => `${prefix}${MANIFESTS_DIRECTORY}${validateBackupName(backupName)}`,
|
|
121
|
+
file: (sha256) => {
|
|
122
|
+
if (!isSha256Hex(sha256)) {
|
|
123
|
+
throw new NarsilError(ErrorCodes.BACKUP_DAMAGED, "A backup lists a file under a digest that is not SHA-256", {
|
|
124
|
+
sha256: String(sha256).slice(0, 80)
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
return `${prefix}${FILES_DIRECTORY}${sha256}`;
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
var BACKUP_MANIFEST_ENVELOPE_VERSION = 1;
|
|
132
|
+
var PATH_SEGMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
133
|
+
function damaged(backupName, reason) {
|
|
134
|
+
return new NarsilError(ErrorCodes.BACKUP_DAMAGED, `The manifest of backup '${backupName}' ${reason}`, {
|
|
135
|
+
backupName
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
function isCount(value) {
|
|
139
|
+
return isValidInteger(value) && value >= 0;
|
|
140
|
+
}
|
|
141
|
+
function isSafeFileKey(key) {
|
|
142
|
+
return typeof key === "string" && key.length <= MAX_BACKUP_FILE_PATH_LENGTH && key.split("/").every((segment) => PATH_SEGMENT_PATTERN.test(segment));
|
|
143
|
+
}
|
|
144
|
+
function readAllocationConstraints(value) {
|
|
145
|
+
if (!isRecord(value) || typeof value.zoneAwareness !== "boolean" || typeof value.zoneAttribute !== "string" || value.zoneAttribute.length === 0 || !(value.maxShardsPerNode === null || isValidInteger(value.maxShardsPerNode) && value.maxShardsPerNode > 0)) {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
zoneAwareness: value.zoneAwareness,
|
|
150
|
+
zoneAttribute: value.zoneAttribute,
|
|
151
|
+
maxShardsPerNode: value.maxShardsPerNode
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
function readStoredFile(value, backupName) {
|
|
155
|
+
if (!isRecord(value) || !isSha256Hex(value.sha256) || !isCount(value.bytes) || value.bytes > MAX_FILE_BYTES) {
|
|
156
|
+
throw damaged(backupName, "lists a file without a valid SHA-256 digest and length");
|
|
157
|
+
}
|
|
158
|
+
return { sha256: value.sha256, bytes: value.bytes };
|
|
159
|
+
}
|
|
160
|
+
function readListedFile(value, backupName) {
|
|
161
|
+
const stored = readStoredFile(value, backupName);
|
|
162
|
+
if (!isRecord(value) || !isSafeFileKey(value.key)) throw damaged(backupName, "lists a file under an unsafe key");
|
|
163
|
+
return { key: value.key, ...stored };
|
|
164
|
+
}
|
|
165
|
+
function readBackupPartition(value, backupName, partitionCount) {
|
|
166
|
+
if (!isRecord(value) || !isCount(value.partitionId) || value.partitionId >= partitionCount || !isCount(value.lastSeqNo) || !Array.isArray(value.files) || value.files.length > MAX_BACKUP_FILES_PER_PARTITION) {
|
|
167
|
+
throw damaged(backupName, "holds a malformed partition entry");
|
|
168
|
+
}
|
|
169
|
+
const files = value.files.map((file) => readListedFile(file, backupName));
|
|
170
|
+
if (new Set(files.map((file) => file.key)).size !== files.length) {
|
|
171
|
+
throw damaged(backupName, `lists one key twice in partition ${value.partitionId}`);
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
partitionId: value.partitionId,
|
|
175
|
+
lastSeqNo: value.lastSeqNo,
|
|
176
|
+
manifest: readStoredFile(value.manifest, backupName),
|
|
177
|
+
files
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function validateManifest(raw, backupName) {
|
|
181
|
+
if (!isRecord(raw) || raw.name !== backupName || typeof raw.clusterUuid !== "string" || typeof raw.indexName !== "string" || !isCount(raw.startedAt) || !isCount(raw.completedAt) || !isValidInteger(raw.partitionCount) || raw.partitionCount < 1 || raw.partitionCount > MAX_PARTITION_COUNT || !isCount(raw.replicationFactor) || raw.replicationFactor > MAX_REPLICATION_FACTOR || !Array.isArray(raw.partitions) || raw.partitions.length !== raw.partitionCount) {
|
|
182
|
+
throw damaged(backupName, "holds a malformed header");
|
|
183
|
+
}
|
|
184
|
+
const constraints = readAllocationConstraints(raw.constraints);
|
|
185
|
+
if (constraints === null) throw damaged(backupName, "holds malformed allocation constraints");
|
|
186
|
+
const partitionCount = raw.partitionCount;
|
|
187
|
+
const partitions = raw.partitions.map((entry) => readBackupPartition(entry, backupName, partitionCount));
|
|
188
|
+
if (partitions.some((entry, position) => entry.partitionId !== position)) {
|
|
189
|
+
throw damaged(backupName, "lists its partitions out of partitionId order");
|
|
190
|
+
}
|
|
191
|
+
return {
|
|
192
|
+
name: backupName,
|
|
193
|
+
clusterUuid: raw.clusterUuid,
|
|
194
|
+
indexName: raw.indexName,
|
|
195
|
+
startedAt: raw.startedAt,
|
|
196
|
+
completedAt: raw.completedAt,
|
|
197
|
+
partitionCount,
|
|
198
|
+
replicationFactor: raw.replicationFactor,
|
|
199
|
+
constraints,
|
|
200
|
+
config: readStoredFile(raw.config, backupName),
|
|
201
|
+
partitions
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
async function decodeBackupManifest(bytes, backupName) {
|
|
205
|
+
let raw;
|
|
206
|
+
try {
|
|
207
|
+
const { payload } = await openChecksummedEnvelope(bytes, {
|
|
208
|
+
envelopeFormatVersion: BACKUP_MANIFEST_ENVELOPE_VERSION,
|
|
209
|
+
what: `manifest of backup '${backupName}'`,
|
|
210
|
+
allowsCompression: false
|
|
211
|
+
});
|
|
212
|
+
raw = decode(payload);
|
|
213
|
+
} catch (error) {
|
|
214
|
+
throw damaged(backupName, `is not readable: ${error instanceof Error ? error.message : String(error)}`);
|
|
215
|
+
}
|
|
216
|
+
return validateManifest(raw, backupName);
|
|
217
|
+
}
|
|
218
|
+
async function encodeBackupManifest(manifest) {
|
|
219
|
+
const packed = await packChecksummedEnvelope([encode(manifest)], BACKUP_MANIFEST_ENVELOPE_VERSION, "backup manifest");
|
|
220
|
+
return joinChunks(packed.chunks, packed.byteLength);
|
|
221
|
+
}
|
|
222
|
+
function storedFilesOf(manifest) {
|
|
223
|
+
const files = [manifest.config];
|
|
224
|
+
for (const partition of manifest.partitions) files.push(partition.manifest, ...partition.files);
|
|
225
|
+
return files;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/distribution/backups/restore-options.ts
|
|
229
|
+
function readRestorePlacement(request) {
|
|
230
|
+
const { replicationFactor, constraints } = request;
|
|
231
|
+
const placement = {};
|
|
232
|
+
if (replicationFactor !== void 0 && replicationFactor !== null) {
|
|
233
|
+
if (!isValidInteger(replicationFactor) || replicationFactor < 0 || replicationFactor > MAX_REPLICATION_FACTOR) {
|
|
234
|
+
return `"replicationFactor" must be an integer from 0 to ${MAX_REPLICATION_FACTOR}`;
|
|
235
|
+
}
|
|
236
|
+
placement.replicationFactor = replicationFactor;
|
|
237
|
+
}
|
|
238
|
+
if (constraints !== void 0 && constraints !== null) {
|
|
239
|
+
const read = readAllocationConstraints(constraints);
|
|
240
|
+
if (read === null) {
|
|
241
|
+
return '"constraints" must hold "zoneAwareness" as a boolean, "zoneAttribute" as a non-empty string, and "maxShardsPerNode" as a positive integer or null';
|
|
242
|
+
}
|
|
243
|
+
placement.constraints = read;
|
|
244
|
+
}
|
|
245
|
+
return placement;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export { BACKUP_DELETES_AT_ONCE, BACKUP_FILES_AT_ONCE, BACKUP_FILE_CHUNK_BYTES, BACKUP_JOBS_KEY, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_MANIFEST_READS_AT_ONCE, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_PRIMARY_WAIT_MS, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PUBLISH_RETRY_MS, BACKUP_SWEEP_INTERVAL_MS, CLUSTER_UUID_KEY, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, backupKeys, bindEngineWriteBlock, bindIndexReads, bindWriteBlockLookup, decodeBackupManifest, encodeBackupManifest, indexReadsOf, isCount, isSha256Hex, readBackupPartition, readRestorePlacement, readStoredFile, storeWithoutSaveIfAbsent, storedFilesOf, validateBackupConfig, validateBackupName, writeBlockLookupOf };
|
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import { encodeDocumentJson } from './chunk-
|
|
2
|
-
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-
|
|
1
|
+
import { encodeDocumentJson } from './chunk-6FOJKNUR.mjs';
|
|
2
|
+
import { loadFailed, sectionNamed, PAGED_SEGMENT_ENVELOPE_VERSION, encodePagedSegmentHead, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, PAYLOAD_PREFIX_BYTES, decodePagedSegmentHead, BLOCK_SUMMARY_BYTES, SMALLEST_BLOCK_POSTINGS, DOCUMENT_BLOCK_BYTES, decodeSections, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, MAX_DOCUMENT_BLOCK_RAW_BYTES } from './chunk-ONDEEBTA.mjs';
|
|
3
3
|
import { createVectorFileReader } from './chunk-FOCV4CBZ.mjs';
|
|
4
|
-
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-
|
|
5
|
-
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-
|
|
6
|
-
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-
|
|
7
|
-
import { wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-
|
|
8
|
-
import { isCompositePartition, buildPartialWordTableFrom, createPartitionIndex, createCompositePartition, mergeDocFrequencies, scanPostingBlocks } from './chunk-
|
|
9
|
-
import {
|
|
10
|
-
import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
|
|
11
|
-
import { flattenSchema, encodeDocument } from './chunk-KGOXHLLZ.mjs';
|
|
12
|
-
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
+
import { writeSegmentsOf, MessagePackChunks, writeSurfaceForms, writeFieldLengths, writeValueColumns, writeBooleanColumns, writeEnumColumns, writeGeopointColumns, writePatternColumns, writePartialWords, FILE_ASSEMBLY_CHUNK_BYTES, REWRITE_SPILL_BYTES, LIST_VIEWS_HELD_PER_SEGMENT, TOMBSTONES_BETWEEN_YIELDS, readSegmentFile, READS_AHEAD_AT_ONCE, WRITER_YIELD_INTERVAL_TOKENS, LARGEST_CACHED_ENTRY_SHARE, MAX_THREAD_FILE_NUMBER, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, SECTION_WRITE_BUFFER_BYTES, READ_AHEAD_CHUNK_BYTES, LARGEST_REMOVAL_BUCKET_COUNT, REMOVAL_BUCKET_TARGET_BYTES, THREAD_READ_CACHE_BYTES, MERGE_READ_CACHE_BYTES } from './chunk-RVBUQITB.mjs';
|
|
5
|
+
import { requireOffsets, requireUtf8Entries, requireEntriesAscend, decodeFieldLengths, decodePartialWords, decodePatternColumns, decodeGeopointColumns, decodeEnumColumns, decodeBooleanColumns, decodeValueColumns, setNestedValue, littleEndianBytes, uint32sFrom, uint64Bytes, decodeSurfaceForms } from './chunk-QU2XF7VI.mjs';
|
|
6
|
+
import { HEADER_SIZE, requireWithinFileLimit, crc32Init, crc32Update, crc32Final, engineVersionParts, writeHeader, crc32, readHeader } from './chunk-YUCGFPMS.mjs';
|
|
7
|
+
import { generateId, wrapFrozenTokenTable, createEncodedDocumentSource, assemblePagedFrozenSegment, decodeStoredDocument, mergeFieldLengths, mergeSurfaceForms, encodeSurfaceTable, mergePartialWords, mergeFieldIndexes, resolvePartitionInsertOptions, decodeSurfaceTable, createSharedFrozenSegment, shareSegmentArrays, selectionKeeps, fieldIndexRemapOf, freezeSegmentShared, createFrozenSegment, holdsADiscardedField, endOfValue } from './chunk-OH7SX5MY.mjs';
|
|
8
|
+
import { isCompositePartition, buildPartialWordTableFrom, createPartitionIndex, createCompositePartition, mergeDocFrequencies, scanPostingBlocks } from './chunk-QYMAYU7M.mjs';
|
|
9
|
+
import { flattenSchema, parseVectorDimension, encodeDocument } from './chunk-BTARFTEN.mjs';
|
|
13
10
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
11
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
12
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
13
|
import { decode, encode } from '@msgpack/msgpack';
|
|
16
14
|
import { deflateRawSync, inflateRawSync } from 'zlib';
|
|
17
15
|
|
|
@@ -1309,7 +1307,7 @@ async function coldFieldOf(checkpoint, field) {
|
|
|
1309
1307
|
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
1310
1308
|
if (dimension === null) return null;
|
|
1311
1309
|
const withPath = async (ref) => ({ ref, path: await directory.pathOf(ref.key) });
|
|
1312
|
-
const { openColdVectorField } = await import('./cold-module-
|
|
1310
|
+
const { openColdVectorField } = await import('./cold-module-NUD332YD.mjs');
|
|
1313
1311
|
return openColdVectorField({
|
|
1314
1312
|
fieldPath: field.fieldPath,
|
|
1315
1313
|
dimension,
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { untagDocumentLine } from './chunk-
|
|
1
|
+
import { untagDocumentLine } from './chunk-5K2JHKM3.mjs';
|
|
2
2
|
import { chunksOf } from './chunk-CZQ33DLW.mjs';
|
|
3
|
-
import { restoredConfigFields, restoredEmbedding } from './chunk-
|
|
4
|
-
import { requireConfigurationShape } from './chunk-
|
|
3
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-FTQLF5JI.mjs';
|
|
4
|
+
import { requireConfigurationShape } from './chunk-YUCGFPMS.mjs';
|
|
5
5
|
import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
|
|
6
|
-
import { flattenSchema } from './chunk-
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { flattenSchema } from './chunk-BTARFTEN.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
8
8
|
|
|
9
9
|
// src/engine/export/lines.ts
|
|
10
10
|
var NEWLINE = 10;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { packChecksummedEnvelope } from './chunk-
|
|
2
|
-
import { fnv1a } from './chunk-
|
|
3
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { packChecksummedEnvelope } from './chunk-YUCGFPMS.mjs';
|
|
2
|
+
import { fnv1a } from './chunk-QYMAYU7M.mjs';
|
|
3
|
+
import { ErrorCodes, NarsilError } from './chunk-IQHZMCLX.mjs';
|
|
4
4
|
import { encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/persistence/durability/segment/store.ts
|
|
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
|
|
|
4
4
|
}
|
|
5
5
|
async function abandonChangeOf(core, indexName) {
|
|
6
6
|
if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
|
|
7
|
-
const { abandonAnalysisChange } = await import('./analysis-change-
|
|
7
|
+
const { abandonAnalysisChange } = await import('./analysis-change-D5APH4Z3.mjs');
|
|
8
8
|
await abandonAnalysisChange(core, indexName);
|
|
9
9
|
}
|
|
10
10
|
async function resumeChangeOf(core, indexName) {
|
|
11
11
|
if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
12
12
|
if (core.mutationCtx.capacityScope === "partition") return;
|
|
13
13
|
if (core.executor.getManager(indexName) === void 0) return;
|
|
14
|
-
const { resumeAnalysisChange } = await import('./analysis-change-
|
|
14
|
+
const { resumeAnalysisChange } = await import('./analysis-change-D5APH4Z3.mjs');
|
|
15
15
|
void resumeAnalysisChange(core, indexName).catch(() => void 0);
|
|
16
16
|
}
|
|
17
17
|
async function resumeEveryChange(core) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createVectorFileReader } from './chunk-FOCV4CBZ.mjs';
|
|
2
|
-
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-
|
|
3
|
-
import { MAX_VECTOR_ORDINALS } from './chunk-
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-MJTEU2XK.mjs';
|
|
3
|
+
import { MAX_VECTOR_ORDINALS } from './chunk-QXH2CS4V.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
5
5
|
|
|
6
6
|
// src/vector/vector-store/hot-block-distance.ts
|
|
7
7
|
function jsDistance(a, b, magA, magB, metric) {
|
|
@@ -705,6 +705,60 @@ function partitionFilterOf(store, documents, partitionIds) {
|
|
|
705
705
|
return filter;
|
|
706
706
|
}
|
|
707
707
|
|
|
708
|
+
// src/vector/vector-store/retired-pins.ts
|
|
709
|
+
function createRetiredVectorPins() {
|
|
710
|
+
let pins = 0;
|
|
711
|
+
let lost = false;
|
|
712
|
+
const retired = /* @__PURE__ */ new Set();
|
|
713
|
+
return {
|
|
714
|
+
pin() {
|
|
715
|
+
pins += 1;
|
|
716
|
+
},
|
|
717
|
+
unpin() {
|
|
718
|
+
if (pins === 0) return;
|
|
719
|
+
pins -= 1;
|
|
720
|
+
if (pins > 0) return;
|
|
721
|
+
retired.clear();
|
|
722
|
+
lost = false;
|
|
723
|
+
},
|
|
724
|
+
note(ordinal) {
|
|
725
|
+
if (pins > 0) retired.add(ordinal);
|
|
726
|
+
},
|
|
727
|
+
holds(ordinal) {
|
|
728
|
+
return retired.has(ordinal);
|
|
729
|
+
},
|
|
730
|
+
keptWith(replaced) {
|
|
731
|
+
if (retired.size === 0) return replaced;
|
|
732
|
+
const kept = new Set(replaced);
|
|
733
|
+
for (const ordinal of retired) kept.add(ordinal);
|
|
734
|
+
return kept;
|
|
735
|
+
},
|
|
736
|
+
forget() {
|
|
737
|
+
retired.clear();
|
|
738
|
+
if (pins > 0) lost = true;
|
|
739
|
+
},
|
|
740
|
+
requireIntact() {
|
|
741
|
+
if (!lost) return;
|
|
742
|
+
throw new NarsilError(
|
|
743
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
744
|
+
"The vector store was emptied while a checkpoint read the vectors it captured, so the checkpoint stops"
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
function pinnedReads(pins, openStore, readable) {
|
|
750
|
+
return {
|
|
751
|
+
pinRetiredVectors: () => pins.pin(),
|
|
752
|
+
unpinRetiredVectors: () => pins.unpin(),
|
|
753
|
+
entryForPinnedOrdinal(ordinal) {
|
|
754
|
+
pins.requireIntact();
|
|
755
|
+
const open = openStore();
|
|
756
|
+
if (open === null || !(readable(ordinal) || pins.holds(ordinal))) return void 0;
|
|
757
|
+
return entryAt(open, ordinal);
|
|
758
|
+
}
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
|
|
708
762
|
// src/vector/vector-store/index.ts
|
|
709
763
|
var UNKNOWN_PARTITION = -1;
|
|
710
764
|
var NO_PARTITION = -2;
|
|
@@ -714,6 +768,7 @@ function createVectorStore(options) {
|
|
|
714
768
|
const docToOrd = /* @__PURE__ */ new Map();
|
|
715
769
|
const ordToDoc = [];
|
|
716
770
|
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
771
|
+
const pins = createRetiredVectorPins();
|
|
717
772
|
let unknownPartitions = 0;
|
|
718
773
|
let liveCount = 0;
|
|
719
774
|
let readBackVector = null;
|
|
@@ -722,6 +777,7 @@ function createVectorStore(options) {
|
|
|
722
777
|
docToOrd.clear();
|
|
723
778
|
ordToDoc.length = 0;
|
|
724
779
|
replacedOrdinals.clear();
|
|
780
|
+
pins.forget();
|
|
725
781
|
unknownPartitions = 0;
|
|
726
782
|
liveCount = 0;
|
|
727
783
|
}
|
|
@@ -742,6 +798,7 @@ function createVectorStore(options) {
|
|
|
742
798
|
function retireOrdinal(store, ordinal) {
|
|
743
799
|
ordToDoc[ordinal] = void 0;
|
|
744
800
|
Atomics.store(store.present, ordinal, 0);
|
|
801
|
+
pins.note(ordinal);
|
|
745
802
|
if (store.partitions[ordinal] === UNKNOWN_PARTITION) unknownPartitions -= 1;
|
|
746
803
|
}
|
|
747
804
|
function keepReplacedOrdinal(store, ordinal) {
|
|
@@ -819,9 +876,11 @@ function createVectorStore(options) {
|
|
|
819
876
|
releaseReplacedVectors() {
|
|
820
877
|
const released = [...replacedOrdinals];
|
|
821
878
|
replacedOrdinals.clear();
|
|
879
|
+
for (const ordinal of released) pins.note(ordinal);
|
|
822
880
|
if (open !== null) for (const ordinal of released) Atomics.store(open.present, ordinal, 0);
|
|
823
881
|
return released;
|
|
824
882
|
},
|
|
883
|
+
...pinnedReads(pins, () => open, readable),
|
|
825
884
|
insertCold(docId, vectorMagnitude, location, partitionId) {
|
|
826
885
|
const store = requireOpen();
|
|
827
886
|
const previous = docToOrd.get(docId);
|
|
@@ -872,13 +931,14 @@ function createVectorStore(options) {
|
|
|
872
931
|
},
|
|
873
932
|
releaseColdBlocks() {
|
|
874
933
|
const store = requireOpen();
|
|
934
|
+
const kept = pins.keptWith(replacedOrdinals);
|
|
875
935
|
let released = 0;
|
|
876
936
|
for (let index = 0; index < store.handles.blocks.length; index++) {
|
|
877
|
-
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc,
|
|
937
|
+
if (store.handles.blocks[index] === null || !blockFullyCold(store, index, ordToDoc, kept)) continue;
|
|
878
938
|
store.handles.blocks[index] = null;
|
|
879
939
|
released += 1;
|
|
880
940
|
}
|
|
881
|
-
if (released + forgetUnreferencedFiles(store, ordToDoc,
|
|
941
|
+
if (released + forgetUnreferencedFiles(store, ordToDoc, kept) > 0) relayout(store);
|
|
882
942
|
return released;
|
|
883
943
|
},
|
|
884
944
|
setPartition(docId, partitionId) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readMetadataEnvelope } from './chunk-
|
|
2
|
-
import { nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope } from './chunk-IV2XTSS2.mjs';
|
|
2
|
+
import { nextSegmentKey, readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-KG5N7LE6.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-QU2XF7VI.mjs';
|
|
4
4
|
|
|
5
5
|
// src/persistence/durability/recovery.ts
|
|
6
6
|
async function removeLeftoverRewriteFiles(directory, names) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { rerankQueryTextOf } from './chunk-KZ3NX2P7.mjs';
|
|
2
|
-
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-
|
|
2
|
+
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-ISUOSACK.mjs';
|
|
3
3
|
import { DEFAULT_RERANK_WINDOW, MIN_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
4
|
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
|
|
5
5
|
import { RESULT_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
7
7
|
|
|
8
8
|
// src/rerank/params.ts
|
|
9
9
|
function invalid(message, details = {}) {
|