@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,6 +1,20 @@
|
|
|
1
|
+
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
1
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
2
4
|
|
|
3
|
-
// src/
|
|
5
|
+
// src/languages/registry.ts
|
|
6
|
+
var languages = /* @__PURE__ */ new Map();
|
|
7
|
+
languages.set("english", english);
|
|
8
|
+
function registerLanguage(module) {
|
|
9
|
+
languages.set(module.name, module);
|
|
10
|
+
}
|
|
11
|
+
function getLanguage(name) {
|
|
12
|
+
const lang = languages.get(name);
|
|
13
|
+
if (!lang) {
|
|
14
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
15
|
+
}
|
|
16
|
+
return lang;
|
|
17
|
+
}
|
|
4
18
|
|
|
5
19
|
// src/core/synonyms/constants.ts
|
|
6
20
|
var MAX_SYNONYM_RULES = 1e5;
|
|
@@ -111,4 +125,4 @@ function decodeSynonymRules(bytes, raise) {
|
|
|
111
125
|
return parseSynonymRules(decoded, raise);
|
|
112
126
|
}
|
|
113
127
|
|
|
114
|
-
export { SYNONYMS_ADDED_PER_QUERY, SYNONYM_EXPANSION_CACHE_ENTRIES, decodeSynonymRules, encodeSynonymRules, parseSynonymRules };
|
|
128
|
+
export { SYNONYMS_ADDED_PER_QUERY, SYNONYM_EXPANSION_CACHE_ENTRIES, decodeSynonymRules, encodeSynonymRules, getLanguage, parseSynonymRules, registerLanguage };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { rerankQueryTextOf } from './chunk-KZ3NX2P7.mjs';
|
|
2
|
-
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-
|
|
2
|
+
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-C3YDYX6C.mjs';
|
|
3
3
|
import { DEFAULT_RERANK_WINDOW, MIN_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
|
-
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-
|
|
4
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
|
|
5
5
|
import { RESULT_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
7
7
|
|
|
8
8
|
// src/rerank/params.ts
|
|
9
9
|
function invalid(message, details = {}) {
|
|
@@ -1,56 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createVectorFileReader } from './chunk-FOCV4CBZ.mjs';
|
|
2
|
+
import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, STORE_DOC_ID_BYTES, unplacedChunkCount, blocksGrowthBytes, growBufferTo, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, grownByteLength, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-TNBIK2LQ.mjs';
|
|
2
3
|
import { MAX_VECTOR_ORDINALS } from './chunk-QSK35UC2.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
-
import { readSync, closeSync, openSync } from 'fs';
|
|
5
|
-
|
|
6
|
-
function createVectorFileReader() {
|
|
7
|
-
const descriptors = /* @__PURE__ */ new Map();
|
|
8
|
-
function descriptorOf(path) {
|
|
9
|
-
const open = descriptors.get(path);
|
|
10
|
-
if (open !== void 0) return open;
|
|
11
|
-
try {
|
|
12
|
-
const descriptor = openSync(path, "r");
|
|
13
|
-
descriptors.set(path, descriptor);
|
|
14
|
-
return descriptor;
|
|
15
|
-
} catch {
|
|
16
|
-
return null;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
function closeDescriptor(path, descriptor) {
|
|
20
|
-
descriptors.delete(path);
|
|
21
|
-
try {
|
|
22
|
-
closeSync(descriptor);
|
|
23
|
-
} catch {
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
return {
|
|
27
|
-
readInto(path, offset, target) {
|
|
28
|
-
const descriptor = descriptorOf(path);
|
|
29
|
-
if (descriptor === null) return false;
|
|
30
|
-
let read = 0;
|
|
31
|
-
while (read < target.byteLength) {
|
|
32
|
-
let chunk;
|
|
33
|
-
try {
|
|
34
|
-
chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
|
|
35
|
-
} catch {
|
|
36
|
-
closeDescriptor(path, descriptor);
|
|
37
|
-
return false;
|
|
38
|
-
}
|
|
39
|
-
if (chunk <= 0) return false;
|
|
40
|
-
read += chunk;
|
|
41
|
-
}
|
|
42
|
-
return true;
|
|
43
|
-
},
|
|
44
|
-
retainOnly(paths) {
|
|
45
|
-
for (const [path, descriptor] of descriptors) {
|
|
46
|
-
if (!paths.has(path)) closeDescriptor(path, descriptor);
|
|
47
|
-
}
|
|
48
|
-
},
|
|
49
|
-
close() {
|
|
50
|
-
for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
|
|
51
|
-
}
|
|
52
|
-
};
|
|
53
|
-
}
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
54
5
|
|
|
55
6
|
// src/vector/vector-store/hot-block-distance.ts
|
|
56
7
|
function jsDistance(a, b, magA, magB, metric) {
|
|
@@ -63,6 +14,11 @@ function jsDistance(a, b, magA, magB, metric) {
|
|
|
63
14
|
return euclideanDistance(a, b);
|
|
64
15
|
}
|
|
65
16
|
}
|
|
17
|
+
function distanceFromQueryOf(source, coldScratch, query, queryMagnitude, ordinal, metric) {
|
|
18
|
+
if (!source.holdsOrdinal(ordinal)) return Number.POSITIVE_INFINITY;
|
|
19
|
+
const vector = source.isCold(ordinal) && source.readColdInto(ordinal, coldScratch) ? coldScratch : source.vectorAt(ordinal);
|
|
20
|
+
return jsDistance(query, vector, queryMagnitude, source.magnitudeAt(ordinal), metric);
|
|
21
|
+
}
|
|
66
22
|
function inMemory(reads, ordinal) {
|
|
67
23
|
return ordinal >= 0 && ordinal < reads.limit && reads.diskFile[ordinal] === IN_MEMORY;
|
|
68
24
|
}
|
|
@@ -159,9 +115,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
159
115
|
}
|
|
160
116
|
return diskFile[ordinal];
|
|
161
117
|
}
|
|
162
|
-
|
|
163
|
-
return fileOf(ordinal) !== IN_MEMORY;
|
|
164
|
-
}
|
|
118
|
+
const isCold = (ordinal) => fileOf(ordinal) !== IN_MEMORY;
|
|
165
119
|
function holdState(ordinal) {
|
|
166
120
|
if (ordinal < 0) return NOT_HELD;
|
|
167
121
|
if (ordinal >= present.length) {
|
|
@@ -173,9 +127,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
173
127
|
const index = Math.floor(ordinal / capacity);
|
|
174
128
|
return index < handles.blocks.length && handles.blocks[index] !== null ? HELD_HOT : NOT_HELD;
|
|
175
129
|
}
|
|
176
|
-
|
|
177
|
-
return holdState(ordinal) !== NOT_HELD;
|
|
178
|
-
}
|
|
130
|
+
const holds = (ordinal) => holdState(ordinal) !== NOT_HELD;
|
|
179
131
|
function readFromFile(file, offset, target) {
|
|
180
132
|
const path = handles.vectorFiles[file];
|
|
181
133
|
if (path === void 0 || path.length === 0) return false;
|
|
@@ -383,6 +335,7 @@ function openSharedVectorStore(initial, threadSlot) {
|
|
|
383
335
|
}
|
|
384
336
|
return jsDistance(currentQuery, vectorIn(block, local), prepared.magnitude, magnitudes[ordinal], metric);
|
|
385
337
|
},
|
|
338
|
+
distanceFromQuery: (query, queryMagnitude, ordinal, metric) => distanceFromQueryOf(view, coldScratchB, query, queryMagnitude, ordinal, metric),
|
|
386
339
|
hotBlockReads,
|
|
387
340
|
queryDistance: (prepared, metric) => queryDistanceOf(view, prepared, metric),
|
|
388
341
|
ordinalDistance: (from, metric) => ordinalDistanceOf(view, from, metric),
|
|
@@ -526,6 +479,11 @@ function createOrdinalReads(openStore, readable) {
|
|
|
526
479
|
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
527
480
|
return open.view.distanceFromArena(prepared, ordinal, metric);
|
|
528
481
|
},
|
|
482
|
+
distanceFromQuery(query, queryMagnitude, ordinal, metric) {
|
|
483
|
+
const open = openStore();
|
|
484
|
+
if (open === null || !readable(ordinal)) return Number.POSITIVE_INFINITY;
|
|
485
|
+
return open.view.distanceFromQuery(query, queryMagnitude, ordinal, metric);
|
|
486
|
+
},
|
|
529
487
|
queryDistance(prepared, metric) {
|
|
530
488
|
const open = openStore();
|
|
531
489
|
if (open === null) return () => Number.POSITIVE_INFINITY;
|
|
@@ -702,13 +660,12 @@ function blockFullyCold(store, blockIndex, documents, replacedOrdinals) {
|
|
|
702
660
|
}
|
|
703
661
|
return true;
|
|
704
662
|
}
|
|
705
|
-
function fileHoldsSameBytes(store, ordinal, location) {
|
|
706
|
-
const fromFile = new Float32Array(store.handles.dimension);
|
|
663
|
+
function fileHoldsSameBytes(store, ordinal, location, fromFile) {
|
|
707
664
|
if (!store.view.readFromFile(location.fileIndex, location.offset, fromFile)) return false;
|
|
708
665
|
const held = store.view.vectorAt(ordinal);
|
|
709
|
-
const
|
|
710
|
-
const
|
|
711
|
-
for (let i = 0; i <
|
|
666
|
+
const fileWords = new Uint32Array(fromFile.buffer, fromFile.byteOffset, fromFile.length);
|
|
667
|
+
const heldWords = new Uint32Array(held.buffer, held.byteOffset, held.length);
|
|
668
|
+
for (let i = 0; i < fileWords.length; i++) if (fileWords[i] !== heldWords[i]) return false;
|
|
712
669
|
return true;
|
|
713
670
|
}
|
|
714
671
|
function forgetUnreferencedFiles(store, documents, replacedOrdinals) {
|
|
@@ -759,6 +716,7 @@ function createVectorStore(options) {
|
|
|
759
716
|
const replacedOrdinals = /* @__PURE__ */ new Set();
|
|
760
717
|
let unknownPartitions = 0;
|
|
761
718
|
let liveCount = 0;
|
|
719
|
+
let readBackVector = null;
|
|
762
720
|
let open = options?.dimension === void 0 ? null : openHandles(options.dimension, codeBits, blockBytes);
|
|
763
721
|
function forgetDocuments() {
|
|
764
722
|
docToOrd.clear();
|
|
@@ -903,7 +861,10 @@ function createVectorStore(options) {
|
|
|
903
861
|
releaseToFile(ordinal, location) {
|
|
904
862
|
const store = requireOpen();
|
|
905
863
|
if (ordToDoc[ordinal] === void 0) return false;
|
|
906
|
-
if (store.diskFile[ordinal] === IN_MEMORY
|
|
864
|
+
if (store.diskFile[ordinal] === IN_MEMORY) {
|
|
865
|
+
readBackVector ??= new Float32Array(store.handles.dimension);
|
|
866
|
+
if (!fileHoldsSameBytes(store, ordinal, location, readBackVector)) return false;
|
|
867
|
+
}
|
|
907
868
|
Atomics.store(store.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 1);
|
|
908
869
|
store.diskOffset[ordinal] = location.offset;
|
|
909
870
|
store.diskFile[ordinal] = location.fileIndex;
|