@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,10 +1,72 @@
|
|
|
1
|
-
import { flattenSchema } from './chunk-
|
|
1
|
+
import { flattenSchema } from './chunk-KGOXHLLZ.mjs';
|
|
2
2
|
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
3
3
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
4
|
-
import { isWordIndexedBase } from './chunk-
|
|
5
|
-
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-
|
|
4
|
+
import { isWordIndexedBase } from './chunk-ZSWHPKGM.mjs';
|
|
5
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-NPNK57CA.mjs';
|
|
6
6
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
7
7
|
|
|
8
|
+
// src/core/partition/utils.ts
|
|
9
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
10
|
+
const existing = table.indexMap.get(fieldName);
|
|
11
|
+
if (existing !== void 0) return existing;
|
|
12
|
+
const idx = table.names.length;
|
|
13
|
+
table.names.push(fieldName);
|
|
14
|
+
table.indexMap.set(fieldName, idx);
|
|
15
|
+
return idx;
|
|
16
|
+
}
|
|
17
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
18
|
+
function getNestedValue(obj, path) {
|
|
19
|
+
const parts = path.split(".");
|
|
20
|
+
let current = obj;
|
|
21
|
+
for (let i = 0; i < parts.length; i++) {
|
|
22
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
23
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
24
|
+
current = current[parts[i]];
|
|
25
|
+
}
|
|
26
|
+
return current;
|
|
27
|
+
}
|
|
28
|
+
function stringArraysEqual(a, b) {
|
|
29
|
+
if (a === b) return true;
|
|
30
|
+
if (!a || !b) return false;
|
|
31
|
+
if (a.length !== b.length) return false;
|
|
32
|
+
for (let i = 0; i < a.length; i++) {
|
|
33
|
+
if (a[i] !== b[i]) return false;
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
38
|
+
for (const { path, base } of fields) {
|
|
39
|
+
if (!isWordIndexedBase(base)) continue;
|
|
40
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
41
|
+
const newVal = getNestedValue(newDoc, path);
|
|
42
|
+
if (base === "string") {
|
|
43
|
+
if (oldVal !== newVal) return true;
|
|
44
|
+
} else {
|
|
45
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
function parseVectorDimension(fieldType) {
|
|
51
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
52
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
53
|
+
}
|
|
54
|
+
function tokenizeOptions(options) {
|
|
55
|
+
return {
|
|
56
|
+
stem: true,
|
|
57
|
+
removeStopWords: true,
|
|
58
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
59
|
+
stopWordOverride: options?.stopWordOverride,
|
|
60
|
+
customTokenizer: options?.customTokenizer
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
function getFlatSchema(state, schema) {
|
|
64
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
65
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
66
|
+
state.lastSchemaRef = schema;
|
|
67
|
+
return state.flatSchemaCache;
|
|
68
|
+
}
|
|
69
|
+
|
|
8
70
|
// src/core/partition/suggestions.ts
|
|
9
71
|
function verbatimOccurrences(state, token) {
|
|
10
72
|
const postingList = state.invertedIdx.lookup(token);
|
|
@@ -167,68 +229,6 @@ var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
|
167
229
|
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
168
230
|
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
169
231
|
|
|
170
|
-
// src/core/partition/utils.ts
|
|
171
|
-
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
172
|
-
const existing = table.indexMap.get(fieldName);
|
|
173
|
-
if (existing !== void 0) return existing;
|
|
174
|
-
const idx = table.names.length;
|
|
175
|
-
table.names.push(fieldName);
|
|
176
|
-
table.indexMap.set(fieldName, idx);
|
|
177
|
-
return idx;
|
|
178
|
-
}
|
|
179
|
-
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
180
|
-
function getNestedValue(obj, path) {
|
|
181
|
-
const parts = path.split(".");
|
|
182
|
-
let current = obj;
|
|
183
|
-
for (let i = 0; i < parts.length; i++) {
|
|
184
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
185
|
-
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
186
|
-
current = current[parts[i]];
|
|
187
|
-
}
|
|
188
|
-
return current;
|
|
189
|
-
}
|
|
190
|
-
function stringArraysEqual(a, b) {
|
|
191
|
-
if (a === b) return true;
|
|
192
|
-
if (!a || !b) return false;
|
|
193
|
-
if (a.length !== b.length) return false;
|
|
194
|
-
for (let i = 0; i < a.length; i++) {
|
|
195
|
-
if (a[i] !== b[i]) return false;
|
|
196
|
-
}
|
|
197
|
-
return true;
|
|
198
|
-
}
|
|
199
|
-
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
200
|
-
for (const { path, base } of fields) {
|
|
201
|
-
if (!isWordIndexedBase(base)) continue;
|
|
202
|
-
const oldVal = getNestedValue(oldDoc, path);
|
|
203
|
-
const newVal = getNestedValue(newDoc, path);
|
|
204
|
-
if (base === "string") {
|
|
205
|
-
if (oldVal !== newVal) return true;
|
|
206
|
-
} else {
|
|
207
|
-
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
return false;
|
|
211
|
-
}
|
|
212
|
-
function parseVectorDimension(fieldType) {
|
|
213
|
-
const match = VECTOR_PATTERN.exec(fieldType);
|
|
214
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
215
|
-
}
|
|
216
|
-
function tokenizeOptions(options) {
|
|
217
|
-
return {
|
|
218
|
-
stem: true,
|
|
219
|
-
removeStopWords: true,
|
|
220
|
-
collectSurfaces: options?.collectSurfaces === true,
|
|
221
|
-
stopWordOverride: options?.stopWordOverride,
|
|
222
|
-
customTokenizer: options?.customTokenizer
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
function getFlatSchema(state, schema) {
|
|
226
|
-
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
227
|
-
state.flatSchemaCache = flattenSchema(schema);
|
|
228
|
-
state.lastSchemaRef = schema;
|
|
229
|
-
return state.flatSchemaCache;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
232
|
// src/core/partition/partial-words/spellings.ts
|
|
233
233
|
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
234
234
|
function codePointCountOf(text) {
|
|
@@ -3,6 +3,8 @@ import { DEFAULT_PAGE_SIZE } from './chunk-FQBFZAWG.mjs';
|
|
|
3
3
|
// src/engine/constants.ts
|
|
4
4
|
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
5
5
|
var BATCH_CHUNK_SIZE = 1e3;
|
|
6
|
+
var ANALYSIS_HOLD_APPLY_INTERVAL_MS = 100;
|
|
7
|
+
var ANALYSIS_CHANGE_MEMBERS = ["language", "tokenizer", "stopWords"];
|
|
6
8
|
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
9
|
var DEFAULT_OFFSET = 0;
|
|
8
10
|
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
@@ -27,4 +29,4 @@ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
|
27
29
|
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
28
30
|
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
29
31
|
|
|
30
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|
|
32
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { readSync, openSync, read, closeSync } from 'fs';
|
|
2
|
+
|
|
3
|
+
// src/vector/vector-store/vector-file.ts
|
|
4
|
+
function createVectorFileReader() {
|
|
5
|
+
const descriptors = /* @__PURE__ */ new Map();
|
|
6
|
+
const readsWaiting = /* @__PURE__ */ new Map();
|
|
7
|
+
const closingAfterReads = /* @__PURE__ */ new Set();
|
|
8
|
+
function descriptorOf(path) {
|
|
9
|
+
const open = descriptors.get(path);
|
|
10
|
+
if (open !== void 0) return open;
|
|
11
|
+
try {
|
|
12
|
+
const descriptor = openSync(path, "r");
|
|
13
|
+
descriptors.set(path, descriptor);
|
|
14
|
+
return descriptor;
|
|
15
|
+
} catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
function closeNow(descriptor) {
|
|
20
|
+
try {
|
|
21
|
+
closeSync(descriptor);
|
|
22
|
+
} catch {
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function closeDescriptor(path, descriptor) {
|
|
26
|
+
descriptors.delete(path);
|
|
27
|
+
if ((readsWaiting.get(descriptor) ?? 0) > 0) closingAfterReads.add(descriptor);
|
|
28
|
+
else closeNow(descriptor);
|
|
29
|
+
}
|
|
30
|
+
function finishRead(descriptor) {
|
|
31
|
+
const waiting = (readsWaiting.get(descriptor) ?? 1) - 1;
|
|
32
|
+
if (waiting > 0) {
|
|
33
|
+
readsWaiting.set(descriptor, waiting);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
readsWaiting.delete(descriptor);
|
|
37
|
+
if (closingAfterReads.delete(descriptor)) closeNow(descriptor);
|
|
38
|
+
}
|
|
39
|
+
function readRest(descriptor, offset, target, done, finish) {
|
|
40
|
+
read(descriptor, target, done, target.byteLength - done, offset + done, (error, chunk) => {
|
|
41
|
+
if (error !== null || chunk <= 0) finish(false);
|
|
42
|
+
else if (done + chunk === target.byteLength) finish(true);
|
|
43
|
+
else readRest(descriptor, offset, target, done + chunk, finish);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
readInto(path, offset, target) {
|
|
48
|
+
const descriptor = descriptorOf(path);
|
|
49
|
+
if (descriptor === null) return false;
|
|
50
|
+
let read = 0;
|
|
51
|
+
while (read < target.byteLength) {
|
|
52
|
+
let chunk;
|
|
53
|
+
try {
|
|
54
|
+
chunk = readSync(descriptor, target, read, target.byteLength - read, offset + read);
|
|
55
|
+
} catch {
|
|
56
|
+
closeDescriptor(path, descriptor);
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
if (chunk <= 0) return false;
|
|
60
|
+
read += chunk;
|
|
61
|
+
}
|
|
62
|
+
return true;
|
|
63
|
+
},
|
|
64
|
+
readLater(path, offset, target) {
|
|
65
|
+
const descriptor = descriptorOf(path);
|
|
66
|
+
if (descriptor === null) return Promise.resolve(false);
|
|
67
|
+
if (target.byteLength === 0) return Promise.resolve(true);
|
|
68
|
+
readsWaiting.set(descriptor, (readsWaiting.get(descriptor) ?? 0) + 1);
|
|
69
|
+
return new Promise((resolve) => {
|
|
70
|
+
readRest(descriptor, offset, target, 0, (whole) => {
|
|
71
|
+
finishRead(descriptor);
|
|
72
|
+
resolve(whole);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
},
|
|
76
|
+
retainOnly(paths) {
|
|
77
|
+
for (const [path, descriptor] of descriptors) {
|
|
78
|
+
if (!paths.has(path)) closeDescriptor(path, descriptor);
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
close() {
|
|
82
|
+
for (const [path, descriptor] of descriptors) closeDescriptor(path, descriptor);
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export { createVectorFileReader };
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-DXSJ5LT2.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|