@delali/narsil 0.3.1-next.128 → 0.3.1-next.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-2Y3CO4EQ.mjs} +878 -310
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-NSCSR6MA.mjs} +24 -9
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-UDGB2MWT.mjs} +7 -28
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-GZHR6AVG.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-Bl8G9taj.d.ts} +10 -3
- package/dist/{chunk-VK45J47R.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-QSCVJJW5.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-2MKX2SCO.mjs} +481 -121
- package/dist/{chunk-P34VZEXB.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-HFJXEFFV.mjs → chunk-4GDLFDPZ.mjs} +3 -3
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-ZR63OGT2.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-UKXMSO5H.mjs → chunk-6M6SDH3S.mjs} +189 -45
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-6SSVIW7Y.mjs} +49 -31
- package/dist/{chunk-4IHMMDRF.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-OBWUWZRA.mjs → chunk-CZ7ISLXA.mjs} +3 -0
- package/dist/{chunk-OFYKBKKV.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-KCQG2PJT.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-HWQDZB2U.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-FQBFZAWG.mjs} +4 -1
- package/dist/{chunk-U6NGTWI3.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-SJEIIR56.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-N57XXGPG.mjs → chunk-JT3YZGCE.mjs} +23 -4
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-JIXO442E.mjs → chunk-K77FJXKS.mjs} +26 -10
- package/dist/{chunk-4PQNDHSA.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-6PN3U6XW.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-542TD4BL.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-DG62H7OD.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-22CK2LQO.mjs → chunk-QQTXT65X.mjs} +6 -5
- package/dist/{chunk-LXWDQLTW.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RWESFSJI.mjs} +28 -13
- package/dist/{chunk-SEG4CL6W.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-OGFSUXYY.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-LOBYK2FG.mjs → chunk-T4OFG4IQ.mjs} +20 -9
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-TGHNXN7W.mjs} +150 -52
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-ANFNYCQG.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-62IPM62F.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-IE43SRL3.mjs → chunk-XABMCBHA.mjs} +12 -6
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-MFPJVITN.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-N6J6EJW3.mjs → chunk-YBWSV5DV.mjs} +2 -2
- package/dist/{chunk-FUUP45IS.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-CsuhZ13c.d.ts → client-Cx-TiqPE.d.ts} +19 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +10 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CzB9YvF_.d.ts} +2 -2
- package/dist/{config-DLVnlByZ.d.ts → config-hOWGBeC6.d.ts} +7 -5
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +411 -127
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-S7OX2MAI.mjs} +28 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-B_W9sdZK.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-DErop1LM.d.ts} +23 -2
- package/dist/{index-Cl5qQauj.d.ts → index-CKI1fDEl.d.ts} +37 -3
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-WVSMIBYP.mjs} +17 -15
- package/dist/{module-7OHZ2OPI.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +3 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -22
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +3 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-CS-clLun.d.ts} +54 -28
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-DOQLGGEX.mjs} +45 -30
- package/dist/synonyms.mjs +4 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-BfF-VRhm.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-DNK5XWXG.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-ZKR7SNRT.mjs} +23 -21
- package/package.json +7 -7
- package/dist/direct-executor-6KCVVHYT.mjs +0 -26
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { createBitSet, bitsetIsEmpty, GrowableUint32, bitsetHas, bitsetSet } from './chunk-JUKYYXSP.mjs';
|
|
2
|
+
import { MAX_QUERY_PHRASES, MAX_PHRASE_WORDS } from './chunk-FQBFZAWG.mjs';
|
|
3
|
+
import { PHRASE_ROWS_START, PHRASE_SORT_RADIX_BITS, PHRASE_SORT_CLOCK_ROWS } from './chunk-4UMZ7KQL.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
5
|
+
|
|
6
|
+
// src/core/partition/phrase-matching.ts
|
|
7
|
+
var UINT32_BITS = 32;
|
|
8
|
+
var WorkTally = class {
|
|
9
|
+
constructor(meter) {
|
|
10
|
+
this.meter = meter;
|
|
11
|
+
}
|
|
12
|
+
pending = 0;
|
|
13
|
+
add(units) {
|
|
14
|
+
this.pending += units;
|
|
15
|
+
if (this.meter !== void 0 && this.pending >= this.meter.flushUnits) this.flush();
|
|
16
|
+
}
|
|
17
|
+
tickClock(units) {
|
|
18
|
+
this.meter?.clock?.tick(units);
|
|
19
|
+
}
|
|
20
|
+
flush() {
|
|
21
|
+
const units = this.pending;
|
|
22
|
+
this.pending = 0;
|
|
23
|
+
this.meter?.add(units);
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
function placeRowsByDigit(docIds, from, to, shift, starts, tally) {
|
|
27
|
+
const count = from.length;
|
|
28
|
+
const mask = starts.length - 1;
|
|
29
|
+
starts.fill(0);
|
|
30
|
+
for (let row = 0; row < count; row++) starts[docIds[row] >>> shift & mask]++;
|
|
31
|
+
let next = 0;
|
|
32
|
+
for (let digit = 0; digit < starts.length; digit++) {
|
|
33
|
+
const held = starts[digit];
|
|
34
|
+
starts[digit] = next;
|
|
35
|
+
next += held;
|
|
36
|
+
}
|
|
37
|
+
for (let begin = 0; begin < count; begin += PHRASE_SORT_CLOCK_ROWS) {
|
|
38
|
+
const end = Math.min(count, begin + PHRASE_SORT_CLOCK_ROWS);
|
|
39
|
+
for (let at = begin; at < end; at++) {
|
|
40
|
+
const row = from[at];
|
|
41
|
+
to[starts[docIds[row] >>> shift & mask]++] = row;
|
|
42
|
+
}
|
|
43
|
+
tally.tickClock(end - begin);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function sortedRowOrder(docIds, count, tally) {
|
|
47
|
+
let order = new Uint32Array(count);
|
|
48
|
+
for (let row = 0; row < count; row++) order[row] = row;
|
|
49
|
+
let spare = new Uint32Array(count);
|
|
50
|
+
const starts = new Uint32Array(1 << PHRASE_SORT_RADIX_BITS);
|
|
51
|
+
for (let shift = 0; shift < UINT32_BITS; shift += PHRASE_SORT_RADIX_BITS) {
|
|
52
|
+
placeRowsByDigit(docIds, order, spare, shift, starts, tally);
|
|
53
|
+
const placed = spare;
|
|
54
|
+
spare = order;
|
|
55
|
+
order = placed;
|
|
56
|
+
}
|
|
57
|
+
return order;
|
|
58
|
+
}
|
|
59
|
+
function tokenRowsOf(list, tally) {
|
|
60
|
+
const store = list.positionStore ?? null;
|
|
61
|
+
if (store === null) return null;
|
|
62
|
+
const count = list.length;
|
|
63
|
+
const docIds = list.docIds;
|
|
64
|
+
if (list.ordered) {
|
|
65
|
+
return { list, keys: docIds, order: null, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
66
|
+
}
|
|
67
|
+
tally.add(count);
|
|
68
|
+
const order = sortedRowOrder(docIds, count, tally);
|
|
69
|
+
const keys = new Uint32Array(count);
|
|
70
|
+
for (let row = 0; row < count; row++) keys[row] = docIds[order[row]];
|
|
71
|
+
return { list, keys, order, fields: list.fieldNameIndices, store, count, cursor: 0, positionRow: 0 };
|
|
72
|
+
}
|
|
73
|
+
function resolvePhrase(index, phrase, sharedRows, tally) {
|
|
74
|
+
const entryRows = [];
|
|
75
|
+
const rows = [];
|
|
76
|
+
for (const token of phrase.tokens) {
|
|
77
|
+
let found = sharedRows.get(token);
|
|
78
|
+
if (found === void 0) {
|
|
79
|
+
const list = index.lookupReadOnce?.(token) ?? index.lookup(token);
|
|
80
|
+
if (list === void 0) return null;
|
|
81
|
+
const built = tokenRowsOf(list, tally);
|
|
82
|
+
if (built === null) return null;
|
|
83
|
+
found = built;
|
|
84
|
+
sharedRows.set(token, found);
|
|
85
|
+
}
|
|
86
|
+
if (!rows.includes(found)) rows.push(found);
|
|
87
|
+
entryRows.push(found);
|
|
88
|
+
}
|
|
89
|
+
let leadEntry = 0;
|
|
90
|
+
for (let entry = 1; entry < entryRows.length; entry++) {
|
|
91
|
+
if (entryRows[entry].count < entryRows[leadEntry].count) leadEntry = entry;
|
|
92
|
+
}
|
|
93
|
+
return { phrase, rows, entryRows, lead: entryRows[leadEntry], leadOffset: phrase.offsets[leadEntry] };
|
|
94
|
+
}
|
|
95
|
+
function firstRowFor(rows, doc) {
|
|
96
|
+
const { keys, count } = rows;
|
|
97
|
+
let low = rows.cursor < count && keys[rows.cursor] <= doc ? rows.cursor : 0;
|
|
98
|
+
let high = low;
|
|
99
|
+
let step = 1;
|
|
100
|
+
while (high < count && keys[high] < doc) {
|
|
101
|
+
low = high + 1;
|
|
102
|
+
high += step;
|
|
103
|
+
step *= 2;
|
|
104
|
+
}
|
|
105
|
+
if (high > count) high = count;
|
|
106
|
+
while (low < high) {
|
|
107
|
+
const middle = low + high >>> 1;
|
|
108
|
+
if (keys[middle] < doc) low = middle + 1;
|
|
109
|
+
else high = middle;
|
|
110
|
+
}
|
|
111
|
+
rows.cursor = low;
|
|
112
|
+
return low;
|
|
113
|
+
}
|
|
114
|
+
function positionRowFor(rows, doc, field) {
|
|
115
|
+
const { keys, order, fields, count } = rows;
|
|
116
|
+
for (let at = firstRowFor(rows, doc); at < count && keys[at] === doc; at++) {
|
|
117
|
+
const row = order === null ? at : order[at];
|
|
118
|
+
if (fields[row] === field) return row;
|
|
119
|
+
}
|
|
120
|
+
return -1;
|
|
121
|
+
}
|
|
122
|
+
function countPhraseStarts(resolved, leadRow, cursors, ends, tally) {
|
|
123
|
+
const { entryRows, lead, leadOffset, phrase } = resolved;
|
|
124
|
+
const offsets = phrase.offsets;
|
|
125
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
126
|
+
const rows = entryRows[entry];
|
|
127
|
+
const start = rows.list.positionStartAt?.(rows.positionRow) ?? 0;
|
|
128
|
+
cursors[entry] = start;
|
|
129
|
+
ends[entry] = start + (rows.list.positionCountAt?.(rows.positionRow) ?? 0);
|
|
130
|
+
}
|
|
131
|
+
const leadStore = lead.store;
|
|
132
|
+
const leadStart = lead.list.positionStartAt?.(leadRow) ?? 0;
|
|
133
|
+
const leadEnd = leadStart + (lead.list.positionCountAt?.(leadRow) ?? 0);
|
|
134
|
+
let starts = 0;
|
|
135
|
+
for (let at = leadStart; at < leadEnd; at++) {
|
|
136
|
+
const anchor = leadStore[at] - leadOffset;
|
|
137
|
+
if (anchor < 0) continue;
|
|
138
|
+
let holdsEveryEntry = true;
|
|
139
|
+
for (let entry = 0; entry < entryRows.length; entry++) {
|
|
140
|
+
const store = entryRows[entry].store;
|
|
141
|
+
const wanted = anchor + offsets[entry];
|
|
142
|
+
let cursor = cursors[entry];
|
|
143
|
+
const end = ends[entry];
|
|
144
|
+
let compared = 1;
|
|
145
|
+
while (cursor < end && store[cursor] < wanted) {
|
|
146
|
+
cursor++;
|
|
147
|
+
compared++;
|
|
148
|
+
}
|
|
149
|
+
cursors[entry] = cursor;
|
|
150
|
+
tally.add(compared);
|
|
151
|
+
if (cursor === end) {
|
|
152
|
+
return starts;
|
|
153
|
+
}
|
|
154
|
+
if (store[cursor] !== wanted) {
|
|
155
|
+
holdsEveryEntry = false;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (holdsEveryEntry) starts++;
|
|
160
|
+
}
|
|
161
|
+
return starts;
|
|
162
|
+
}
|
|
163
|
+
function collectPhraseRows(resolved, searchable, allowed, capacity, keepRows, tally) {
|
|
164
|
+
const hits = createBitSet(capacity);
|
|
165
|
+
const collected = {
|
|
166
|
+
phrase: resolved.phrase,
|
|
167
|
+
lists: resolved.entryRows.map((rows2) => rows2.list),
|
|
168
|
+
docIds: new GrowableUint32(PHRASE_ROWS_START),
|
|
169
|
+
fieldIndices: new GrowableUint32(PHRASE_ROWS_START),
|
|
170
|
+
frequencies: new GrowableUint32(PHRASE_ROWS_START)
|
|
171
|
+
};
|
|
172
|
+
const { lead, rows } = resolved;
|
|
173
|
+
const leadList = lead.list;
|
|
174
|
+
const docIds = leadList.docIds;
|
|
175
|
+
const fields = leadList.fieldNameIndices;
|
|
176
|
+
const deleted = leadList.deletedDocs;
|
|
177
|
+
const hasDeleted = deleted.size > 0;
|
|
178
|
+
const cursors = new Int32Array(resolved.entryRows.length);
|
|
179
|
+
const ends = new Int32Array(resolved.entryRows.length);
|
|
180
|
+
for (const each of rows) each.cursor = 0;
|
|
181
|
+
for (let leadRow = 0; leadRow < lead.count; leadRow++) {
|
|
182
|
+
tally.add(1);
|
|
183
|
+
const doc = docIds[leadRow];
|
|
184
|
+
const field = fields[leadRow];
|
|
185
|
+
if (searchable[field] === 0) continue;
|
|
186
|
+
if (hasDeleted && deleted.has(doc)) continue;
|
|
187
|
+
if (allowed !== void 0 && !bitsetHas(allowed, doc)) continue;
|
|
188
|
+
let holdsEveryToken = true;
|
|
189
|
+
for (const each of rows) {
|
|
190
|
+
if (each === lead) {
|
|
191
|
+
each.positionRow = leadRow;
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
const row = positionRowFor(each, doc, field);
|
|
195
|
+
if (row === -1) {
|
|
196
|
+
holdsEveryToken = false;
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
each.positionRow = row;
|
|
200
|
+
}
|
|
201
|
+
if (!holdsEveryToken) continue;
|
|
202
|
+
const starts = countPhraseStarts(resolved, leadRow, cursors, ends, tally);
|
|
203
|
+
if (starts === 0) continue;
|
|
204
|
+
bitsetSet(hits, doc);
|
|
205
|
+
if (keepRows) {
|
|
206
|
+
collected.docIds.push(doc);
|
|
207
|
+
collected.fieldIndices.push(field);
|
|
208
|
+
collected.frequencies.push(starts);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return { hits, rows: collected };
|
|
212
|
+
}
|
|
213
|
+
function matchPhrases(index, phrases, searchable, filterBitset, capacity, meter, keepRows) {
|
|
214
|
+
const tally = new WorkTally(meter);
|
|
215
|
+
const sharedRows = /* @__PURE__ */ new Map();
|
|
216
|
+
const resolved = [];
|
|
217
|
+
for (const phrase of phrases) {
|
|
218
|
+
const found = resolvePhrase(index, phrase, sharedRows, tally);
|
|
219
|
+
if (found === null) {
|
|
220
|
+
tally.flush();
|
|
221
|
+
return { matched: createBitSet(capacity), rows: [] };
|
|
222
|
+
}
|
|
223
|
+
resolved.push(found);
|
|
224
|
+
}
|
|
225
|
+
resolved.sort((first, second) => first.lead.count - second.lead.count);
|
|
226
|
+
const rows = [];
|
|
227
|
+
let allowed = filterBitset;
|
|
228
|
+
for (const each of resolved) {
|
|
229
|
+
const collected = collectPhraseRows(each, searchable, allowed, capacity, keepRows, tally);
|
|
230
|
+
allowed = collected.hits;
|
|
231
|
+
rows.push(collected.rows);
|
|
232
|
+
if (bitsetIsEmpty(allowed)) break;
|
|
233
|
+
}
|
|
234
|
+
tally.flush();
|
|
235
|
+
return { matched: allowed ?? createBitSet(capacity), rows };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// src/search/phrase-analysis.ts
|
|
239
|
+
function phraseKeyOf(tokens, offsets) {
|
|
240
|
+
const slots = [];
|
|
241
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
242
|
+
const gap = index === 0 ? 0 : offsets[index] - offsets[index - 1] - 1;
|
|
243
|
+
for (let missing = 0; missing < gap; missing++) slots.push("?");
|
|
244
|
+
slots.push(tokens[index]);
|
|
245
|
+
}
|
|
246
|
+
return `"${slots.join(" ")}"`;
|
|
247
|
+
}
|
|
248
|
+
function identityOf(tokens, offsets) {
|
|
249
|
+
return JSON.stringify([tokens, offsets]);
|
|
250
|
+
}
|
|
251
|
+
function analyseQueryPhrases(parts, analyseText) {
|
|
252
|
+
const wordParts = [];
|
|
253
|
+
const phrases = [];
|
|
254
|
+
const seen = /* @__PURE__ */ new Set();
|
|
255
|
+
let keptPhraseCount = 0;
|
|
256
|
+
let largestPhraseSize = 0;
|
|
257
|
+
let largestTokenCount = 0;
|
|
258
|
+
let wordsAfterLastPhrase = "";
|
|
259
|
+
for (let index = 0; index < parts.length; index++) {
|
|
260
|
+
const part = parts[index];
|
|
261
|
+
if (index % 2 === 0) {
|
|
262
|
+
wordParts.push(part);
|
|
263
|
+
wordsAfterLastPhrase = `${wordsAfterLastPhrase} ${part}`;
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
if (part.trim().length === 0) continue;
|
|
267
|
+
const entries = analyseText(part);
|
|
268
|
+
if (entries.length === 0) continue;
|
|
269
|
+
const ordered = [...entries].sort((first, second) => first.position - second.position);
|
|
270
|
+
const firstPosition = ordered[0].position;
|
|
271
|
+
const size = ordered[ordered.length - 1].position - firstPosition + 1;
|
|
272
|
+
keptPhraseCount++;
|
|
273
|
+
wordsAfterLastPhrase = "";
|
|
274
|
+
if (size > largestPhraseSize) largestPhraseSize = size;
|
|
275
|
+
if (ordered.length > largestTokenCount) largestTokenCount = ordered.length;
|
|
276
|
+
const tokens = ordered.map((entry) => entry.token);
|
|
277
|
+
const offsets = ordered.map((entry) => entry.position - firstPosition);
|
|
278
|
+
const identity = identityOf(tokens, offsets);
|
|
279
|
+
if (seen.has(identity)) continue;
|
|
280
|
+
seen.add(identity);
|
|
281
|
+
phrases.push({ key: phraseKeyOf(tokens, offsets), tokens, offsets });
|
|
282
|
+
}
|
|
283
|
+
const endsWithPhrase = keptPhraseCount > 0 && (wordsAfterLastPhrase.trim().length === 0 || analyseText(wordsAfterLastPhrase).length === 0);
|
|
284
|
+
return {
|
|
285
|
+
words: wordParts.join(" "),
|
|
286
|
+
phrases,
|
|
287
|
+
keptPhraseCount,
|
|
288
|
+
largestPhraseSize,
|
|
289
|
+
largestTokenCount,
|
|
290
|
+
endsWithPhrase
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
function withWordsOnly(params, phrases) {
|
|
294
|
+
return { ...params, term: phrases.words, prefix: phrases.endsWithPhrase ? false : params.prefix };
|
|
295
|
+
}
|
|
296
|
+
function refusePhraseUntilRebuilt(indexName, rebuildMode) {
|
|
297
|
+
const remedy = rebuildMode === "manual" ? "This engine leaves the rebuild to its caller, so somebody has to start it with rebuildAnalysis" : "Try the query again once the rebuild finishes";
|
|
298
|
+
throw new NarsilError(
|
|
299
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
300
|
+
`Index "${indexName}" holds array fields whose terms need a rebuild, so it cannot match a phrase yet. ${remedy}`,
|
|
301
|
+
{ indexName, rebuild: rebuildMode }
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
function requirePhrasesWithinLimits(phrases, trackPositions) {
|
|
305
|
+
if (phrases.keptPhraseCount === 0) return;
|
|
306
|
+
if (phrases.keptPhraseCount > MAX_QUERY_PHRASES) {
|
|
307
|
+
throw new NarsilError(
|
|
308
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
309
|
+
`A query takes at most ${MAX_QUERY_PHRASES} phrases, while this query holds ${phrases.keptPhraseCount}`,
|
|
310
|
+
{ limit: MAX_QUERY_PHRASES, phrases: phrases.keptPhraseCount }
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
if (phrases.largestPhraseSize > MAX_PHRASE_WORDS || phrases.largestTokenCount > MAX_PHRASE_WORDS) {
|
|
314
|
+
throw new NarsilError(
|
|
315
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
316
|
+
`A phrase takes at most ${MAX_PHRASE_WORDS} words, counting the stop words between its first and last word, while this query holds one of ${Math.max(phrases.largestPhraseSize, phrases.largestTokenCount)}`,
|
|
317
|
+
{ limit: MAX_PHRASE_WORDS, words: Math.max(phrases.largestPhraseSize, phrases.largestTokenCount) }
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
if (!trackPositions) {
|
|
321
|
+
throw new NarsilError(
|
|
322
|
+
ErrorCodes.SEARCH_INVALID_PHRASE,
|
|
323
|
+
"This index stores no word positions because its trackPositions setting is off, so it cannot match a phrase. Remove the quotes, or create the index with trackPositions on",
|
|
324
|
+
{ trackPositions: false }
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export { analyseQueryPhrases, matchPhrases, refusePhraseUntilRebuilt, requirePhrasesWithinLimits, withWordsOnly };
|
package/dist/react.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, P as PreflightResult, e as QueryResult, h as SuggestParams, j as SuggestResult } from './events-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-
|
|
3
|
-
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-
|
|
4
|
-
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-
|
|
1
|
+
import { N as NarsilError, L as ListParams, g as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, P as PreflightResult, e as QueryResult, h as SuggestParams, j as SuggestResult } from './events-DErop1LM.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-DErop1LM.js';
|
|
3
|
+
import { e as TaskRecord, d as TaskProgress, b as ImportResult, c as TaskListQuery, T as TaskListPage } from './tasks-BfF-VRhm.js';
|
|
4
|
+
export { I as ImportError, f as TaskStatus, g as TaskType } from './tasks-BfF-VRhm.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
|
-
import { N as Narsil } from './engine-
|
|
7
|
-
import { N as NarsilClient, I as ImportSource } from './client-
|
|
8
|
-
import { A as AnyDocument } from './schema-
|
|
6
|
+
import { N as Narsil } from './engine-B_W9sdZK.js';
|
|
7
|
+
import { N as NarsilClient, I as ImportSource } from './client-Cx-TiqPE.js';
|
|
8
|
+
import { A as AnyDocument } from './schema-CS-clLun.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
10
|
import './adapters-BQFQ4PRz.js';
|
|
11
11
|
|
package/dist/react.mjs
CHANGED
|
@@ -34,6 +34,9 @@ var ErrorCodes = {
|
|
|
34
34
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
35
35
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
36
36
|
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
37
|
+
SEARCH_INVALID_PHRASE: "SEARCH_INVALID_PHRASE",
|
|
38
|
+
SEARCH_PHRASE_NOT_READY: "SEARCH_PHRASE_NOT_READY",
|
|
39
|
+
BM25_NOT_READY: "BM25_NOT_READY",
|
|
37
40
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
38
41
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
39
42
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-SZGRYEIC.mjs';
|
|
2
2
|
import './chunk-KZ3NX2P7.mjs';
|
|
3
|
-
import { rerankOnHost } from './chunk-
|
|
4
|
-
import './chunk-
|
|
3
|
+
import { rerankOnHost } from './chunk-PATMDIMN.mjs';
|
|
4
|
+
import './chunk-5LX6MOIB.mjs';
|
|
5
5
|
import './chunk-HJSN7UYG.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import './chunk-RNSC4NFI.mjs';
|
|
7
|
+
import './chunk-FQBFZAWG.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
9
9
|
|
|
10
10
|
// src/distribution/cluster-node/reads/rerank.ts
|
|
11
11
|
async function rerankClusterQuery(deps, indexName, params, firstStage, readDocuments, host) {
|
package/dist/rerankers/http.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from '../chunk-
|
|
1
|
+
import { resolveApiKey, buildSignal, postJsonWithRetry } from '../chunk-2WBY4Y4M.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from '../chunk-CZ7ISLXA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/rerankers/constants.ts
|
|
5
5
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
@@ -18,7 +18,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
18
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
19
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
20
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
21
|
+
const { restoreFromSnapshot } = await import('./snapshot-DOQLGGEX.mjs');
|
|
22
22
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
23
|
});
|
|
24
24
|
const settled = restored.then(
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { runRerankedQuery, rerankTextOf } from './chunk-
|
|
1
|
+
import { runRerankedQuery, rerankTextOf } from './chunk-SZGRYEIC.mjs';
|
|
2
2
|
export { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
|
-
import { rerankOnHost } from './chunk-
|
|
4
|
-
export { rerankOnHost } from './chunk-
|
|
5
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
3
|
+
import { rerankOnHost } from './chunk-PATMDIMN.mjs';
|
|
4
|
+
export { rerankOnHost } from './chunk-PATMDIMN.mjs';
|
|
5
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-5LX6MOIB.mjs';
|
|
6
6
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { HYBRID_RESCORE_WINDOW } from './chunk-
|
|
9
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import './chunk-RNSC4NFI.mjs';
|
|
8
|
+
import { HYBRID_RESCORE_WINDOW } from './chunk-FQBFZAWG.mjs';
|
|
9
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
10
10
|
|
|
11
11
|
// src/engine/query/hybrid-depth.ts
|
|
12
12
|
function countOfUnion(text, vectorScored) {
|
|
@@ -1,6 +1,50 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
2
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* The BM25 parameters of one text field, which override the same parameters at
|
|
6
|
+
* the level that names the field.
|
|
7
|
+
*
|
|
8
|
+
* @public
|
|
9
|
+
*/
|
|
10
|
+
interface BM25FieldParams {
|
|
11
|
+
/** This saturates term frequency in this field. It takes a number from 0 to 1,000. */
|
|
12
|
+
k1?: number;
|
|
13
|
+
/** This normalises by the length of this field. It takes a number from 0 to 1. */
|
|
14
|
+
b?: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* BM25 tuning for an index or for one search.
|
|
18
|
+
*
|
|
19
|
+
* Both parameters default to the values the literature settles on, so change
|
|
20
|
+
* them only once you have measured relevance on your own corpus. For each
|
|
21
|
+
* field, the engine takes the first value that is set among the search's value
|
|
22
|
+
* for the field, the search's value, the index's value for the field, the
|
|
23
|
+
* index's value, and the default.
|
|
24
|
+
*
|
|
25
|
+
* @public
|
|
26
|
+
*/
|
|
27
|
+
interface BM25Params {
|
|
28
|
+
/**
|
|
29
|
+
* This saturates term frequency, so a higher value lets a repeated term
|
|
30
|
+
* keep raising a score. It defaults to 1.2, and any value that is not a
|
|
31
|
+
* number from 0 to 1,000 fails with `CONFIG_INVALID`.
|
|
32
|
+
*/
|
|
33
|
+
k1?: number;
|
|
34
|
+
/**
|
|
35
|
+
* This normalises by field length, from 0 for none to 1 for full. It
|
|
36
|
+
* defaults to 0.75, and any value that is not a number from 0 to 1 fails
|
|
37
|
+
* with `CONFIG_INVALID`.
|
|
38
|
+
*/
|
|
39
|
+
b?: number;
|
|
40
|
+
/**
|
|
41
|
+
* These parameters apply to the text fields that this map holds. A name that
|
|
42
|
+
* is not a text field that the schema declares fails with `CONFIG_INVALID`
|
|
43
|
+
* on an index and with `SEARCH_INVALID_FIELD` on a search.
|
|
44
|
+
*/
|
|
45
|
+
fields?: Record<string, BM25FieldParams>;
|
|
46
|
+
}
|
|
47
|
+
|
|
4
48
|
/**
|
|
5
49
|
* A document as the engine stores it: any set of fields, with an optional `id`.
|
|
6
50
|
*
|
|
@@ -190,17 +234,21 @@ interface IndexConfig {
|
|
|
190
234
|
partitions?: PartitionConfig;
|
|
191
235
|
/** A query scores this way unless it asks for another mode. */
|
|
192
236
|
defaultScoring?: ScoringMode;
|
|
193
|
-
/**
|
|
237
|
+
/**
|
|
238
|
+
* These parameters tune BM25 for this index and for each of its text fields.
|
|
239
|
+
* A search can override them with {@link QueryParams.bm25}, and
|
|
240
|
+
* {@link BM25Operations.updateBM25} replaces them on a live index.
|
|
241
|
+
*/
|
|
194
242
|
bm25?: BM25Params;
|
|
195
243
|
/** This replaces the language's stop words, either inline or by the name a stop word list registered under. */
|
|
196
244
|
stopWords?: StopWordOverride | string;
|
|
197
245
|
/** This replaces the language's tokeniser, either inline or by the name a tokeniser registered under. */
|
|
198
246
|
tokenizer?: CustomTokenizer | string;
|
|
199
247
|
/**
|
|
200
|
-
* Setting this records term positions in each posting, which
|
|
201
|
-
*
|
|
202
|
-
*
|
|
203
|
-
*
|
|
248
|
+
* Setting this records term positions in each posting, which phrase search
|
|
249
|
+
* compares. An index that opts out stores less, and the engine refuses a
|
|
250
|
+
* phrase on it with `SEARCH_INVALID_PHRASE`. Highlighting works either way,
|
|
251
|
+
* because the engine's highlighter analyses the stored text again.
|
|
204
252
|
*/
|
|
205
253
|
trackPositions?: boolean;
|
|
206
254
|
/** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
|
|
@@ -244,28 +292,6 @@ interface IndexConfig {
|
|
|
244
292
|
*/
|
|
245
293
|
synonyms?: SynonymRule[];
|
|
246
294
|
}
|
|
247
|
-
/**
|
|
248
|
-
* BM25 tuning for an index.
|
|
249
|
-
*
|
|
250
|
-
* Both parameters default to the values the literature settles on, so change
|
|
251
|
-
* them only once you have measured relevance on your own corpus.
|
|
252
|
-
*
|
|
253
|
-
* @public
|
|
254
|
-
*/
|
|
255
|
-
interface BM25Params {
|
|
256
|
-
/**
|
|
257
|
-
* This saturates term frequency, so a higher value lets a repeated term
|
|
258
|
-
* keep raising a score. It defaults to 1.2, and it must be a finite number
|
|
259
|
-
* at or above 0 or index creation fails with `CONFIG_INVALID`.
|
|
260
|
-
*/
|
|
261
|
-
k1?: number;
|
|
262
|
-
/**
|
|
263
|
-
* This normalises by field length, from 0 for none to 1 for full. It
|
|
264
|
-
* defaults to 0.75, and it must lie between 0 and 1 or index creation
|
|
265
|
-
* fails with `CONFIG_INVALID`.
|
|
266
|
-
*/
|
|
267
|
-
b?: number;
|
|
268
|
-
}
|
|
269
295
|
/**
|
|
270
296
|
* A tokeniser you supply, which replaces the language module's own.
|
|
271
297
|
*
|
|
@@ -356,4 +382,4 @@ interface InsertOptions extends WriteOptions {
|
|
|
356
382
|
skipClone?: boolean;
|
|
357
383
|
}
|
|
358
384
|
|
|
359
|
-
export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldOptionChain as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, TextFieldOption as T, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b,
|
|
385
|
+
export type { AnyDocument as A, BM25Params as B, CustomTokenizer as C, EmbeddingFieldConfig as E, FieldOptionChain as F, IndexConfig as I, PartitionConfig as P, SchemaDefinition as S, TextFieldOption as T, VectorIndexConfig as V, WriteOptions as W, InsertOptions as a, StopWordOverride as b, BM25FieldParams as c, FieldType as d, ScoringMode as e, VectorQuantizationMode as f, VectorStorageMode as g };
|
package/dist/server.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-
|
|
2
|
-
import { c as ServerOptions, N as NarsilServer } from './capabilities-
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
4
|
-
import { N as Narsil } from './engine-
|
|
5
|
-
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-
|
|
6
|
-
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-
|
|
7
|
-
import './schema-
|
|
1
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './events-DErop1LM.js';
|
|
2
|
+
import { c as ServerOptions, N as NarsilServer } from './capabilities-Bl8G9taj.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, d as RequestContext, e as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, f as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-Bl8G9taj.js';
|
|
4
|
+
import { N as Narsil } from './engine-B_W9sdZK.js';
|
|
5
|
+
import { d as TaskProgress, b as ImportResult, h as TaskStore, e as TaskRecord } from './tasks-BfF-VRhm.js';
|
|
6
|
+
export { C as CapabilitiesResponse, a as CreateIndexEmbedding, i as CreateIndexRequest, H as HttpIndexConfig, I as ImportError, M as MemoryStatsResponse, T as TaskListPage, c as TaskListQuery, f as TaskStatus, g as TaskType } from './tasks-BfF-VRhm.js';
|
|
7
|
+
import './schema-CS-clLun.js';
|
|
8
8
|
import './adapters-BQFQ4PRz.js';
|
|
9
9
|
import './synonyms-CpOAHPa-.js';
|
|
10
|
-
import './config-
|
|
11
|
-
import './config-
|
|
12
|
-
import './index-
|
|
10
|
+
import './config-CzB9YvF_.js';
|
|
11
|
+
import './config-hOWGBeC6.js';
|
|
12
|
+
import './index-CKI1fDEl.js';
|
|
13
13
|
import './types-DvncmvuQ.js';
|
|
14
14
|
|
|
15
15
|
/**
|
package/dist/server.mjs
CHANGED
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
import { answerRequest } from './chunk-
|
|
2
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-
|
|
3
|
-
export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
4
|
-
import { rerankOnHost } from './chunk-
|
|
5
|
-
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-
|
|
6
|
-
import { readMemoryFigures } from './chunk-
|
|
1
|
+
import { answerRequest } from './chunk-YBWSV5DV.mjs';
|
|
2
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-XABMCBHA.mjs';
|
|
3
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-XABMCBHA.mjs';
|
|
4
|
+
import { rerankOnHost } from './chunk-PATMDIMN.mjs';
|
|
5
|
+
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-T4OFG4IQ.mjs';
|
|
6
|
+
import { readMemoryFigures } from './chunk-4GDLFDPZ.mjs';
|
|
7
7
|
import './chunk-IWVEG5CX.mjs';
|
|
8
8
|
import './chunk-DJHHF2L5.mjs';
|
|
9
9
|
import './chunk-P36MUFEJ.mjs';
|
|
10
|
-
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-
|
|
10
|
+
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-JT3YZGCE.mjs';
|
|
11
11
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-ZR4VM4A6.mjs';
|
|
13
13
|
import './chunk-ZATRDAYU.mjs';
|
|
14
14
|
import './chunk-EBIHRL7B.mjs';
|
|
15
15
|
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
16
|
-
import './chunk-
|
|
16
|
+
import './chunk-RNSC4NFI.mjs';
|
|
17
17
|
import './chunk-TGP5M5R7.mjs';
|
|
18
18
|
import './chunk-U2BKYEC7.mjs';
|
|
19
|
-
import './chunk-
|
|
19
|
+
import './chunk-2C3WXBQS.mjs';
|
|
20
20
|
import './chunk-WQBCBKY3.mjs';
|
|
21
|
-
import './chunk-
|
|
21
|
+
import './chunk-PU6SKH7O.mjs';
|
|
22
22
|
import './chunk-U476IXKQ.mjs';
|
|
23
|
-
import './chunk-
|
|
23
|
+
import './chunk-FQBFZAWG.mjs';
|
|
24
24
|
import './chunk-VJS4JFBQ.mjs';
|
|
25
|
-
import './chunk-
|
|
25
|
+
import './chunk-7XY6VD7B.mjs';
|
|
26
26
|
import './chunk-7X5JST5L.mjs';
|
|
27
27
|
import './chunk-R6A773WK.mjs';
|
|
28
28
|
import './chunk-YTJCGKM4.mjs';
|
|
29
29
|
import { startIdleHeapCleanup } from './chunk-RIIUZJQ3.mjs';
|
|
30
30
|
import './chunk-22YPZVMG.mjs';
|
|
31
31
|
import './chunk-L5ORED7V.mjs';
|
|
32
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-
|
|
33
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
32
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-CZ7ISLXA.mjs';
|
|
33
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-CZ7ISLXA.mjs';
|
|
34
34
|
import { randomUUID } from 'crypto';
|
|
35
35
|
import { createReadStream } from 'fs';
|
|
36
36
|
import { MessageChannel } from 'worker_threads';
|
|
@@ -105,6 +105,16 @@ function createAdminHandlers(deps) {
|
|
|
105
105
|
respondError(ctx, err);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
|
+
async function updateBm25(ctx) {
|
|
109
|
+
const body = parseJson(ctx);
|
|
110
|
+
if (body === void 0) return;
|
|
111
|
+
try {
|
|
112
|
+
await engine.updateBM25(ctx.params[0], body);
|
|
113
|
+
respondJson(ctx, { ok: true });
|
|
114
|
+
} catch (err) {
|
|
115
|
+
respondError(ctx, err);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
108
118
|
async function memory(ctx) {
|
|
109
119
|
try {
|
|
110
120
|
const stats = await engine.getMemoryStats();
|
|
@@ -235,6 +245,7 @@ function createAdminHandlers(deps) {
|
|
|
235
245
|
partitionConfig,
|
|
236
246
|
readSynonyms,
|
|
237
247
|
replaceSynonyms,
|
|
248
|
+
updateBm25,
|
|
238
249
|
memory,
|
|
239
250
|
optimize,
|
|
240
251
|
rebalance,
|
|
@@ -1704,6 +1715,7 @@ function mainThreadHandlers(deps) {
|
|
|
1704
1715
|
partitionConfig: admin.partitionConfig,
|
|
1705
1716
|
readSynonyms: admin.readSynonyms,
|
|
1706
1717
|
replaceSynonyms: admin.replaceSynonyms,
|
|
1718
|
+
updateBm25: admin.updateBm25,
|
|
1707
1719
|
restore: admin.restore,
|
|
1708
1720
|
exportDocuments: documentExport.exportDocuments,
|
|
1709
1721
|
importExport: documentExport.importExport
|