@delali/narsil 0.3.1-next.129 → 0.3.1-next.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-5D6NJD6C.mjs → chunk-2Y3CO4EQ.mjs} +342 -201
- package/dist/browser/{chunk-WLD6DDIX.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-YAGXZU4W.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-WZBTVTSU.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-XTFJZHQP.mjs → chunk-NSCSR6MA.mjs} +15 -5
- package/dist/browser/{chunk-3NVZD52E.mjs → chunk-UDGB2MWT.mjs} +5 -28
- package/dist/browser/{chunk-II5POT55.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-HCCC6BJB.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-S2VDU2JE.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-BZ7WWEC4.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-PL4TTFJE.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-RTUCHI2L.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-IMU7G5NU.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-BTjv-tK8.d.ts → capabilities-B6WxrWxt.d.ts} +3 -10
- package/dist/{chunk-3XHFVYIQ.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-37RTD6KD.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-K76F5YYX.mjs → chunk-2MKX2SCO.mjs} +219 -58
- package/dist/{chunk-AN3CUMPZ.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-OPO5SE4J.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-PTIJAZWL.mjs → chunk-4GDLFDPZ.mjs} +1 -1
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-TY64A3ZW.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-3LJVM6XM.mjs → chunk-5W7AEVTH.mjs} +2 -2
- package/dist/{chunk-PCRWCOZC.mjs → chunk-6M6SDH3S.mjs} +43 -23
- package/dist/{chunk-MGD7XETB.mjs → chunk-6SSVIW7Y.mjs} +19 -3
- package/dist/{chunk-5LC5VMDQ.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-4XEVROSP.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-WI5Q3L43.mjs → chunk-CZ7ISLXA.mjs} +2 -0
- package/dist/{chunk-RC27KF7F.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-F4UKMTKM.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-CTF6WFTX.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-DX2FI4LU.mjs → chunk-FQBFZAWG.mjs} +3 -1
- package/dist/{chunk-XTER67YM.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-WQYB4TNR.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-2HQQ7Y5D.mjs → chunk-JT3YZGCE.mjs} +2 -2
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-WUIDPBSY.mjs → chunk-K77FJXKS.mjs} +17 -7
- package/dist/{chunk-CSSCLRGY.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-44TB6AIZ.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-EZIGAOUJ.mjs → chunk-OGLV4I3L.mjs} +6 -5
- package/dist/{chunk-FRSWYMXH.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-AFOZR4ZQ.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/{chunk-TB7YMDD5.mjs → chunk-RCMNL7RI.mjs} +650 -100
- package/dist/{chunk-Y4NOHICF.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-RYH3PASD.mjs → chunk-RWESFSJI.mjs} +21 -11
- package/dist/{chunk-QQS4QZQA.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-K73PZTBR.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-43Q7HAI2.mjs → chunk-T4OFG4IQ.mjs} +4 -4
- package/dist/{chunk-2YOAGAGF.mjs → chunk-TGHNXN7W.mjs} +59 -30
- package/dist/{chunk-J4LRE5Q7.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-XRXBCOA4.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-Y4US5EGH.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-5VSTAJET.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-M6MBSVIJ.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-BOcveV7A.d.ts → client-gNAfaHcE.d.ts} +3 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +2 -0
- package/dist/{config-CPmIY3aH.d.ts → config-CmYJO34X.d.ts} +3 -3
- package/dist/{events-pM0lna7M.d.ts → config-D1cDv3sE.d.ts} +431 -3
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -8
- package/dist/distribution.mjs +85 -45
- package/dist/{durability-wiring-6R26AJDI.mjs → durability-wiring-S7OX2MAI.mjs} +27 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-JjRAXpT3.d.ts → engine-m_Ddr1bz.d.ts} +2 -2
- package/dist/{index-DkQ2DlgR.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -18
- package/dist/{long-query-thread-6VLA6YJI.mjs → long-query-thread-BSL25PD6.mjs} +17 -15
- package/dist/{module-S4WGMFNW.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -21
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +2 -0
- package/dist/{rerank-V4J4MIWR.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-PUGWCPCX.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-TBFBQQQO.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-DGAP_8HJ.d.ts → schema-BTXbHCqA.d.ts} +8 -5
- package/dist/server.d.ts +198 -11
- package/dist/server.mjs +30 -19
- package/dist/{snapshot-PUVSMV7E.mjs → snapshot-DOQLGGEX.mjs} +44 -29
- package/dist/synonyms.mjs +4 -3
- package/dist/tasks-Co8HfRD7.d.ts +704 -0
- package/dist/{thread-TA7QQBWP.mjs → thread-SKO7CN3R.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-TGKYT3C4.mjs → write-ZKR7SNRT.mjs} +22 -20
- package/openapi.json +5215 -0
- package/package.json +16 -8
- package/dist/config-C3YXk5Rn.d.ts +0 -412
- package/dist/direct-executor-ULA6V5HC.mjs +0 -26
- package/dist/tasks-txGvNz6h.d.ts +0 -302
|
@@ -1,70 +1,9 @@
|
|
|
1
1
|
import { MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
2
|
+
import { bitsetHas, bitsetSet, GrowableUint32, createBitSet, bitsetAnd, bitsetNot } from './chunk-JUKYYXSP.mjs';
|
|
2
3
|
import { tabulatedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
3
|
-
import { parsedTypeOf } from './chunk-
|
|
4
|
+
import { parsedTypeOf } from './chunk-7XY6VD7B.mjs';
|
|
4
5
|
import { codePointCount, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
-
|
|
7
|
-
// src/core/bitset.ts
|
|
8
|
-
function createBitSet(capacity) {
|
|
9
|
-
return new Uint32Array(capacity + 31 >>> 5 || 1);
|
|
10
|
-
}
|
|
11
|
-
function bitsetSet(bs, index) {
|
|
12
|
-
const word = index >>> 5;
|
|
13
|
-
if (word < bs.length) {
|
|
14
|
-
bs[word] |= 1 << (index & 31);
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
function bitsetHas(bs, index) {
|
|
18
|
-
const word = index >>> 5;
|
|
19
|
-
if (word >= bs.length) return false;
|
|
20
|
-
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
21
|
-
}
|
|
22
|
-
function bitsetAnd(a, b) {
|
|
23
|
-
const len = Math.min(a.length, b.length);
|
|
24
|
-
const result = new Uint32Array(len);
|
|
25
|
-
for (let i = 0; i < len; i++) {
|
|
26
|
-
result[i] = a[i] & b[i];
|
|
27
|
-
}
|
|
28
|
-
return result;
|
|
29
|
-
}
|
|
30
|
-
function bitsetAndInto(target, other) {
|
|
31
|
-
const shared = Math.min(target.length, other.length);
|
|
32
|
-
for (let i = 0; i < shared; i++) target[i] &= other[i];
|
|
33
|
-
target.fill(0, shared);
|
|
34
|
-
}
|
|
35
|
-
function bitsetOrInto(target, other) {
|
|
36
|
-
const shared = Math.min(target.length, other.length);
|
|
37
|
-
for (let i = 0; i < shared; i++) target[i] |= other[i];
|
|
38
|
-
}
|
|
39
|
-
function bitsetAndNotInto(target, removed) {
|
|
40
|
-
const shared = Math.min(target.length, removed.length);
|
|
41
|
-
for (let i = 0; i < shared; i++) target[i] &= ~removed[i];
|
|
42
|
-
}
|
|
43
|
-
function bitsetNot(bs, capacity) {
|
|
44
|
-
const result = new Uint32Array(bs.length);
|
|
45
|
-
const fullWords = capacity >>> 5;
|
|
46
|
-
for (let i = 0; i < fullWords; i++) {
|
|
47
|
-
result[i] = ~bs[i];
|
|
48
|
-
}
|
|
49
|
-
const remainder = capacity & 31;
|
|
50
|
-
if (remainder > 0 && fullWords < bs.length) {
|
|
51
|
-
result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
|
|
52
|
-
}
|
|
53
|
-
return result;
|
|
54
|
-
}
|
|
55
|
-
function bitsetIsEmpty(bs) {
|
|
56
|
-
for (let i = 0; i < bs.length; i++) {
|
|
57
|
-
if (bs[i] !== 0) return false;
|
|
58
|
-
}
|
|
59
|
-
return true;
|
|
60
|
-
}
|
|
61
|
-
function bitsetFromSet(set, capacity) {
|
|
62
|
-
const bs = createBitSet(capacity);
|
|
63
|
-
for (const id of set) {
|
|
64
|
-
bs[id >>> 5] |= 1 << (id & 31);
|
|
65
|
-
}
|
|
66
|
-
return bs;
|
|
67
|
-
}
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
68
7
|
|
|
69
8
|
// src/core/pattern-index/constants.ts
|
|
70
9
|
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|
|
@@ -2035,36 +1974,6 @@ function largestPatternFootprintIn(expression, patternFields, host) {
|
|
|
2035
1974
|
return largest;
|
|
2036
1975
|
}
|
|
2037
1976
|
|
|
2038
|
-
// src/core/partition/growable-uint32.ts
|
|
2039
|
-
var GrowableUint32 = class {
|
|
2040
|
-
values;
|
|
2041
|
-
length = 0;
|
|
2042
|
-
constructor(capacity) {
|
|
2043
|
-
this.values = new Uint32Array(capacity);
|
|
2044
|
-
}
|
|
2045
|
-
growTo(needed, kept) {
|
|
2046
|
-
let capacity = this.values.length * 2;
|
|
2047
|
-
while (capacity < needed) capacity *= 2;
|
|
2048
|
-
const next = new Uint32Array(capacity);
|
|
2049
|
-
next.set(this.values.subarray(0, kept));
|
|
2050
|
-
this.values = next;
|
|
2051
|
-
}
|
|
2052
|
-
reserve(extra) {
|
|
2053
|
-
const needed = this.length + extra;
|
|
2054
|
-
if (needed > this.values.length) this.growTo(needed, this.length);
|
|
2055
|
-
}
|
|
2056
|
-
push(value) {
|
|
2057
|
-
if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
|
|
2058
|
-
this.values[this.length++] = value;
|
|
2059
|
-
}
|
|
2060
|
-
ensureIndex(index) {
|
|
2061
|
-
if (index >= this.values.length) this.growTo(index + 1, this.values.length);
|
|
2062
|
-
}
|
|
2063
|
-
clear() {
|
|
2064
|
-
this.length = 0;
|
|
2065
|
-
}
|
|
2066
|
-
};
|
|
2067
|
-
|
|
2068
1977
|
// src/pattern/runs-arrays.ts
|
|
2069
1978
|
var EMPTY_POSTINGS = new Uint32Array(0);
|
|
2070
1979
|
function remapAscending(source, remap, out) {
|
|
@@ -2596,4 +2505,4 @@ var patternSearchModule = {
|
|
|
2596
2505
|
decodeIndex: decodePatternIndex
|
|
2597
2506
|
};
|
|
2598
2507
|
|
|
2599
|
-
export { DEFAULT_PATTERN_WORK_CAP,
|
|
2508
|
+
export { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES, PATTERN_ELEMENT_OPERATORS, PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_WORK_FLUSH_UNITS, TYPO_CLAIM_SLOTS_PER_WORD, invalid, isFiniteNumber, isObjectRecord, isPatternTolerance, kindOfValue, patternSearchModule, requireFieldKind, requireListLength, requireTextRangeField };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { admittedHeapBytes, collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
2
2
|
import { readOldGenerationStatistics, processLimitBytes } from './chunk-ZATRDAYU.mjs';
|
|
3
3
|
import { engineThreadYoungGenerationBytes } from './chunk-EBIHRL7B.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/cluster-node/constants.ts
|
|
7
7
|
var DEFAULT_PARTITION_COUNT = 5;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// src/core/partition/constants.ts
|
|
2
|
+
var TARGET_BLOCK_ENTRIES = 128;
|
|
3
|
+
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
4
|
+
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
5
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
6
|
+
var PHRASE_ROWS_START = 16;
|
|
7
|
+
var PHRASE_SORT_RADIX_BITS = 11;
|
|
8
|
+
var PHRASE_SORT_CLOCK_ROWS = 65536;
|
|
9
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
10
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
11
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
12
|
+
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
13
|
+
var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
|
|
14
|
+
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
15
|
+
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
16
|
+
|
|
17
|
+
export { FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, LOOKUP_KEY_SCRATCH_BYTES, PHRASE_ROWS_START, PHRASE_SORT_CLOCK_ROWS, PHRASE_SORT_RADIX_BITS, PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-FQBFZAWG.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/search/hybrid-depth.ts
|
|
5
5
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-
|
|
2
|
-
import { ServerErrorCodes } from './chunk-
|
|
1
|
+
import { sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-RCMNL7RI.mjs';
|
|
2
|
+
import { ServerErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
3
3
|
import { rm } from 'fs/promises';
|
|
4
4
|
import path from 'path';
|
|
5
5
|
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, fanOutQuery, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-
|
|
1
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-2MKX2SCO.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-RNSC4NFI.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
|
|
4
|
-
import { flattenSchema, partialFieldsOf } from './chunk-
|
|
5
|
-
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-PU6SKH7O.mjs';
|
|
5
|
+
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import { tokenize } from './chunk-
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-7XY6VD7B.mjs';
|
|
8
|
+
import { tokenize } from './chunk-D5TFRB5G.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -509,6 +509,13 @@ function validQueryFilter(filters, schema) {
|
|
|
509
509
|
if (filters === void 0 || filters === null) return filters;
|
|
510
510
|
return validatedFilterOf(filters, flattenSchema(schema));
|
|
511
511
|
}
|
|
512
|
+
function requireUsablePhrases(params, context) {
|
|
513
|
+
if (params.term === void 0) return;
|
|
514
|
+
const quoted = analyseQueryPhrases(params.term, context.language, context.manager.analysis);
|
|
515
|
+
if (quoted === null || quoted.keptPhraseCount === 0) return;
|
|
516
|
+
requireLoadedPhraseCode().requirePhrasesWithinLimits(quoted, context.config.trackPositions !== false);
|
|
517
|
+
context.requirePhraseReady?.();
|
|
518
|
+
}
|
|
512
519
|
function requireVectorSearchable(params, context, needsVector) {
|
|
513
520
|
if (!needsVector) return;
|
|
514
521
|
const field = params.vector?.field;
|
|
@@ -1057,8 +1064,8 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
1057
1064
|
}
|
|
1058
1065
|
|
|
1059
1066
|
// src/engine/query/highlight.ts
|
|
1060
|
-
function writtenWordsOf(
|
|
1061
|
-
return tokenize(
|
|
1067
|
+
function writtenWordsOf(term, language, analysis) {
|
|
1068
|
+
return tokenize(term, language, {
|
|
1062
1069
|
stem: false,
|
|
1063
1070
|
removeStopWords: true,
|
|
1064
1071
|
customTokenizer: analysis.customTokenizer,
|
|
@@ -1076,18 +1083,21 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
1076
1083
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
1077
1084
|
};
|
|
1078
1085
|
}
|
|
1079
|
-
function applyHighlights(hits,
|
|
1080
|
-
if (!
|
|
1081
|
-
const
|
|
1086
|
+
function applyHighlights(hits, request, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
1087
|
+
if (!request.highlight) return;
|
|
1088
|
+
const quoted = analyseQueryPhrases(request.term ?? "", language, analysis);
|
|
1089
|
+
const params = quoted === null ? request : requireLoadedPhraseCode().withWordsOnly(request, quoted);
|
|
1090
|
+
const words = params.term ?? "";
|
|
1091
|
+
const highlightConfig = request.highlight;
|
|
1092
|
+
const queryTokens = quoted !== null && words.trim().length === 0 ? [] : tokenize(words, language, {
|
|
1082
1093
|
stem: true,
|
|
1083
1094
|
removeStopWords: true,
|
|
1084
1095
|
customTokenizer: analysis.customTokenizer,
|
|
1085
1096
|
stopWordOverride: analysis.stopWords
|
|
1086
|
-
});
|
|
1087
|
-
const queryTokens = queryTokenResult.tokens;
|
|
1097
|
+
}).tokens;
|
|
1088
1098
|
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
1089
1099
|
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
1090
|
-
const written = usesPrefix || usesPartialWords ? writtenWordsOf(
|
|
1100
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(words, language, analysis) : [];
|
|
1091
1101
|
let prefixToken;
|
|
1092
1102
|
let completedToken;
|
|
1093
1103
|
if (usesPrefix) {
|
|
@@ -1101,12 +1111,14 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
1101
1111
|
}
|
|
1102
1112
|
}
|
|
1103
1113
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
1104
|
-
const
|
|
1114
|
+
const wordSynonyms = params.exact === true || queryTokens.length === 0 ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
1115
|
+
const phraseTokens = quoted?.phrases.flatMap((phrase) => phrase.tokens) ?? [];
|
|
1116
|
+
const synonyms = phraseTokens.length === 0 ? wordSynonyms : [...wordSynonyms ?? [], ...phraseTokens];
|
|
1105
1117
|
for (const hit of hits) {
|
|
1106
1118
|
const highlights = {};
|
|
1107
1119
|
const projected = hit.document;
|
|
1108
1120
|
let stored;
|
|
1109
|
-
for (const field of
|
|
1121
|
+
for (const field of highlightConfig.fields) {
|
|
1110
1122
|
let fieldValue = projected[field];
|
|
1111
1123
|
if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
|
|
1112
1124
|
stored ??= readStoredDocument(hit.id);
|
|
@@ -1114,9 +1126,9 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
1114
1126
|
}
|
|
1115
1127
|
if (typeof fieldValue === "string") {
|
|
1116
1128
|
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
1117
|
-
preTag:
|
|
1118
|
-
postTag:
|
|
1119
|
-
maxSnippetLength:
|
|
1129
|
+
preTag: highlightConfig.preTag,
|
|
1130
|
+
postTag: highlightConfig.postTag,
|
|
1131
|
+
maxSnippetLength: highlightConfig.maxSnippetLength,
|
|
1120
1132
|
prefixToken,
|
|
1121
1133
|
completedToken,
|
|
1122
1134
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
@@ -1538,13 +1550,17 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1538
1550
|
}
|
|
1539
1551
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1540
1552
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1541
|
-
return import('./runtime-
|
|
1553
|
+
return import('./runtime-7MLQ7J64.mjs').then(
|
|
1542
1554
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1543
1555
|
);
|
|
1544
1556
|
}
|
|
1545
1557
|
function executeQuery(request, callerContext) {
|
|
1546
|
-
if (request.rerank === void 0)
|
|
1547
|
-
|
|
1558
|
+
if (request.rerank === void 0) {
|
|
1559
|
+
const loadingPhraseCode = loadPhraseCodeFor(request.term);
|
|
1560
|
+
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
1561
|
+
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
1562
|
+
}
|
|
1563
|
+
return import('./runtime-7MLQ7J64.mjs').then(
|
|
1548
1564
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1549
1565
|
);
|
|
1550
1566
|
}
|
|
@@ -1571,6 +1587,7 @@ async function runQuery(request, callerContext) {
|
|
|
1571
1587
|
}
|
|
1572
1588
|
const params = validQueryParams(request, config);
|
|
1573
1589
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1590
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
1574
1591
|
const projection = resolveProjection(params.document);
|
|
1575
1592
|
let paginated;
|
|
1576
1593
|
let nextCursor;
|
|
@@ -1747,6 +1764,9 @@ async function executePreflight(request, callerContext) {
|
|
|
1747
1764
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
1748
1765
|
const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
1749
1766
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
1767
|
+
const loadingPhraseCode = loadPhraseCodeFor(params.term);
|
|
1768
|
+
if (loadingPhraseCode !== null) await loadingPhraseCode;
|
|
1769
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
1750
1770
|
const requestedVectorField = params.vector?.field;
|
|
1751
1771
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
1752
1772
|
let totalMatched;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-D5TFRB5G.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
7
7
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -55,10 +55,26 @@ function getStopWords(name) {
|
|
|
55
55
|
}
|
|
56
56
|
return stopWords;
|
|
57
57
|
}
|
|
58
|
+
function holdsPositionsOutOfOrder(entries) {
|
|
59
|
+
for (let index = 1; index < entries.length; index++) {
|
|
60
|
+
if (entries[index].position < entries[index - 1].position) return true;
|
|
61
|
+
}
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
function inPositionOrder(tokenizer) {
|
|
65
|
+
return {
|
|
66
|
+
tokenize(text) {
|
|
67
|
+
const entries = tokenizer.tokenize(text);
|
|
68
|
+
if (!holdsPositionsOutOfOrder(entries)) return entries;
|
|
69
|
+
return [...entries].sort((first, second) => first.position - second.position);
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
}
|
|
58
73
|
function resolveIndexAnalysis(config) {
|
|
74
|
+
const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
|
|
59
75
|
return {
|
|
60
76
|
stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
|
|
61
|
-
customTokenizer:
|
|
77
|
+
customTokenizer: tokenizer === void 0 ? void 0 : inPositionOrder(tokenizer)
|
|
62
78
|
};
|
|
63
79
|
}
|
|
64
80
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
3
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-TGHNXN7W.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-EUCX5K74.mjs';
|
|
3
|
+
import { requireValidFieldTypes } from './chunk-PU6SKH7O.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
5
|
+
import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
|
|
6
6
|
import { decode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
function validateReplicationEntry(entry, localPrimaryTerm, log) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/schema/validator/field-type.ts
|
|
5
5
|
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
@@ -32,6 +32,8 @@ var ErrorCodes = {
|
|
|
32
32
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
33
33
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
34
34
|
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
35
|
+
SEARCH_INVALID_PHRASE: "SEARCH_INVALID_PHRASE",
|
|
36
|
+
SEARCH_PHRASE_NOT_READY: "SEARCH_PHRASE_NOT_READY",
|
|
35
37
|
BM25_NOT_READY: "BM25_NOT_READY",
|
|
36
38
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
37
39
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-4UMZ7KQL.mjs';
|
|
1
2
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { createNarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
3
4
|
|
|
4
5
|
// src/core/tokenizer/constants.ts
|
|
5
6
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -421,19 +422,6 @@ function tokenize(text, language, options) {
|
|
|
421
422
|
return { tokens, originalTokens, surfaces };
|
|
422
423
|
}
|
|
423
424
|
|
|
424
|
-
// src/core/partition/constants.ts
|
|
425
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
426
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
427
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
428
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
429
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
430
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
431
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
432
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
433
|
-
var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
|
|
434
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
435
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
436
|
-
|
|
437
425
|
// src/core/partition/frozen/string-blob.ts
|
|
438
426
|
var SURROGATE = /[\ud800-\udfff]/;
|
|
439
427
|
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
@@ -504,4 +492,4 @@ function codePointOrder(strings) {
|
|
|
504
492
|
return order;
|
|
505
493
|
}
|
|
506
494
|
|
|
507
|
-
export {
|
|
495
|
+
export { clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, getNormalizationCacheSize, resetNormalizationCache, resolveStopWords, tokenize, transformToken, visitSurfaceSpellings, visitTokens };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { flattenSchema } from './chunk-
|
|
2
|
-
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-
|
|
1
|
+
import { flattenSchema } from './chunk-PU6SKH7O.mjs';
|
|
2
|
+
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
3
3
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
4
|
-
import { isWordIndexedBase } from './chunk-
|
|
5
|
-
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-
|
|
4
|
+
import { isWordIndexedBase } from './chunk-7XY6VD7B.mjs';
|
|
5
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-D5TFRB5G.mjs';
|
|
6
6
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/partition/suggestions.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { readMetadataEnvelope } from './chunk-
|
|
2
|
-
import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
1
|
+
import { readMetadataEnvelope } from './chunk-K77FJXKS.mjs';
|
|
2
|
+
import { readWalSegments, durableRecordMissing, checkpointLastSeqNo, SEGMENT_HEADER_SIZE } from './chunk-2JWNJSNE.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-7OIQ2SIO.mjs';
|
|
4
4
|
|
|
5
5
|
// src/persistence/durability/recovery.ts
|
|
6
6
|
async function listPersistedIndexes(directory) {
|
|
@@ -12,6 +12,8 @@ var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
|
12
12
|
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
13
13
|
var MAX_TOLERANCE = 10;
|
|
14
14
|
var MAX_PREFIX_LENGTH = 1024;
|
|
15
|
+
var MAX_PHRASE_WORDS = 32;
|
|
16
|
+
var MAX_QUERY_PHRASES = 32;
|
|
15
17
|
var DEFAULT_MAX_PER_GROUP = 1;
|
|
16
18
|
var DEFAULT_RANK_CONSTANT = 60;
|
|
17
19
|
var MIN_RANK_CONSTANT = 1;
|
|
@@ -22,4 +24,4 @@ var MIN_HYBRID_DEPTH = 1;
|
|
|
22
24
|
var MAX_HYBRID_DEPTH = 1e4;
|
|
23
25
|
var HYBRID_RESCORE_WINDOW = 10;
|
|
24
26
|
|
|
25
|
-
export { BM25_DISPLAYED_NAME_LENGTH, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|
|
27
|
+
export { BM25_DISPLAYED_NAME_LENGTH, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-2E4XYSGT.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-
|
|
2
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-K77FJXKS.mjs';
|
|
2
|
+
import { requireValidFieldTypes } from './chunk-PU6SKH7O.mjs';
|
|
3
3
|
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
4
4
|
import { decode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-PU6SKH7O.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// src/core/bitset.ts
|
|
2
|
+
function createBitSet(capacity) {
|
|
3
|
+
return new Uint32Array(capacity + 31 >>> 5 || 1);
|
|
4
|
+
}
|
|
5
|
+
function bitsetSet(bs, index) {
|
|
6
|
+
const word = index >>> 5;
|
|
7
|
+
if (word < bs.length) {
|
|
8
|
+
bs[word] |= 1 << (index & 31);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function bitsetHas(bs, index) {
|
|
12
|
+
const word = index >>> 5;
|
|
13
|
+
if (word >= bs.length) return false;
|
|
14
|
+
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
15
|
+
}
|
|
16
|
+
function bitsetAnd(a, b) {
|
|
17
|
+
const len = Math.min(a.length, b.length);
|
|
18
|
+
const result = new Uint32Array(len);
|
|
19
|
+
for (let i = 0; i < len; i++) {
|
|
20
|
+
result[i] = a[i] & b[i];
|
|
21
|
+
}
|
|
22
|
+
return result;
|
|
23
|
+
}
|
|
24
|
+
function bitsetAndInto(target, other) {
|
|
25
|
+
const shared = Math.min(target.length, other.length);
|
|
26
|
+
for (let i = 0; i < shared; i++) target[i] &= other[i];
|
|
27
|
+
target.fill(0, shared);
|
|
28
|
+
}
|
|
29
|
+
function bitsetOrInto(target, other) {
|
|
30
|
+
const shared = Math.min(target.length, other.length);
|
|
31
|
+
for (let i = 0; i < shared; i++) target[i] |= other[i];
|
|
32
|
+
}
|
|
33
|
+
function bitsetAndNotInto(target, removed) {
|
|
34
|
+
const shared = Math.min(target.length, removed.length);
|
|
35
|
+
for (let i = 0; i < shared; i++) target[i] &= ~removed[i];
|
|
36
|
+
}
|
|
37
|
+
function bitsetNot(bs, capacity) {
|
|
38
|
+
const result = new Uint32Array(bs.length);
|
|
39
|
+
const fullWords = capacity >>> 5;
|
|
40
|
+
for (let i = 0; i < fullWords; i++) {
|
|
41
|
+
result[i] = ~bs[i];
|
|
42
|
+
}
|
|
43
|
+
const remainder = capacity & 31;
|
|
44
|
+
if (remainder > 0 && fullWords < bs.length) {
|
|
45
|
+
result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
|
|
46
|
+
}
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
function bitsetIsEmpty(bs) {
|
|
50
|
+
for (let i = 0; i < bs.length; i++) {
|
|
51
|
+
if (bs[i] !== 0) return false;
|
|
52
|
+
}
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
function bitsetFromSet(set, capacity) {
|
|
56
|
+
const bs = createBitSet(capacity);
|
|
57
|
+
for (const id of set) {
|
|
58
|
+
bs[id >>> 5] |= 1 << (id & 31);
|
|
59
|
+
}
|
|
60
|
+
return bs;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// src/core/partition/growable-uint32.ts
|
|
64
|
+
var GrowableUint32 = class {
|
|
65
|
+
values;
|
|
66
|
+
length = 0;
|
|
67
|
+
constructor(capacity) {
|
|
68
|
+
this.values = new Uint32Array(capacity);
|
|
69
|
+
}
|
|
70
|
+
growTo(needed, kept) {
|
|
71
|
+
let capacity = this.values.length * 2;
|
|
72
|
+
while (capacity < needed) capacity *= 2;
|
|
73
|
+
const next = new Uint32Array(capacity);
|
|
74
|
+
next.set(this.values.subarray(0, kept));
|
|
75
|
+
this.values = next;
|
|
76
|
+
}
|
|
77
|
+
reserve(extra) {
|
|
78
|
+
const needed = this.length + extra;
|
|
79
|
+
if (needed > this.values.length) this.growTo(needed, this.length);
|
|
80
|
+
}
|
|
81
|
+
push(value) {
|
|
82
|
+
if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
|
|
83
|
+
this.values[this.length++] = value;
|
|
84
|
+
}
|
|
85
|
+
ensureIndex(index) {
|
|
86
|
+
if (index >= this.values.length) this.growTo(index + 1, this.values.length);
|
|
87
|
+
}
|
|
88
|
+
clear() {
|
|
89
|
+
this.length = 0;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
export { GrowableUint32, bitsetAnd, bitsetAndInto, bitsetAndNotInto, bitsetFromSet, bitsetHas, bitsetIsEmpty, bitsetNot, bitsetOrInto, bitsetSet, createBitSet };
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-
|
|
3
|
-
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-
|
|
5
|
-
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-
|
|
6
|
-
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-Y2U5VVWX.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-2JWNJSNE.mjs';
|
|
3
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-TGHNXN7W.mjs';
|
|
4
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-RWESFSJI.mjs';
|
|
5
|
+
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-2MKX2SCO.mjs';
|
|
6
|
+
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-PU6SKH7O.mjs';
|
|
7
7
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
8
8
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
9
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
10
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
11
11
|
import { decode, encode } from '@msgpack/msgpack';
|
|
12
12
|
|
|
13
13
|
// src/engine/durability-metadata.ts
|
|
@@ -35,6 +35,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
35
35
|
if (config.analysisRevision !== void 0) {
|
|
36
36
|
metadata.analysisRevision = config.analysisRevision;
|
|
37
37
|
}
|
|
38
|
+
if (config.arrayPositionGap !== void 0) {
|
|
39
|
+
metadata.arrayPositionGap = config.arrayPositionGap;
|
|
40
|
+
}
|
|
38
41
|
if (config.tokenizer !== void 0) {
|
|
39
42
|
metadata.tokenizer = config.tokenizer;
|
|
40
43
|
}
|
|
@@ -94,6 +97,7 @@ function metadataConfigOf(entry) {
|
|
|
94
97
|
...embedding !== void 0 ? { embedding } : {},
|
|
95
98
|
surfaceForms: entry.config.surfaceForms !== false,
|
|
96
99
|
...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
|
|
100
|
+
...entry.arrayPositionGap !== null ? { arrayPositionGap: entry.arrayPositionGap } : {},
|
|
97
101
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
98
102
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
99
103
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -306,6 +310,9 @@ function metadataToWire(meta) {
|
|
|
306
310
|
if (meta.analysisRevision !== void 0) {
|
|
307
311
|
wire.analysis_revision = meta.analysisRevision;
|
|
308
312
|
}
|
|
313
|
+
if (meta.arrayPositionGap !== void 0) {
|
|
314
|
+
wire.array_position_gap = meta.arrayPositionGap;
|
|
315
|
+
}
|
|
309
316
|
if (meta.tokenizer !== void 0) {
|
|
310
317
|
wire.tokenizer = meta.tokenizer;
|
|
311
318
|
}
|
|
@@ -398,6 +405,9 @@ function wireToMetadata(raw) {
|
|
|
398
405
|
if (typeof raw.analysis_revision === "string") {
|
|
399
406
|
meta.analysisRevision = raw.analysis_revision;
|
|
400
407
|
}
|
|
408
|
+
if (typeof raw.array_position_gap === "number") {
|
|
409
|
+
meta.arrayPositionGap = raw.array_position_gap;
|
|
410
|
+
}
|
|
401
411
|
if (typeof raw.tokenizer === "string") {
|
|
402
412
|
meta.tokenizer = raw.tokenizer;
|
|
403
413
|
}
|