@delali/narsil 0.3.1-next.129 → 0.3.1-next.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-5D6NJD6C.mjs → chunk-2Y3CO4EQ.mjs} +342 -201
- package/dist/browser/{chunk-WLD6DDIX.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-YAGXZU4W.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-WZBTVTSU.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-XTFJZHQP.mjs → chunk-NSCSR6MA.mjs} +15 -5
- package/dist/browser/{chunk-3NVZD52E.mjs → chunk-UDGB2MWT.mjs} +5 -28
- package/dist/browser/{chunk-II5POT55.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-HCCC6BJB.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-S2VDU2JE.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-BZ7WWEC4.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-PL4TTFJE.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-RTUCHI2L.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-IMU7G5NU.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-BTjv-tK8.d.ts → capabilities-B6WxrWxt.d.ts} +3 -10
- package/dist/{chunk-3XHFVYIQ.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-37RTD6KD.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-K76F5YYX.mjs → chunk-2MKX2SCO.mjs} +219 -58
- package/dist/{chunk-AN3CUMPZ.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-OPO5SE4J.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-PTIJAZWL.mjs → chunk-4GDLFDPZ.mjs} +1 -1
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-TY64A3ZW.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-3LJVM6XM.mjs → chunk-5W7AEVTH.mjs} +2 -2
- package/dist/{chunk-PCRWCOZC.mjs → chunk-6M6SDH3S.mjs} +43 -23
- package/dist/{chunk-MGD7XETB.mjs → chunk-6SSVIW7Y.mjs} +19 -3
- package/dist/{chunk-5LC5VMDQ.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-4XEVROSP.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-WI5Q3L43.mjs → chunk-CZ7ISLXA.mjs} +2 -0
- package/dist/{chunk-RC27KF7F.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-F4UKMTKM.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-CTF6WFTX.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-DX2FI4LU.mjs → chunk-FQBFZAWG.mjs} +3 -1
- package/dist/{chunk-XTER67YM.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-WQYB4TNR.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-2HQQ7Y5D.mjs → chunk-JT3YZGCE.mjs} +2 -2
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-WUIDPBSY.mjs → chunk-K77FJXKS.mjs} +17 -7
- package/dist/{chunk-CSSCLRGY.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-44TB6AIZ.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-EZIGAOUJ.mjs → chunk-OGLV4I3L.mjs} +6 -5
- package/dist/{chunk-FRSWYMXH.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-AFOZR4ZQ.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/{chunk-TB7YMDD5.mjs → chunk-RCMNL7RI.mjs} +650 -100
- package/dist/{chunk-Y4NOHICF.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-RYH3PASD.mjs → chunk-RWESFSJI.mjs} +21 -11
- package/dist/{chunk-QQS4QZQA.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-K73PZTBR.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-43Q7HAI2.mjs → chunk-T4OFG4IQ.mjs} +4 -4
- package/dist/{chunk-2YOAGAGF.mjs → chunk-TGHNXN7W.mjs} +59 -30
- package/dist/{chunk-J4LRE5Q7.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-XRXBCOA4.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-Y4US5EGH.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-5VSTAJET.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-M6MBSVIJ.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-BOcveV7A.d.ts → client-gNAfaHcE.d.ts} +3 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +2 -0
- package/dist/{config-CPmIY3aH.d.ts → config-CmYJO34X.d.ts} +3 -3
- package/dist/{events-pM0lna7M.d.ts → config-D1cDv3sE.d.ts} +431 -3
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -8
- package/dist/distribution.mjs +85 -45
- package/dist/{durability-wiring-6R26AJDI.mjs → durability-wiring-S7OX2MAI.mjs} +27 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-JjRAXpT3.d.ts → engine-m_Ddr1bz.d.ts} +2 -2
- package/dist/{index-DkQ2DlgR.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -18
- package/dist/{long-query-thread-6VLA6YJI.mjs → long-query-thread-BSL25PD6.mjs} +17 -15
- package/dist/{module-S4WGMFNW.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -21
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +2 -0
- package/dist/{rerank-V4J4MIWR.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-PUGWCPCX.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-TBFBQQQO.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-DGAP_8HJ.d.ts → schema-BTXbHCqA.d.ts} +8 -5
- package/dist/server.d.ts +198 -11
- package/dist/server.mjs +30 -19
- package/dist/{snapshot-PUVSMV7E.mjs → snapshot-DOQLGGEX.mjs} +44 -29
- package/dist/synonyms.mjs +4 -3
- package/dist/tasks-Co8HfRD7.d.ts +704 -0
- package/dist/{thread-TA7QQBWP.mjs → thread-SKO7CN3R.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-TGKYT3C4.mjs → write-ZKR7SNRT.mjs} +22 -20
- package/openapi.json +5215 -0
- package/package.json +16 -8
- package/dist/config-C3YXk5Rn.d.ts +0 -412
- package/dist/direct-executor-ULA6V5HC.mjs +0 -26
- package/dist/tasks-txGvNz6h.d.ts +0 -302
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
2
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
|
|
2
3
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
3
|
-
import {
|
|
4
|
+
import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
|
|
5
|
+
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
|
|
4
6
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
5
7
|
|
|
6
8
|
// src/analysis/registry.ts
|
|
@@ -55,10 +57,26 @@ function getStopWords(name) {
|
|
|
55
57
|
}
|
|
56
58
|
return stopWords2;
|
|
57
59
|
}
|
|
60
|
+
function holdsPositionsOutOfOrder(entries) {
|
|
61
|
+
for (let index = 1; index < entries.length; index++) {
|
|
62
|
+
if (entries[index].position < entries[index - 1].position) return true;
|
|
63
|
+
}
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
function inPositionOrder(tokenizer) {
|
|
67
|
+
return {
|
|
68
|
+
tokenize(text) {
|
|
69
|
+
const entries = tokenizer.tokenize(text);
|
|
70
|
+
if (!holdsPositionsOutOfOrder(entries)) return entries;
|
|
71
|
+
return [...entries].sort((first, second) => first.position - second.position);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
58
75
|
function resolveIndexAnalysis(config) {
|
|
76
|
+
const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
|
|
59
77
|
return {
|
|
60
78
|
stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
|
|
61
|
-
customTokenizer:
|
|
79
|
+
customTokenizer: tokenizer === void 0 ? void 0 : inPositionOrder(tokenizer)
|
|
62
80
|
};
|
|
63
81
|
}
|
|
64
82
|
|
|
@@ -2232,6 +2250,17 @@ function requireValidPatternWorkCap(cap) {
|
|
|
2232
2250
|
);
|
|
2233
2251
|
}
|
|
2234
2252
|
|
|
2253
|
+
// src/core/constants.ts
|
|
2254
|
+
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
2255
|
+
var MAX_TERM_FREQUENCY = 65535;
|
|
2256
|
+
var ARRAY_POSITION_GAP = 100;
|
|
2257
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
2258
|
+
var DEFAULT_BM25_B = 0.75;
|
|
2259
|
+
var MAX_BM25_K1 = 1e3;
|
|
2260
|
+
var MAX_BM25_B = 1;
|
|
2261
|
+
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
2262
|
+
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
2263
|
+
|
|
2235
2264
|
// src/core/id-generator.ts
|
|
2236
2265
|
function generateId() {
|
|
2237
2266
|
const bytes = new Uint8Array(16);
|
|
@@ -3243,16 +3272,6 @@ function createDocumentStore() {
|
|
|
3243
3272
|
};
|
|
3244
3273
|
}
|
|
3245
3274
|
|
|
3246
|
-
// src/core/constants.ts
|
|
3247
|
-
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
3248
|
-
var MAX_TERM_FREQUENCY = 65535;
|
|
3249
|
-
var DEFAULT_BM25_K1 = 1.2;
|
|
3250
|
-
var DEFAULT_BM25_B = 0.75;
|
|
3251
|
-
var MAX_BM25_K1 = 1e3;
|
|
3252
|
-
var MAX_BM25_B = 1;
|
|
3253
|
-
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
3254
|
-
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
3255
|
-
|
|
3256
3275
|
// src/core/fuzzy.ts
|
|
3257
3276
|
var FIRST_SURROGATE = 55296;
|
|
3258
3277
|
var LAST_SURROGATE = 57343;
|
|
@@ -3750,6 +3769,13 @@ var LivePostingView = class {
|
|
|
3750
3769
|
const starts = this.store.pool.positionStarts;
|
|
3751
3770
|
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
3752
3771
|
}
|
|
3772
|
+
get positionStore() {
|
|
3773
|
+
return this.holdsPositions ? this.store.positions.values : null;
|
|
3774
|
+
}
|
|
3775
|
+
positionStartAt(row) {
|
|
3776
|
+
const starts = this.store.pool.positionStarts;
|
|
3777
|
+
return starts === null ? 0 : starts[this.syncedStart + row] + 1;
|
|
3778
|
+
}
|
|
3753
3779
|
copyPositionsAt(row, target, offset) {
|
|
3754
3780
|
if (!this.holdsPositions) return 0;
|
|
3755
3781
|
const starts = this.store.pool.positionStarts;
|
|
@@ -4472,48 +4498,6 @@ function createSurfaceRegistry() {
|
|
|
4472
4498
|
};
|
|
4473
4499
|
}
|
|
4474
4500
|
|
|
4475
|
-
// src/core/bitset.ts
|
|
4476
|
-
function createBitSet(capacity) {
|
|
4477
|
-
return new Uint32Array(capacity + 31 >>> 5 || 1);
|
|
4478
|
-
}
|
|
4479
|
-
function bitsetSet(bs, index) {
|
|
4480
|
-
const word = index >>> 5;
|
|
4481
|
-
if (word < bs.length) {
|
|
4482
|
-
bs[word] |= 1 << (index & 31);
|
|
4483
|
-
}
|
|
4484
|
-
}
|
|
4485
|
-
function bitsetHas(bs, index) {
|
|
4486
|
-
const word = index >>> 5;
|
|
4487
|
-
if (word >= bs.length) return false;
|
|
4488
|
-
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
4489
|
-
}
|
|
4490
|
-
function bitsetAndInto(target, other) {
|
|
4491
|
-
const shared2 = Math.min(target.length, other.length);
|
|
4492
|
-
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
4493
|
-
target.fill(0, shared2);
|
|
4494
|
-
}
|
|
4495
|
-
function bitsetOrInto(target, other) {
|
|
4496
|
-
const shared2 = Math.min(target.length, other.length);
|
|
4497
|
-
for (let i = 0; i < shared2; i++) target[i] |= other[i];
|
|
4498
|
-
}
|
|
4499
|
-
function bitsetAndNotInto(target, removed) {
|
|
4500
|
-
const shared2 = Math.min(target.length, removed.length);
|
|
4501
|
-
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4502
|
-
}
|
|
4503
|
-
function bitsetIsEmpty(bs) {
|
|
4504
|
-
for (let i = 0; i < bs.length; i++) {
|
|
4505
|
-
if (bs[i] !== 0) return false;
|
|
4506
|
-
}
|
|
4507
|
-
return true;
|
|
4508
|
-
}
|
|
4509
|
-
function bitsetFromSet(set, capacity) {
|
|
4510
|
-
const bs = createBitSet(capacity);
|
|
4511
|
-
for (const id of set) {
|
|
4512
|
-
bs[id >>> 5] |= 1 << (id & 31);
|
|
4513
|
-
}
|
|
4514
|
-
return bs;
|
|
4515
|
-
}
|
|
4516
|
-
|
|
4517
4501
|
// src/core/partition/facets.ts
|
|
4518
4502
|
var UNDECLARED_FIELD_TYPE = "";
|
|
4519
4503
|
function keptFacetValues(limit) {
|
|
@@ -5784,7 +5768,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5784
5768
|
if (state.trackPositions) {
|
|
5785
5769
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
5786
5770
|
let positionOffset = 0;
|
|
5787
|
-
let
|
|
5771
|
+
let lastItemPosition = -1;
|
|
5788
5772
|
const addToken = (token, position, surface) => {
|
|
5789
5773
|
const existing = tokenFreqs.get(token);
|
|
5790
5774
|
if (existing) {
|
|
@@ -5795,12 +5779,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5795
5779
|
}
|
|
5796
5780
|
fieldTokenList.push(token);
|
|
5797
5781
|
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5798
|
-
|
|
5782
|
+
lastItemPosition = position;
|
|
5799
5783
|
};
|
|
5800
5784
|
for (const item of arr) {
|
|
5801
|
-
|
|
5785
|
+
lastItemPosition = -1;
|
|
5802
5786
|
visitTokens(item, language, opts, addToken);
|
|
5803
|
-
positionOffset +=
|
|
5787
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
5804
5788
|
}
|
|
5805
5789
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5806
5790
|
for (const [token, freq] of tokenFreqs) {
|
|
@@ -6797,18 +6781,6 @@ function evaluateFilters(expression, context) {
|
|
|
6797
6781
|
return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
|
|
6798
6782
|
}
|
|
6799
6783
|
|
|
6800
|
-
// src/core/partition/constants.ts
|
|
6801
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
6802
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6803
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6804
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
6805
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
6806
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
6807
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
6808
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
6809
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6810
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6811
|
-
|
|
6812
6784
|
// src/core/partition/sort-columns/order.ts
|
|
6813
6785
|
var MISSING_RANK = -1;
|
|
6814
6786
|
var BYTES_PER_RANK = 4;
|
|
@@ -7549,6 +7521,57 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7549
7521
|
};
|
|
7550
7522
|
}
|
|
7551
7523
|
|
|
7524
|
+
// src/search/quotes.ts
|
|
7525
|
+
var QUOTE_CHARACTER = /["“”„]/;
|
|
7526
|
+
function holdsQuoteCharacter(term) {
|
|
7527
|
+
return QUOTE_CHARACTER.test(term);
|
|
7528
|
+
}
|
|
7529
|
+
|
|
7530
|
+
// src/search/phrase-code.ts
|
|
7531
|
+
var loaded = null;
|
|
7532
|
+
var loading = null;
|
|
7533
|
+
function requireLoadedPhraseCode() {
|
|
7534
|
+
if (loaded === null) {
|
|
7535
|
+
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
7536
|
+
}
|
|
7537
|
+
return loaded;
|
|
7538
|
+
}
|
|
7539
|
+
function loadPhraseCode() {
|
|
7540
|
+
if (loading === null) {
|
|
7541
|
+
loading = import('./phrase-module-WESYVU7K.mjs').then(
|
|
7542
|
+
(code) => {
|
|
7543
|
+
loaded = code;
|
|
7544
|
+
return code;
|
|
7545
|
+
},
|
|
7546
|
+
(err) => {
|
|
7547
|
+
loading = null;
|
|
7548
|
+
throw new NarsilError(
|
|
7549
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
7550
|
+
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
7551
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
7552
|
+
);
|
|
7553
|
+
}
|
|
7554
|
+
);
|
|
7555
|
+
}
|
|
7556
|
+
return loading;
|
|
7557
|
+
}
|
|
7558
|
+
function loadPhraseCodeFor(term) {
|
|
7559
|
+
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
7560
|
+
return loadPhraseCode();
|
|
7561
|
+
}
|
|
7562
|
+
function analyseQueryPhrases(term, language, analysis) {
|
|
7563
|
+
if (!holdsQuoteCharacter(term)) return null;
|
|
7564
|
+
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
7565
|
+
term.split(QUOTE_CHARACTER),
|
|
7566
|
+
(text) => tokenize(text, language, {
|
|
7567
|
+
stem: true,
|
|
7568
|
+
removeStopWords: true,
|
|
7569
|
+
customTokenizer: analysis.customTokenizer,
|
|
7570
|
+
stopWordOverride: analysis.stopWords
|
|
7571
|
+
}).tokens
|
|
7572
|
+
);
|
|
7573
|
+
}
|
|
7574
|
+
|
|
7552
7575
|
// src/core/scorer.ts
|
|
7553
7576
|
function computeIDF(docFrequency, totalDocs) {
|
|
7554
7577
|
if (totalDocs === 0) return 0;
|
|
@@ -7861,11 +7884,20 @@ function searchedFieldMarks(fieldNames, fields) {
|
|
|
7861
7884
|
function searchFulltextMatches(state, params) {
|
|
7862
7885
|
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
7863
7886
|
const fieldNames = state.fieldNameTable.names;
|
|
7864
|
-
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
7865
7887
|
const searched = searchedFieldMarks(fieldNames, fields);
|
|
7866
7888
|
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
7867
7889
|
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
7868
|
-
|
|
7890
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
7891
|
+
const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
|
|
7892
|
+
state.invertedIdx,
|
|
7893
|
+
params.phrases,
|
|
7894
|
+
searched,
|
|
7895
|
+
filterBitset,
|
|
7896
|
+
capacity,
|
|
7897
|
+
params.phraseWork,
|
|
7898
|
+
false
|
|
7899
|
+
).matched;
|
|
7900
|
+
for (const qt of params.phrases === void 0 ? queryTokens : []) {
|
|
7869
7901
|
const group = termGroupOf(termGroups, qt.token);
|
|
7870
7902
|
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7871
7903
|
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
@@ -8751,10 +8783,10 @@ function searchFulltext(state, params) {
|
|
|
8751
8783
|
globalStats,
|
|
8752
8784
|
maxResults,
|
|
8753
8785
|
termMatch,
|
|
8754
|
-
|
|
8786
|
+
phrases
|
|
8755
8787
|
} = params;
|
|
8756
8788
|
const collectComponents = params.collectComponents !== false;
|
|
8757
|
-
if (queryTokens.length === 0) {
|
|
8789
|
+
if (queryTokens.length === 0 && phrases === void 0) {
|
|
8758
8790
|
return { scored: [], totalMatched: 0 };
|
|
8759
8791
|
}
|
|
8760
8792
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
@@ -8772,9 +8804,48 @@ function searchFulltext(state, params) {
|
|
|
8772
8804
|
const onePair = boundPairs.length === 2;
|
|
8773
8805
|
const onlyK1 = boundPairs[0];
|
|
8774
8806
|
const onlyB = boundPairs[1];
|
|
8807
|
+
const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
|
|
8808
|
+
state.invertedIdx,
|
|
8809
|
+
phrases,
|
|
8810
|
+
searchable,
|
|
8811
|
+
params.filterBitset,
|
|
8812
|
+
state.docStore.internalIdCapacity(),
|
|
8813
|
+
params.phraseWork,
|
|
8814
|
+
true
|
|
8815
|
+
);
|
|
8816
|
+
if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
|
|
8817
|
+
return { scored: [], totalMatched: 0 };
|
|
8818
|
+
}
|
|
8819
|
+
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
8775
8820
|
function documentFrequency(token, list) {
|
|
8776
8821
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
8777
8822
|
}
|
|
8823
|
+
function scorePhraseRows(rows, matched) {
|
|
8824
|
+
let idf = 0;
|
|
8825
|
+
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
8826
|
+
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
8827
|
+
}
|
|
8828
|
+
for (let row = 0; row < rows.docIds.length; row++) {
|
|
8829
|
+
const internalId = rows.docIds.values[row];
|
|
8830
|
+
if (!bitsetHas(matched, internalId)) continue;
|
|
8831
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
8832
|
+
const frequency = rows.frequencies.values[row];
|
|
8833
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8834
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8835
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8836
|
+
frequency,
|
|
8837
|
+
idf,
|
|
8838
|
+
fieldLength,
|
|
8839
|
+
avgLen,
|
|
8840
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8841
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8842
|
+
);
|
|
8843
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
8844
|
+
if (components !== null) {
|
|
8845
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
8846
|
+
}
|
|
8847
|
+
}
|
|
8848
|
+
}
|
|
8778
8849
|
const groupContext = {
|
|
8779
8850
|
index: state.invertedIdx,
|
|
8780
8851
|
fields: fieldScoring,
|
|
@@ -8857,7 +8928,7 @@ function searchFulltext(state, params) {
|
|
|
8857
8928
|
mergeGroupContribution(internalId, contribution);
|
|
8858
8929
|
}
|
|
8859
8930
|
}
|
|
8860
|
-
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
8931
|
+
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
8861
8932
|
if (prunableList !== null && maxResults !== void 0) {
|
|
8862
8933
|
return singleTermTopK({
|
|
8863
8934
|
list: prunableList,
|
|
@@ -8868,7 +8939,7 @@ function searchFulltext(state, params) {
|
|
|
8868
8939
|
resolver
|
|
8869
8940
|
});
|
|
8870
8941
|
}
|
|
8871
|
-
const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
|
|
8942
|
+
const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
|
|
8872
8943
|
if (prunableLists !== null && maxResults !== void 0) {
|
|
8873
8944
|
const pruned = multiTermTopK({
|
|
8874
8945
|
terms: prunableLists,
|
|
@@ -8909,6 +8980,9 @@ function searchFulltext(state, params) {
|
|
|
8909
8980
|
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
8910
8981
|
}
|
|
8911
8982
|
}
|
|
8983
|
+
if (phraseMatches !== null) {
|
|
8984
|
+
for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
|
|
8985
|
+
}
|
|
8912
8986
|
const totalMatched = scoreBuffer.touchedCount;
|
|
8913
8987
|
const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
|
|
8914
8988
|
const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
|
|
@@ -10566,6 +10640,9 @@ function heapPop(heap) {
|
|
|
10566
10640
|
function collectCandidateTerms(subs, params) {
|
|
10567
10641
|
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10568
10642
|
const terms = /* @__PURE__ */ new Set();
|
|
10643
|
+
for (const phrase of params.phrases ?? []) {
|
|
10644
|
+
for (const token of phrase.tokens) terms.add(token);
|
|
10645
|
+
}
|
|
10569
10646
|
for (const qt of queryTokens) {
|
|
10570
10647
|
const group = termGroupOf(termGroups, qt.token);
|
|
10571
10648
|
if (group !== void 0) {
|
|
@@ -10667,7 +10744,7 @@ function searchSingleSub(subs, layout, params, index) {
|
|
|
10667
10744
|
return { ...result, matchedOrdinalBitset };
|
|
10668
10745
|
}
|
|
10669
10746
|
function compositeSearchFulltext(subs, layout, params) {
|
|
10670
|
-
if (params.queryTokens.length === 0) {
|
|
10747
|
+
if (params.queryTokens.length === 0 && params.phrases === void 0) {
|
|
10671
10748
|
return { scored: [], totalMatched: 0 };
|
|
10672
10749
|
}
|
|
10673
10750
|
const single = onlyPopulatedSub(subs);
|
|
@@ -11556,7 +11633,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
|
|
|
11556
11633
|
function capExceeded(cap) {
|
|
11557
11634
|
return new NarsilError(
|
|
11558
11635
|
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
11559
|
-
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
11636
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
|
|
11560
11637
|
{ cap }
|
|
11561
11638
|
);
|
|
11562
11639
|
}
|
|
@@ -11787,15 +11864,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
|
|
|
11787
11864
|
}
|
|
11788
11865
|
}
|
|
11789
11866
|
}
|
|
11790
|
-
function addReachedTokens(tokens, partitions, term,
|
|
11867
|
+
function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
|
|
11868
|
+
if (requested.options.exact === true) return [];
|
|
11869
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
11870
|
+
const words = quoted?.words ?? term;
|
|
11871
|
+
if (quoted !== null && words.trim().length === 0) return [];
|
|
11872
|
+
const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
|
|
11791
11873
|
const { options } = expansion;
|
|
11792
|
-
if (options.exact === true) return [];
|
|
11793
11874
|
if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
|
|
11794
|
-
const analysed = queryTokensOf(
|
|
11875
|
+
const analysed = queryTokensOf(words, language, analysis);
|
|
11795
11876
|
if (analysed === null) return [];
|
|
11796
11877
|
const addedMembers = /* @__PURE__ */ new Set();
|
|
11797
11878
|
for (const partition of partitions) {
|
|
11798
|
-
addPartitionTokens(tokens, addedMembers, partition,
|
|
11879
|
+
addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
|
|
11799
11880
|
}
|
|
11800
11881
|
for (const token of addedMembers) tokens.add(token);
|
|
11801
11882
|
return [...addedMembers];
|
|
@@ -11818,17 +11899,24 @@ function partitionsIn(manager, partitionIds) {
|
|
|
11818
11899
|
|
|
11819
11900
|
// src/partitioning/distributed-scoring.ts
|
|
11820
11901
|
function analysedQueryTokens(term, language, analysis) {
|
|
11821
|
-
const
|
|
11822
|
-
|
|
11823
|
-
removeStopWords: true,
|
|
11824
|
-
customTokenizer: analysis.customTokenizer,
|
|
11825
|
-
stopWordOverride: analysis.stopWords
|
|
11826
|
-
});
|
|
11902
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
11903
|
+
const words = quoted?.words ?? term;
|
|
11827
11904
|
const unique = /* @__PURE__ */ new Set();
|
|
11828
|
-
|
|
11829
|
-
|
|
11905
|
+
if (quoted === null || words.trim().length > 0) {
|
|
11906
|
+
const { tokens } = tokenize(words, language, {
|
|
11907
|
+
stem: true,
|
|
11908
|
+
removeStopWords: true,
|
|
11909
|
+
customTokenizer: analysis.customTokenizer,
|
|
11910
|
+
stopWordOverride: analysis.stopWords
|
|
11911
|
+
});
|
|
11912
|
+
for (const { token } of tokens) {
|
|
11913
|
+
unique.add(token);
|
|
11914
|
+
}
|
|
11830
11915
|
}
|
|
11831
11916
|
for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
|
|
11917
|
+
for (const phrase of quoted?.phrases ?? []) {
|
|
11918
|
+
for (const token of phrase.tokens) unique.add(token);
|
|
11919
|
+
}
|
|
11832
11920
|
return unique;
|
|
11833
11921
|
}
|
|
11834
11922
|
function copyNumericRecord(source) {
|
|
@@ -12227,6 +12315,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
12227
12315
|
}
|
|
12228
12316
|
|
|
12229
12317
|
// src/search/fulltext.ts
|
|
12318
|
+
function analysedWords(term, language, options) {
|
|
12319
|
+
return tokenize(term, language, {
|
|
12320
|
+
stem: true,
|
|
12321
|
+
removeStopWords: true,
|
|
12322
|
+
customTokenizer: options?.customTokenizer,
|
|
12323
|
+
stopWordOverride: options?.stopWords
|
|
12324
|
+
}).tokens;
|
|
12325
|
+
}
|
|
12230
12326
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
12231
12327
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
12232
12328
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -12234,14 +12330,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
12234
12330
|
if (params.fields) {
|
|
12235
12331
|
validateSearchFields(params.fields, flatSchema);
|
|
12236
12332
|
}
|
|
12237
|
-
const
|
|
12238
|
-
|
|
12239
|
-
|
|
12240
|
-
customTokenizer: options?.customTokenizer,
|
|
12241
|
-
stopWordOverride: options?.stopWords
|
|
12333
|
+
const quoted = analyseQueryPhrases(params.term, language, {
|
|
12334
|
+
stopWords: options?.stopWords,
|
|
12335
|
+
customTokenizer: options?.customTokenizer
|
|
12242
12336
|
});
|
|
12243
|
-
|
|
12244
|
-
const
|
|
12337
|
+
const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
|
|
12338
|
+
const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
|
|
12339
|
+
const words = wordParams.term ?? "";
|
|
12340
|
+
const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
|
|
12341
|
+
if (analysedTokens.length === 0 && phrases === void 0) return null;
|
|
12342
|
+
const queryTokens = deduplicateTokens(analysedTokens);
|
|
12245
12343
|
let filterBitset;
|
|
12246
12344
|
if (params.filters) {
|
|
12247
12345
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -12251,16 +12349,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
12251
12349
|
);
|
|
12252
12350
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
12253
12351
|
}
|
|
12254
|
-
|
|
12352
|
+
if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
|
|
12353
|
+
const lastToken = analysedTokens[analysedTokens.length - 1];
|
|
12255
12354
|
const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
|
|
12256
|
-
const groups = resolveTermGroups(partition,
|
|
12257
|
-
return { queryTokens, ...groups, filterBitset };
|
|
12355
|
+
const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
|
|
12356
|
+
return { queryTokens, phrases, ...groups, filterBitset };
|
|
12258
12357
|
}
|
|
12259
12358
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
12260
12359
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
12261
12360
|
if (prepared === null) return null;
|
|
12262
12361
|
return partition.searchFulltextMatches({
|
|
12263
12362
|
queryTokens: prepared.queryTokens,
|
|
12363
|
+
phrases: prepared.phrases,
|
|
12364
|
+
phraseWork: options?.patternWork,
|
|
12264
12365
|
termGroups: prepared.termGroups,
|
|
12265
12366
|
partialFields: prepared.partialFields,
|
|
12266
12367
|
fields: params.fields,
|
|
@@ -12275,13 +12376,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12275
12376
|
if (prepared === null) {
|
|
12276
12377
|
return { scored: [], totalMatched: 0 };
|
|
12277
12378
|
}
|
|
12278
|
-
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
12379
|
+
const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
|
|
12279
12380
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
12280
12381
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
12281
12382
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
12282
12383
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
12283
12384
|
const rawResult = partition.searchFulltext({
|
|
12284
12385
|
queryTokens,
|
|
12386
|
+
phrases,
|
|
12387
|
+
phraseWork: options?.patternWork,
|
|
12285
12388
|
termGroups,
|
|
12286
12389
|
partialFields,
|
|
12287
12390
|
fields: params.fields,
|
|
@@ -12299,7 +12402,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12299
12402
|
});
|
|
12300
12403
|
let scored = rawResult.scored;
|
|
12301
12404
|
const termMatch = params.termMatch ?? "any";
|
|
12302
|
-
if (termMatch !== "any") {
|
|
12405
|
+
if (termMatch !== "any" && queryTokens.length > 0) {
|
|
12303
12406
|
scored = filterByTermCoverage(
|
|
12304
12407
|
scored,
|
|
12305
12408
|
queryTokens,
|
|
@@ -12308,7 +12411,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12308
12411
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
12309
12412
|
params.exact ?? false,
|
|
12310
12413
|
termGroups,
|
|
12311
|
-
partialFields ?? []
|
|
12414
|
+
partialFields ?? [],
|
|
12415
|
+
phrases
|
|
12312
12416
|
);
|
|
12313
12417
|
}
|
|
12314
12418
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -12350,8 +12454,9 @@ function deduplicateTokens(tokens) {
|
|
|
12350
12454
|
}
|
|
12351
12455
|
return result;
|
|
12352
12456
|
}
|
|
12353
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
12457
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
|
|
12354
12458
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
12459
|
+
const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
|
|
12355
12460
|
if (requiredCount <= 0) return scored;
|
|
12356
12461
|
if (requiredCount > queryTokens.length) return [];
|
|
12357
12462
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -12367,12 +12472,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
|
|
|
12367
12472
|
});
|
|
12368
12473
|
}
|
|
12369
12474
|
return scored.filter((doc) => {
|
|
12370
|
-
const matched = countDocTermMatches(
|
|
12475
|
+
const matched = countDocTermMatches(
|
|
12476
|
+
doc,
|
|
12477
|
+
queryTokens,
|
|
12478
|
+
tolerance,
|
|
12479
|
+
prefixLength,
|
|
12480
|
+
exact,
|
|
12481
|
+
groups,
|
|
12482
|
+
partialFields,
|
|
12483
|
+
phraseKeys
|
|
12484
|
+
);
|
|
12371
12485
|
return matched >= requiredCount;
|
|
12372
12486
|
});
|
|
12373
12487
|
}
|
|
12374
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
12375
|
-
const
|
|
12488
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
|
|
12489
|
+
const componentKeys = Object.keys(doc.idf);
|
|
12490
|
+
const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
|
|
12376
12491
|
if (indexTokens.length === 0) return 0;
|
|
12377
12492
|
let count = 0;
|
|
12378
12493
|
for (const qt of queryTokens) {
|
|
@@ -12527,6 +12642,8 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
12527
12642
|
|
|
12528
12643
|
// src/workers/query-actions.ts
|
|
12529
12644
|
async function runQueryAction(entry, action) {
|
|
12645
|
+
const loadingPhraseCode = loadPhraseCodeFor(action.params.term);
|
|
12646
|
+
if (loadingPhraseCode !== null) await loadingPhraseCode;
|
|
12530
12647
|
const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
|
|
12531
12648
|
const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
|
|
12532
12649
|
const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
|
|
@@ -13647,6 +13764,10 @@ function createFrozenPostingViews(payload, tombstones, counts) {
|
|
|
13647
13764
|
if (positionOffsets === null) return 0;
|
|
13648
13765
|
return positionOffsets[start + row + 1] - positionOffsets[start + row];
|
|
13649
13766
|
},
|
|
13767
|
+
positionStore: positionValues,
|
|
13768
|
+
positionStartAt(row) {
|
|
13769
|
+
return positionOffsets === null ? 0 : positionOffsets[start + row];
|
|
13770
|
+
},
|
|
13650
13771
|
docIdSet: {
|
|
13651
13772
|
get size() {
|
|
13652
13773
|
return counts.documentFrequency(sortedIndex);
|
|
@@ -13765,7 +13886,7 @@ function addForms(target, forms) {
|
|
|
13765
13886
|
function createFrozenSurfaceSlot(load, documents) {
|
|
13766
13887
|
let registry = null;
|
|
13767
13888
|
let uncounted = null;
|
|
13768
|
-
function
|
|
13889
|
+
function loaded3() {
|
|
13769
13890
|
if (registry === null) {
|
|
13770
13891
|
const built = createSurfaceRegistry();
|
|
13771
13892
|
const forms = load();
|
|
@@ -13779,26 +13900,26 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
13779
13900
|
}
|
|
13780
13901
|
return {
|
|
13781
13902
|
reader: {
|
|
13782
|
-
candidatesForPrefix: (prefix) =>
|
|
13783
|
-
stemChangedTotalFor: (token) =>
|
|
13784
|
-
size: () =>
|
|
13785
|
-
serialize: () =>
|
|
13903
|
+
candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
|
|
13904
|
+
stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
|
|
13905
|
+
size: () => loaded3().size(),
|
|
13906
|
+
serialize: () => loaded3().serialize()
|
|
13786
13907
|
},
|
|
13787
13908
|
holdsAny() {
|
|
13788
|
-
return
|
|
13909
|
+
return loaded3().size() > 0;
|
|
13789
13910
|
},
|
|
13790
13911
|
holds(spelling) {
|
|
13791
|
-
return
|
|
13912
|
+
return loaded3().holds(spelling);
|
|
13792
13913
|
},
|
|
13793
13914
|
subtractSpellings(ordinal, spellings) {
|
|
13794
13915
|
uncounted ??= new Uint8Array(documents.count);
|
|
13795
13916
|
uncounted[ordinal] = 1;
|
|
13796
13917
|
if (spellings.length === 0) return;
|
|
13797
|
-
const target =
|
|
13918
|
+
const target = loaded3();
|
|
13798
13919
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
13799
13920
|
},
|
|
13800
13921
|
addKeptForms(target, mergedOrdinals, analysis) {
|
|
13801
|
-
const source =
|
|
13922
|
+
const source = loaded3();
|
|
13802
13923
|
if (analysis === null) {
|
|
13803
13924
|
addForms(target, source.serialize());
|
|
13804
13925
|
return;
|
|
@@ -14853,36 +14974,6 @@ function mergeSegmentContents(selections, analysis = null, textAnalysis = null)
|
|
|
14853
14974
|
return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
|
|
14854
14975
|
}
|
|
14855
14976
|
|
|
14856
|
-
// src/core/partition/growable-uint32.ts
|
|
14857
|
-
var GrowableUint32 = class {
|
|
14858
|
-
values;
|
|
14859
|
-
length = 0;
|
|
14860
|
-
constructor(capacity) {
|
|
14861
|
-
this.values = new Uint32Array(capacity);
|
|
14862
|
-
}
|
|
14863
|
-
growTo(needed, kept) {
|
|
14864
|
-
let capacity = this.values.length * 2;
|
|
14865
|
-
while (capacity < needed) capacity *= 2;
|
|
14866
|
-
const next = new Uint32Array(capacity);
|
|
14867
|
-
next.set(this.values.subarray(0, kept));
|
|
14868
|
-
this.values = next;
|
|
14869
|
-
}
|
|
14870
|
-
reserve(extra) {
|
|
14871
|
-
const needed = this.length + extra;
|
|
14872
|
-
if (needed > this.values.length) this.growTo(needed, this.length);
|
|
14873
|
-
}
|
|
14874
|
-
push(value) {
|
|
14875
|
-
if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
|
|
14876
|
-
this.values[this.length++] = value;
|
|
14877
|
-
}
|
|
14878
|
-
ensureIndex(index) {
|
|
14879
|
-
if (index >= this.values.length) this.growTo(index + 1, this.values.length);
|
|
14880
|
-
}
|
|
14881
|
-
clear() {
|
|
14882
|
-
this.length = 0;
|
|
14883
|
-
}
|
|
14884
|
-
};
|
|
14885
|
-
|
|
14886
14977
|
// src/core/partition/segment-postings.ts
|
|
14887
14978
|
var TextPostings = class {
|
|
14888
14979
|
constructor(trackPositions) {
|
|
@@ -15152,8 +15243,10 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
15152
15243
|
const spellings = /* @__PURE__ */ new Map();
|
|
15153
15244
|
let fieldSpellings = null;
|
|
15154
15245
|
let positionOffset = 0;
|
|
15246
|
+
let lastItemPosition = -1;
|
|
15155
15247
|
const addToken = (token, position, surface) => {
|
|
15156
15248
|
postings.addToken(token, positionOffset + position);
|
|
15249
|
+
lastItemPosition = position;
|
|
15157
15250
|
surfaces.count(surface, token);
|
|
15158
15251
|
if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
15159
15252
|
};
|
|
@@ -15172,8 +15265,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
15172
15265
|
postings.beginField();
|
|
15173
15266
|
positionOffset = 0;
|
|
15174
15267
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
15268
|
+
lastItemPosition = -1;
|
|
15175
15269
|
visitTokens(text, language, tokenizing, addToken);
|
|
15176
|
-
positionOffset
|
|
15270
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
15177
15271
|
}
|
|
15178
15272
|
postings.endField(ordinal, fieldPath);
|
|
15179
15273
|
lengths.record(ordinal, fieldPath, postings.fieldLength);
|
|
@@ -15291,30 +15385,30 @@ function runSegmentAction(entry, action) {
|
|
|
15291
15385
|
|
|
15292
15386
|
// src/workers/vector-code.ts
|
|
15293
15387
|
var VECTOR_TYPE_NAME = "vector";
|
|
15294
|
-
var
|
|
15295
|
-
var
|
|
15388
|
+
var loaded2 = null;
|
|
15389
|
+
var loading2 = null;
|
|
15296
15390
|
function loadedVectorCode() {
|
|
15297
|
-
return
|
|
15391
|
+
return loaded2;
|
|
15298
15392
|
}
|
|
15299
15393
|
function requireLoadedVectorCode() {
|
|
15300
|
-
if (
|
|
15301
|
-
return
|
|
15394
|
+
if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
|
|
15395
|
+
return loaded2;
|
|
15302
15396
|
}
|
|
15303
15397
|
function loadVectorCode() {
|
|
15304
|
-
if (
|
|
15305
|
-
if (
|
|
15306
|
-
|
|
15398
|
+
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15399
|
+
if (loading2 === null) {
|
|
15400
|
+
loading2 = import('./module-4HIMOQM5.mjs').then(
|
|
15307
15401
|
(code) => {
|
|
15308
|
-
|
|
15402
|
+
loaded2 = code;
|
|
15309
15403
|
return code;
|
|
15310
15404
|
},
|
|
15311
15405
|
(err) => {
|
|
15312
|
-
|
|
15406
|
+
loading2 = null;
|
|
15313
15407
|
throw err;
|
|
15314
15408
|
}
|
|
15315
15409
|
);
|
|
15316
15410
|
}
|
|
15317
|
-
return
|
|
15411
|
+
return loading2;
|
|
15318
15412
|
}
|
|
15319
15413
|
function mayDeclareVectorField(schema, depth) {
|
|
15320
15414
|
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
@@ -15326,7 +15420,7 @@ function mayDeclareVectorField(schema, depth) {
|
|
|
15326
15420
|
return false;
|
|
15327
15421
|
}
|
|
15328
15422
|
function loadVectorCodeFor(schema) {
|
|
15329
|
-
if (
|
|
15423
|
+
if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
|
|
15330
15424
|
return loadVectorCode();
|
|
15331
15425
|
}
|
|
15332
15426
|
|
|
@@ -15522,8 +15616,8 @@ function createDirectExecutor(options) {
|
|
|
15522
15616
|
);
|
|
15523
15617
|
}
|
|
15524
15618
|
case "createIndex": {
|
|
15525
|
-
const
|
|
15526
|
-
if (
|
|
15619
|
+
const loading3 = loadVectorCodeFor(action.config.schema);
|
|
15620
|
+
if (loading3 !== null) await loading3;
|
|
15527
15621
|
const language = getLanguage(action.config.language ?? "english");
|
|
15528
15622
|
const manager = addIndex(
|
|
15529
15623
|
action.indexName,
|
|
@@ -15892,6 +15986,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15892
15986
|
}
|
|
15893
15987
|
}
|
|
15894
15988
|
return {
|
|
15989
|
+
rebuildMode,
|
|
15895
15990
|
markStale(index) {
|
|
15896
15991
|
const existing = stale.get(index.indexName);
|
|
15897
15992
|
if (existing !== void 0 && existing.run !== null) {
|
|
@@ -16020,6 +16115,17 @@ function resolveDurabilityTier(config) {
|
|
|
16020
16115
|
return { kind: "snapshot", config: {}, adapter: config.persistence };
|
|
16021
16116
|
}
|
|
16022
16117
|
|
|
16118
|
+
// src/engine/array-positions.ts
|
|
16119
|
+
function holdsStringArrayField(schema) {
|
|
16120
|
+
return schemaFieldsOf(schema).some((field) => field.base === "string[]");
|
|
16121
|
+
}
|
|
16122
|
+
function arrayPositionsAreStale(config, storedGap) {
|
|
16123
|
+
return storedGap !== ARRAY_POSITION_GAP && config.trackPositions !== false && holdsStringArrayField(config.schema);
|
|
16124
|
+
}
|
|
16125
|
+
function storedArrayPositionGap(value) {
|
|
16126
|
+
return typeof value === "number" ? value : null;
|
|
16127
|
+
}
|
|
16128
|
+
|
|
16023
16129
|
// src/engine/recovery-schema.ts
|
|
16024
16130
|
function unflattenSchema(flat) {
|
|
16025
16131
|
const root = {};
|
|
@@ -16342,11 +16448,12 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16342
16448
|
documentCount: metadata.documentCount ?? 0,
|
|
16343
16449
|
partitionCount: metadata.partitionCount,
|
|
16344
16450
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
16451
|
+
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
16345
16452
|
synonyms
|
|
16346
16453
|
});
|
|
16347
16454
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
16348
16455
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
16349
|
-
if (metadata.analysisRevision !== language.revision) {
|
|
16456
|
+
if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
|
|
16350
16457
|
deps.analysisRebuild().markStale({
|
|
16351
16458
|
indexName: metadata.indexName,
|
|
16352
16459
|
language: language.name,
|
|
@@ -17662,10 +17769,10 @@ async function replicateToWorkers(state, action) {
|
|
|
17662
17769
|
return;
|
|
17663
17770
|
}
|
|
17664
17771
|
const detached = detachCallerDocuments(state, action);
|
|
17665
|
-
const
|
|
17666
|
-
if (
|
|
17772
|
+
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17773
|
+
if (loading3 !== void 0) {
|
|
17667
17774
|
queueForCopies(state, detached);
|
|
17668
|
-
if (
|
|
17775
|
+
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
|
|
17669
17776
|
return;
|
|
17670
17777
|
}
|
|
17671
17778
|
const queue = state.replicationQueues.get(action.indexName);
|
|
@@ -17676,9 +17783,9 @@ async function replicateToWorkers(state, action) {
|
|
|
17676
17783
|
}
|
|
17677
17784
|
function queueForCopies(state, action) {
|
|
17678
17785
|
if (!("indexName" in action)) return;
|
|
17679
|
-
const
|
|
17680
|
-
if (
|
|
17681
|
-
|
|
17786
|
+
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17787
|
+
if (loading3 !== void 0) {
|
|
17788
|
+
loading3.push(action);
|
|
17682
17789
|
return;
|
|
17683
17790
|
}
|
|
17684
17791
|
if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
|
|
@@ -19345,10 +19452,10 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19345
19452
|
stopRequestThreads: () => stopRequestThreads(state),
|
|
19346
19453
|
desyncIndex: (indexName) => {
|
|
19347
19454
|
state.desyncedIndexes.add(indexName);
|
|
19348
|
-
const
|
|
19455
|
+
const loading3 = state.copyTransitions.get(indexName);
|
|
19349
19456
|
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
19350
19457
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
19351
|
-
return wasScaledOut ||
|
|
19458
|
+
return wasScaledOut || loading3 !== void 0 && loading3.kind !== "drop";
|
|
19352
19459
|
},
|
|
19353
19460
|
async releaseIndexCopies(indexName) {
|
|
19354
19461
|
const hold = state.copyReleaseHolds.get(indexName);
|
|
@@ -19397,7 +19504,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19397
19504
|
// src/engine/core.ts
|
|
19398
19505
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19399
19506
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19400
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19507
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
|
|
19401
19508
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19402
19509
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19403
19510
|
const vectorCopyPolicy = {
|
|
@@ -19408,7 +19515,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19408
19515
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19409
19516
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19410
19517
|
}
|
|
19411
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19518
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
|
|
19412
19519
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19413
19520
|
const executor = createDirectExecutor({
|
|
19414
19521
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19574,7 +19681,9 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19574
19681
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
19575
19682
|
recordCurrentAnalysis: (indexName) => {
|
|
19576
19683
|
const entry = indexRegistry.get(indexName);
|
|
19577
|
-
if (entry
|
|
19684
|
+
if (entry === void 0 || analysisRebuild.isStale(indexName)) return;
|
|
19685
|
+
entry.analysisRevision = entry.language.revision;
|
|
19686
|
+
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
19578
19687
|
},
|
|
19579
19688
|
durabilityManager: durability?.manager ?? null
|
|
19580
19689
|
});
|
|
@@ -19705,7 +19814,7 @@ async function shutdownEngine(core) {
|
|
|
19705
19814
|
}
|
|
19706
19815
|
}
|
|
19707
19816
|
if (core.rerankAdapters.size > 0) {
|
|
19708
|
-
await (await import('./runtime-
|
|
19817
|
+
await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19709
19818
|
}
|
|
19710
19819
|
await orchestrator.shutdown();
|
|
19711
19820
|
await executor.shutdown();
|
|
@@ -20134,6 +20243,7 @@ function indexConfigurationOf(entry) {
|
|
|
20134
20243
|
entry.language,
|
|
20135
20244
|
entry.embeddingAdapterName,
|
|
20136
20245
|
entry.analysisRevision,
|
|
20246
|
+
entry.arrayPositionGap,
|
|
20137
20247
|
synonyms
|
|
20138
20248
|
);
|
|
20139
20249
|
}
|
|
@@ -20143,7 +20253,8 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
20143
20253
|
config,
|
|
20144
20254
|
language,
|
|
20145
20255
|
typeof adapter === "string" ? adapter : null,
|
|
20146
|
-
language.revision
|
|
20256
|
+
language.revision,
|
|
20257
|
+
ARRAY_POSITION_GAP
|
|
20147
20258
|
);
|
|
20148
20259
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
20149
20260
|
const bytes = encode(payload).byteLength;
|
|
@@ -20154,12 +20265,13 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
20154
20265
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
20155
20266
|
);
|
|
20156
20267
|
}
|
|
20157
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
|
|
20268
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
|
|
20158
20269
|
const bm25 = config.bm25;
|
|
20159
20270
|
return {
|
|
20160
20271
|
schema: config.schema,
|
|
20161
20272
|
language: language.name,
|
|
20162
20273
|
...analysisRevision !== null ? { analysisRevision } : {},
|
|
20274
|
+
...arrayPositionGap !== null ? { arrayPositionGap } : {},
|
|
20163
20275
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
20164
20276
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
20165
20277
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
@@ -20336,6 +20448,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20336
20448
|
documentCount: 0,
|
|
20337
20449
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
20338
20450
|
analysisRevision: language.revision,
|
|
20451
|
+
arrayPositionGap: ARRAY_POSITION_GAP,
|
|
20339
20452
|
synonyms: synonyms?.encoded ?? null
|
|
20340
20453
|
});
|
|
20341
20454
|
await core.indexState.registerOpen(name);
|
|
@@ -23037,14 +23150,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
23037
23150
|
return {
|
|
23038
23151
|
snapshot(indexName) {
|
|
23039
23152
|
return withOpenIndex2(indexName, async () => {
|
|
23040
|
-
const { createSnapshot } = await import('./snapshot-
|
|
23153
|
+
const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
|
|
23041
23154
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
23042
23155
|
});
|
|
23043
23156
|
},
|
|
23044
23157
|
async restore(indexName, data) {
|
|
23045
23158
|
core.guardShutdown();
|
|
23046
23159
|
validateIndexName(indexName);
|
|
23047
|
-
const restored = import('./restore-queue-
|
|
23160
|
+
const restored = import('./restore-queue-GEZ54CA5.mjs').then(
|
|
23048
23161
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
23049
23162
|
);
|
|
23050
23163
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -23053,8 +23166,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
23053
23166
|
exportDocuments(indexName) {
|
|
23054
23167
|
return withOpenIndex2(indexName, async () => {
|
|
23055
23168
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
23056
|
-
import('./snapshot-
|
|
23057
|
-
import('./write-
|
|
23169
|
+
import('./snapshot-V7VF6BN6.mjs'),
|
|
23170
|
+
import('./write-YIC2MKRY.mjs')
|
|
23058
23171
|
]);
|
|
23059
23172
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
23060
23173
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -23461,6 +23574,13 @@ function validQueryFilter(filters, schema) {
|
|
|
23461
23574
|
if (filters === void 0 || filters === null) return filters;
|
|
23462
23575
|
return validatedFilterOf(filters, flattenSchema(schema));
|
|
23463
23576
|
}
|
|
23577
|
+
function requireUsablePhrases(params, context) {
|
|
23578
|
+
if (params.term === void 0) return;
|
|
23579
|
+
const quoted = analyseQueryPhrases(params.term, context.language, context.manager.analysis);
|
|
23580
|
+
if (quoted === null || quoted.keptPhraseCount === 0) return;
|
|
23581
|
+
requireLoadedPhraseCode().requirePhrasesWithinLimits(quoted, context.config.trackPositions !== false);
|
|
23582
|
+
context.requirePhraseReady?.();
|
|
23583
|
+
}
|
|
23464
23584
|
function requireVectorSearchable(params, context, needsVector) {
|
|
23465
23585
|
if (!needsVector) return;
|
|
23466
23586
|
const field = params.vector?.field;
|
|
@@ -23866,8 +23986,8 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
23866
23986
|
}
|
|
23867
23987
|
|
|
23868
23988
|
// src/engine/query/highlight.ts
|
|
23869
|
-
function writtenWordsOf(
|
|
23870
|
-
return tokenize(
|
|
23989
|
+
function writtenWordsOf(term, language, analysis) {
|
|
23990
|
+
return tokenize(term, language, {
|
|
23871
23991
|
stem: false,
|
|
23872
23992
|
removeStopWords: true,
|
|
23873
23993
|
customTokenizer: analysis.customTokenizer,
|
|
@@ -23885,18 +24005,21 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
23885
24005
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
23886
24006
|
};
|
|
23887
24007
|
}
|
|
23888
|
-
function applyHighlights(hits,
|
|
23889
|
-
if (!
|
|
23890
|
-
const
|
|
24008
|
+
function applyHighlights(hits, request, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
24009
|
+
if (!request.highlight) return;
|
|
24010
|
+
const quoted = analyseQueryPhrases(request.term ?? "", language, analysis);
|
|
24011
|
+
const params = quoted === null ? request : requireLoadedPhraseCode().withWordsOnly(request, quoted);
|
|
24012
|
+
const words = params.term ?? "";
|
|
24013
|
+
const highlightConfig = request.highlight;
|
|
24014
|
+
const queryTokens = quoted !== null && words.trim().length === 0 ? [] : tokenize(words, language, {
|
|
23891
24015
|
stem: true,
|
|
23892
24016
|
removeStopWords: true,
|
|
23893
24017
|
customTokenizer: analysis.customTokenizer,
|
|
23894
24018
|
stopWordOverride: analysis.stopWords
|
|
23895
|
-
});
|
|
23896
|
-
const queryTokens = queryTokenResult.tokens;
|
|
24019
|
+
}).tokens;
|
|
23897
24020
|
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
23898
24021
|
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
23899
|
-
const written = usesPrefix || usesPartialWords ? writtenWordsOf(
|
|
24022
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(words, language, analysis) : [];
|
|
23900
24023
|
let prefixToken;
|
|
23901
24024
|
let completedToken;
|
|
23902
24025
|
if (usesPrefix) {
|
|
@@ -23910,12 +24033,14 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
23910
24033
|
}
|
|
23911
24034
|
}
|
|
23912
24035
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
23913
|
-
const
|
|
24036
|
+
const wordSynonyms = params.exact === true || queryTokens.length === 0 ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
24037
|
+
const phraseTokens = quoted?.phrases.flatMap((phrase) => phrase.tokens) ?? [];
|
|
24038
|
+
const synonyms = phraseTokens.length === 0 ? wordSynonyms : [...wordSynonyms ?? [], ...phraseTokens];
|
|
23914
24039
|
for (const hit of hits) {
|
|
23915
24040
|
const highlights = {};
|
|
23916
24041
|
const projected = hit.document;
|
|
23917
24042
|
let stored;
|
|
23918
|
-
for (const field of
|
|
24043
|
+
for (const field of highlightConfig.fields) {
|
|
23919
24044
|
let fieldValue = projected[field];
|
|
23920
24045
|
if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
|
|
23921
24046
|
stored ??= readStoredDocument(hit.id);
|
|
@@ -23923,9 +24048,9 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
23923
24048
|
}
|
|
23924
24049
|
if (typeof fieldValue === "string") {
|
|
23925
24050
|
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
23926
|
-
preTag:
|
|
23927
|
-
postTag:
|
|
23928
|
-
maxSnippetLength:
|
|
24051
|
+
preTag: highlightConfig.preTag,
|
|
24052
|
+
postTag: highlightConfig.postTag,
|
|
24053
|
+
maxSnippetLength: highlightConfig.maxSnippetLength,
|
|
23929
24054
|
prefixToken,
|
|
23930
24055
|
completedToken,
|
|
23931
24056
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
@@ -24347,13 +24472,17 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
24347
24472
|
}
|
|
24348
24473
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
24349
24474
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
24350
|
-
return import('./runtime-
|
|
24475
|
+
return import('./runtime-3X7ZJ5T3.mjs').then(
|
|
24351
24476
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
24352
24477
|
);
|
|
24353
24478
|
}
|
|
24354
24479
|
function executeQuery(request, callerContext) {
|
|
24355
|
-
if (request.rerank === void 0)
|
|
24356
|
-
|
|
24480
|
+
if (request.rerank === void 0) {
|
|
24481
|
+
const loadingPhraseCode = loadPhraseCodeFor(request.term);
|
|
24482
|
+
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
24483
|
+
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
24484
|
+
}
|
|
24485
|
+
return import('./runtime-3X7ZJ5T3.mjs').then(
|
|
24357
24486
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
24358
24487
|
);
|
|
24359
24488
|
}
|
|
@@ -24380,6 +24509,7 @@ async function runQuery(request, callerContext) {
|
|
|
24380
24509
|
}
|
|
24381
24510
|
const params = validQueryParams(request, config);
|
|
24382
24511
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
24512
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
24383
24513
|
const projection = resolveProjection(params.document);
|
|
24384
24514
|
let paginated;
|
|
24385
24515
|
let nextCursor;
|
|
@@ -24556,6 +24686,9 @@ async function executePreflight(request, callerContext) {
|
|
|
24556
24686
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
24557
24687
|
const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
24558
24688
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
24689
|
+
const loadingPhraseCode = loadPhraseCodeFor(params.term);
|
|
24690
|
+
if (loadingPhraseCode !== null) await loadingPhraseCode;
|
|
24691
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
24559
24692
|
const requestedVectorField = params.vector?.field;
|
|
24560
24693
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
24561
24694
|
let totalMatched;
|
|
@@ -24716,6 +24849,12 @@ async function readUnderOneAnalysis(core, indexName, read) {
|
|
|
24716
24849
|
}
|
|
24717
24850
|
return { result, analysisStale };
|
|
24718
24851
|
}
|
|
24852
|
+
function phraseReadinessOf(core, indexName, config) {
|
|
24853
|
+
return () => {
|
|
24854
|
+
if (!core.analysisRebuild.isStale(indexName) || !holdsStringArrayField(config.schema)) return;
|
|
24855
|
+
requireLoadedPhraseCode().refusePhraseUntilRebuilt(indexName, core.analysisRebuild.rebuildMode);
|
|
24856
|
+
};
|
|
24857
|
+
}
|
|
24719
24858
|
function scopedParams(params, options) {
|
|
24720
24859
|
{
|
|
24721
24860
|
return params;
|
|
@@ -24736,7 +24875,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24736
24875
|
const manager = core.requireManager(indexName);
|
|
24737
24876
|
const scoped = scopedParams(params, options);
|
|
24738
24877
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24739
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24878
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
|
|
24740
24879
|
const resolvedParams = await resolveVectorText(
|
|
24741
24880
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24742
24881
|
entry.embeddingAdapter,
|
|
@@ -24756,7 +24895,8 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24756
24895
|
indexBm25: options?.indexBm25,
|
|
24757
24896
|
cursorBinding: lazyQueryBinding(params),
|
|
24758
24897
|
patternWorkCap: core.patternWorkCap,
|
|
24759
|
-
rerank: rerank?.rerankRunnerFor(core)
|
|
24898
|
+
rerank: rerank?.rerankRunnerFor(core),
|
|
24899
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24760
24900
|
};
|
|
24761
24901
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24762
24902
|
core,
|
|
@@ -24805,7 +24945,8 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24805
24945
|
partitionIds: options?.partitionIds,
|
|
24806
24946
|
indexBm25: options?.indexBm25,
|
|
24807
24947
|
cursorBinding: lazyQueryBinding(params),
|
|
24808
|
-
patternWorkCap: core.patternWorkCap
|
|
24948
|
+
patternWorkCap: core.patternWorkCap,
|
|
24949
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24809
24950
|
};
|
|
24810
24951
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24811
24952
|
core,
|
|
@@ -25171,4 +25312,4 @@ async function createNarsil(config) {
|
|
|
25171
25312
|
// src/index.ts
|
|
25172
25313
|
var VERSION = "0.1.0";
|
|
25173
25314
|
|
|
25174
|
-
export { BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|
|
25315
|
+
export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|