@delali/narsil 0.3.1-next.90 → 0.3.1-next.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-66KYAOIK.mjs} +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-B6XE7KXY.mjs} +11 -3
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-NOUZ6RH3.mjs} +2219 -2005
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-F4OGQUSU.mjs} +3 -3
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-5EJ7I7QY.mjs} +12 -6
- package/dist/browser/{write-6SBYTS62.mjs → write-K5V4XQ72.mjs} +2 -2
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-CvExBThq.d.ts} +11 -3
- package/dist/{chunk-MWJIAREY.mjs → chunk-3RDS4UHC.mjs} +2 -2
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-3TTYK5A7.mjs} +7 -6
- package/dist/{chunk-URGLSNKR.mjs → chunk-43ORORN6.mjs} +201 -15
- package/dist/{chunk-A3UB2MXL.mjs → chunk-5ER245FI.mjs} +2 -505
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-67Z2V6TC.mjs} +1 -1
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-6VZ3Q73N.mjs} +2265 -3461
- package/dist/{chunk-S4LQDSUL.mjs → chunk-7OQHSDK6.mjs} +3 -3
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-R6BMMQXH.mjs → chunk-A4G7JSYO.mjs} +106 -41
- package/dist/{chunk-DCK4P5BI.mjs → chunk-BJ2YKRNX.mjs} +16 -7
- package/dist/{chunk-J2BPXTD7.mjs → chunk-CO4MRKST.mjs} +1 -1
- package/dist/{chunk-CL4OM7GB.mjs → chunk-EONIHG3V.mjs} +8 -4
- package/dist/chunk-MKWUXWQH.mjs +35 -0
- package/dist/chunk-NVL4ZR3K.mjs +426 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-T4DB5YFB.mjs} +7 -3
- package/dist/{chunk-XSZVS4S3.mjs → chunk-UXQE2ROB.mjs} +6 -2
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XLJ2S2BH.mjs +3572 -0
- package/dist/chunk-ZIJ63TBT.mjs +507 -0
- package/dist/{client-BMK53IR2.d.ts → client-Cc42b42j.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +15 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-BYne-yCI.d.ts} +2 -2
- package/dist/{config-UJx1dyX1.d.ts → config-DyRowvVP.d.ts} +2 -2
- package/dist/{direct-executor-EGESD26S.mjs → direct-executor-CHXVCBN7.mjs} +9 -5
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +4 -0
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4590 -4263
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-6AN4QTT3.mjs} +20 -16
- package/dist/{engine-9X6W5naj.d.ts → engine-96_kLtSY.d.ts} +43 -7
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +13 -9
- package/dist/{memory-CV9NAxLY.d.ts → memory-dd3GESdc.d.ts} +3 -3
- package/dist/partial-words.mjs +2 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -14
- package/dist/react.d.ts +8 -7
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +33 -6
- package/dist/{snapshot-22CUEYYW.mjs → snapshot-JFZVOGZV.mjs} +28 -18
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-DA1Ff0G4.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-VTDLLYXC.mjs} +7 -5
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/workers/entry.mjs +4 -4
- package/dist/{write-PXDKPG2R.mjs → write-OJR3H4BI.mjs} +14 -10
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-CL4OM7GB.mjs';
|
|
2
1
|
import { vectorsPerFile } from './chunk-MXS5HWIH.mjs';
|
|
2
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-EONIHG3V.mjs';
|
|
3
3
|
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-YCJLVNWR.mjs';
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-3TTYK5A7.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-A4G7JSYO.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
8
8
|
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { encode, decode } from '@msgpack/msgpack';
|
|
2
|
+
|
|
3
|
+
// src/core/synonyms/rules.ts
|
|
4
|
+
|
|
5
|
+
// src/core/synonyms/constants.ts
|
|
6
|
+
var MAX_SYNONYM_RULES = 1e5;
|
|
7
|
+
var MAX_SYNONYM_WORDS = 5e5;
|
|
8
|
+
var MAX_SYNONYM_WORD_BYTES = 4194304;
|
|
9
|
+
var MAX_SYNONYM_RULE_WORDS = 100;
|
|
10
|
+
var MIN_TWO_WAY_SYNONYM_WORDS = 2;
|
|
11
|
+
var MIN_ONE_WAY_SYNONYMS = 1;
|
|
12
|
+
var SYNONYMS_ADDED_PER_QUERY = 50;
|
|
13
|
+
var SYNONYM_EXPANSION_CACHE_ENTRIES = 1024;
|
|
14
|
+
|
|
15
|
+
// src/core/synonyms/rules.ts
|
|
16
|
+
var FIRST_PRINTABLE_CODE = 32;
|
|
17
|
+
function holdsControlCharacter(word) {
|
|
18
|
+
for (let i = 0; i < word.length; i++) if (word.charCodeAt(i) < FIRST_PRINTABLE_CODE) return true;
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
function utf8Length(word) {
|
|
22
|
+
let bytes = 0;
|
|
23
|
+
for (let i = 0; i < word.length; i++) {
|
|
24
|
+
const unit = word.charCodeAt(i);
|
|
25
|
+
if (unit < 128) bytes += 1;
|
|
26
|
+
else if (unit < 2048) bytes += 2;
|
|
27
|
+
else if (unit >= 55296 && unit < 56320 && i + 1 < word.length) {
|
|
28
|
+
const low = word.charCodeAt(i + 1);
|
|
29
|
+
if (low >= 56320 && low < 57344) {
|
|
30
|
+
bytes += 4;
|
|
31
|
+
i++;
|
|
32
|
+
} else bytes += 3;
|
|
33
|
+
} else bytes += 3;
|
|
34
|
+
}
|
|
35
|
+
return bytes;
|
|
36
|
+
}
|
|
37
|
+
function hasOnlyKeys(value, keys) {
|
|
38
|
+
const own = Object.keys(value);
|
|
39
|
+
return own.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
40
|
+
}
|
|
41
|
+
function readWord(value, rule, tally, raise) {
|
|
42
|
+
if (typeof value !== "string") {
|
|
43
|
+
raise(`Synonym rule ${rule} holds a word that is not a string`, { rule });
|
|
44
|
+
}
|
|
45
|
+
if (holdsControlCharacter(value)) {
|
|
46
|
+
raise(`Synonym rule ${rule} holds a word with a control character`, { rule });
|
|
47
|
+
}
|
|
48
|
+
tally.words++;
|
|
49
|
+
tally.bytes += utf8Length(value);
|
|
50
|
+
if (tally.words > MAX_SYNONYM_WORDS) {
|
|
51
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_WORDS} words`, { maxWords: MAX_SYNONYM_WORDS });
|
|
52
|
+
}
|
|
53
|
+
if (tally.bytes > MAX_SYNONYM_WORD_BYTES) {
|
|
54
|
+
raise(`The words of a synonym set hold at most ${MAX_SYNONYM_WORD_BYTES} bytes in UTF-8`, {
|
|
55
|
+
maxWordBytes: MAX_SYNONYM_WORD_BYTES
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return value;
|
|
59
|
+
}
|
|
60
|
+
function readWords(value, rule, key, minimum, maximum, tally, raise) {
|
|
61
|
+
if (!Array.isArray(value) || value.length < minimum || value.length > maximum) {
|
|
62
|
+
raise(`Synonym rule ${rule} needs "${key}" as a list of ${minimum} to ${maximum} words`, { rule });
|
|
63
|
+
}
|
|
64
|
+
const words = new Array(value.length);
|
|
65
|
+
for (let i = 0; i < value.length; i++) words[i] = readWord(value[i], rule, tally, raise);
|
|
66
|
+
return words;
|
|
67
|
+
}
|
|
68
|
+
function readRule(value, rule, tally, raise) {
|
|
69
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
70
|
+
raise(`Synonym rule ${rule} must be an object`, { rule });
|
|
71
|
+
}
|
|
72
|
+
if (hasOnlyKeys(value, ["words"])) {
|
|
73
|
+
const words = Reflect.get(value, "words");
|
|
74
|
+
return { words: readWords(words, rule, "words", MIN_TWO_WAY_SYNONYM_WORDS, MAX_SYNONYM_RULE_WORDS, tally, raise) };
|
|
75
|
+
}
|
|
76
|
+
if (hasOnlyKeys(value, ["word", "synonyms"])) {
|
|
77
|
+
const word = readWord(Reflect.get(value, "word"), rule, tally, raise);
|
|
78
|
+
const synonyms = readWords(
|
|
79
|
+
Reflect.get(value, "synonyms"),
|
|
80
|
+
rule,
|
|
81
|
+
"synonyms",
|
|
82
|
+
MIN_ONE_WAY_SYNONYMS,
|
|
83
|
+
MAX_SYNONYM_RULE_WORDS - 1,
|
|
84
|
+
tally,
|
|
85
|
+
raise
|
|
86
|
+
);
|
|
87
|
+
return { word, synonyms };
|
|
88
|
+
}
|
|
89
|
+
raise(`Synonym rule ${rule} must hold either "words" alone, or "word" and "synonyms"`, { rule });
|
|
90
|
+
}
|
|
91
|
+
function parseSynonymRules(value, raise) {
|
|
92
|
+
if (!Array.isArray(value)) raise("A synonym set must be a list of rules", {});
|
|
93
|
+
if (value.length > MAX_SYNONYM_RULES) {
|
|
94
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_RULES} rules`, { maxRules: MAX_SYNONYM_RULES });
|
|
95
|
+
}
|
|
96
|
+
const tally = { words: 0, bytes: 0 };
|
|
97
|
+
const rules = new Array(value.length);
|
|
98
|
+
for (let rule = 0; rule < value.length; rule++) rules[rule] = readRule(value[rule], rule, tally, raise);
|
|
99
|
+
return rules;
|
|
100
|
+
}
|
|
101
|
+
function encodeSynonymRules(rules) {
|
|
102
|
+
return encode(rules);
|
|
103
|
+
}
|
|
104
|
+
function decodeSynonymRules(bytes, raise) {
|
|
105
|
+
let decoded;
|
|
106
|
+
try {
|
|
107
|
+
decoded = decode(bytes);
|
|
108
|
+
} catch {
|
|
109
|
+
raise("A stored synonym set is not valid MessagePack", {});
|
|
110
|
+
}
|
|
111
|
+
return parseSynonymRules(decoded, raise);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export { SYNONYMS_ADDED_PER_QUERY, SYNONYM_EXPANSION_CACHE_ENTRIES, decodeSynonymRules, encodeSynonymRules, parseSynonymRules };
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { createDirectExecutor, generateId, createPartitionRouter, readHeapStatistics, createPartitionIndex, mergeFrozenSegments, createSharedFrozenSegment, isCompositePartition, freezeSegmentShared, resolvePartitionInsertOptions, encodeSegmentState, createFrozenSegment } from './chunk-EMXAYPTQ.mjs';
|
|
2
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-DCK4P5BI.mjs';
|
|
3
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, fnv1a } from './chunk-X4SEXL4U.mjs';
|
|
4
|
-
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-J2BPXTD7.mjs';
|
|
5
|
-
import { getLanguage } from './chunk-PR4O7CAP.mjs';
|
|
6
1
|
import { splitWorkerBudget, resolveWorkerCount, NATIVE_CRC32_MIN_BYTES, resolveWorkerEntry, missingWorkerEntry, MAX_FILE_BYTES, resolveRequestThreadCount, CHECKSUM_YIELD_CHUNK_BYTES, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-RVM7AODE.mjs';
|
|
2
|
+
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, readHeapStatistics, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-XLJ2S2BH.mjs';
|
|
3
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-BJ2YKRNX.mjs';
|
|
4
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-6VZ3Q73N.mjs';
|
|
5
|
+
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-CO4MRKST.mjs';
|
|
7
6
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
7
|
+
import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
|
|
8
8
|
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-Q5DC6WAB.mjs';
|
|
9
9
|
import { requireSearchBackendSettings } from './chunk-QZN3MIJ3.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-T6CNO72J.mjs';
|
|
11
|
-
import { getNestedValue } from './chunk-
|
|
11
|
+
import { getNestedValue } from './chunk-5ER245FI.mjs';
|
|
12
12
|
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-CVQWQVFX.mjs';
|
|
13
13
|
import { isSingleStringFieldType } from './chunk-BVOJIYRV.mjs';
|
|
14
14
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
|
-
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-
|
|
15
|
+
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-R6PR4WYZ.mjs';
|
|
16
16
|
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-R7OSDAFK.mjs';
|
|
17
17
|
import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
18
18
|
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -852,6 +852,25 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
852
852
|
return config;
|
|
853
853
|
}
|
|
854
854
|
|
|
855
|
+
// src/engine/synonym-sets.ts
|
|
856
|
+
function withoutSynonyms(config) {
|
|
857
|
+
if (!Object.hasOwn(config, "synonyms")) return config;
|
|
858
|
+
const { synonyms: _synonyms, ...rest } = config;
|
|
859
|
+
return rest;
|
|
860
|
+
}
|
|
861
|
+
function prepareStoredSynonyms(value, config, language, raise, strictWords = false) {
|
|
862
|
+
return isEmptySynonymSet(value) ? null : synonymModule().prepare(value, config, language, strictWords, raise);
|
|
863
|
+
}
|
|
864
|
+
function prepareConfiguredSynonyms(config, language, strictWords = true) {
|
|
865
|
+
return prepareStoredSynonyms(config.synonyms, config, language, void 0, strictWords);
|
|
866
|
+
}
|
|
867
|
+
function lookupOfStoredSynonyms(encoded, config, language) {
|
|
868
|
+
return encoded === null || encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
|
|
869
|
+
}
|
|
870
|
+
function rulesOfStoredSynonyms(encoded) {
|
|
871
|
+
return encoded === null ? [] : synonymModule().decodeRules(encoded);
|
|
872
|
+
}
|
|
873
|
+
|
|
855
874
|
// src/engine/vector-fields.ts
|
|
856
875
|
function getVectorFieldPaths(schema) {
|
|
857
876
|
return new Set(extractVectorFieldsFromSchema(schema).keys());
|
|
@@ -896,7 +915,13 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
896
915
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
897
916
|
if (existing !== void 0) {
|
|
898
917
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
899
|
-
deps.executor.createIndex(
|
|
918
|
+
deps.executor.createIndex(
|
|
919
|
+
metadata.indexName,
|
|
920
|
+
existing.config,
|
|
921
|
+
existing.language,
|
|
922
|
+
false,
|
|
923
|
+
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
|
|
924
|
+
);
|
|
900
925
|
}
|
|
901
926
|
return;
|
|
902
927
|
}
|
|
@@ -910,9 +935,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
910
935
|
throw recoveryFailure(metadata, err);
|
|
911
936
|
}
|
|
912
937
|
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
938
|
+
const prepared = prepareStoredSynonyms(
|
|
939
|
+
metadata.synonyms,
|
|
940
|
+
indexConfig,
|
|
941
|
+
language,
|
|
942
|
+
"The synonym set in the index metadata is invalid: "
|
|
943
|
+
);
|
|
944
|
+
const synonyms = prepared?.encoded ?? null;
|
|
913
945
|
try {
|
|
914
946
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
915
|
-
if (loadData)
|
|
947
|
+
if (loadData) {
|
|
948
|
+
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
949
|
+
}
|
|
916
950
|
} catch (err) {
|
|
917
951
|
throw recoveryFailure(metadata, err);
|
|
918
952
|
}
|
|
@@ -926,7 +960,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
926
960
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
927
961
|
documentCount: metadata.documentCount ?? 0,
|
|
928
962
|
partitionCount: metadata.partitionCount,
|
|
929
|
-
analysisRevision: metadata.analysisRevision ?? null
|
|
963
|
+
analysisRevision: metadata.analysisRevision ?? null,
|
|
964
|
+
synonyms
|
|
930
965
|
});
|
|
931
966
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
932
967
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -1752,17 +1787,17 @@ async function sendPartition(manager, executors, captured) {
|
|
|
1752
1787
|
)
|
|
1753
1788
|
);
|
|
1754
1789
|
}
|
|
1755
|
-
function transferIndexToPool(indexName, pool,
|
|
1790
|
+
function transferIndexToPool(indexName, pool, source, manager, analysisStale = false) {
|
|
1756
1791
|
return loadIndexOntoWorkers(
|
|
1757
1792
|
indexName,
|
|
1758
1793
|
pool.getAllExecutors(),
|
|
1759
|
-
|
|
1794
|
+
source,
|
|
1760
1795
|
manager,
|
|
1761
1796
|
() => pool.addIndexToAll(indexName),
|
|
1762
1797
|
analysisStale
|
|
1763
1798
|
);
|
|
1764
1799
|
}
|
|
1765
|
-
async function loadIndexOntoWorkers(indexName, executors,
|
|
1800
|
+
async function loadIndexOntoWorkers(indexName, executors, source, manager, onDropped, analysisStale = false) {
|
|
1766
1801
|
copyLoadSequence += 1;
|
|
1767
1802
|
const copyLoad = `${indexName}#${copyLoadSequence}`;
|
|
1768
1803
|
const captured = capturePartitions(manager);
|
|
@@ -1777,10 +1812,11 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
1777
1812
|
(workerExecutor) => workerExecutor.execute({
|
|
1778
1813
|
type: "createIndex",
|
|
1779
1814
|
indexName,
|
|
1780
|
-
config,
|
|
1815
|
+
config: source.config,
|
|
1781
1816
|
requestId: `resync-create-${indexName}`,
|
|
1782
1817
|
analysisStale,
|
|
1783
|
-
copyLoad
|
|
1818
|
+
copyLoad,
|
|
1819
|
+
...source.synonyms !== null ? { synonyms: source.synonyms } : {}
|
|
1784
1820
|
})
|
|
1785
1821
|
)
|
|
1786
1822
|
);
|
|
@@ -3163,7 +3199,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
3163
3199
|
copyLoad = await loadIndexOntoWorkers(
|
|
3164
3200
|
indexName,
|
|
3165
3201
|
[replacement.executor],
|
|
3166
|
-
entry
|
|
3202
|
+
entry,
|
|
3167
3203
|
manager,
|
|
3168
3204
|
void 0,
|
|
3169
3205
|
state.callbacks?.isAnalysisStale?.(indexName)
|
|
@@ -3353,7 +3389,7 @@ async function loadCopies(state, indexName, reason) {
|
|
|
3353
3389
|
const copyLoad = await transferIndexToPool(
|
|
3354
3390
|
indexName,
|
|
3355
3391
|
pool,
|
|
3356
|
-
entry
|
|
3392
|
+
entry,
|
|
3357
3393
|
manager,
|
|
3358
3394
|
state.callbacks?.isAnalysisStale?.(indexName)
|
|
3359
3395
|
);
|
|
@@ -3776,7 +3812,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
3776
3812
|
const copyLoad = await transferIndexToPool(
|
|
3777
3813
|
indexName,
|
|
3778
3814
|
pool,
|
|
3779
|
-
entry
|
|
3815
|
+
entry,
|
|
3780
3816
|
manager,
|
|
3781
3817
|
callbacks?.isAnalysisStale?.(indexName)
|
|
3782
3818
|
);
|
|
@@ -3920,7 +3956,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
3920
3956
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
3921
3957
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
3922
3958
|
}
|
|
3923
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
3959
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-6AN4QTT3.mjs');
|
|
3924
3960
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
3925
3961
|
const executor = createDirectExecutor({
|
|
3926
3962
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4533,7 +4569,14 @@ function refuseCode(entry) {
|
|
|
4533
4569
|
}
|
|
4534
4570
|
function indexConfigurationOf(entry) {
|
|
4535
4571
|
refuseCode(entry);
|
|
4536
|
-
|
|
4572
|
+
const synonyms = entry.synonyms === null ? void 0 : rulesOfStoredSynonyms(entry.synonyms);
|
|
4573
|
+
return configurationPayloadOf(
|
|
4574
|
+
entry.config,
|
|
4575
|
+
entry.language,
|
|
4576
|
+
entry.embeddingAdapterName,
|
|
4577
|
+
entry.analysisRevision,
|
|
4578
|
+
synonyms
|
|
4579
|
+
);
|
|
4537
4580
|
}
|
|
4538
4581
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
4539
4582
|
const adapter = config.embedding?.adapter;
|
|
@@ -4552,7 +4595,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
4552
4595
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
4553
4596
|
);
|
|
4554
4597
|
}
|
|
4555
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
4598
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
|
|
4556
4599
|
const bm25 = config.bm25;
|
|
4557
4600
|
return {
|
|
4558
4601
|
schema: config.schema,
|
|
@@ -4575,6 +4618,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision)
|
|
|
4575
4618
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
4576
4619
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
4577
4620
|
...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
|
|
4621
|
+
...synonyms !== void 0 ? { synonyms } : {},
|
|
4578
4622
|
...config.embedding !== void 0 ? {
|
|
4579
4623
|
embedding: {
|
|
4580
4624
|
fields: config.embedding.fields,
|
|
@@ -4647,7 +4691,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4647
4691
|
}
|
|
4648
4692
|
const schema = validateSchema(requestedConfig.schema);
|
|
4649
4693
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
4650
|
-
const
|
|
4694
|
+
const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
4695
|
+
const indexConfig = withoutSynonyms(configured);
|
|
4651
4696
|
requirePatternSearchFor(schema);
|
|
4652
4697
|
requirePartialWordSearchFor(schema);
|
|
4653
4698
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
@@ -4708,7 +4753,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4708
4753
|
}
|
|
4709
4754
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
4710
4755
|
if (appliesCreationLimits) requireConfigurationWithinSizeLimit(indexConfig, language);
|
|
4711
|
-
|
|
4756
|
+
const synonyms = prepareConfiguredSynonyms(configured, language, appliesCreationLimits);
|
|
4757
|
+
core.executor.createIndex(name, indexConfig, language, false, synonyms?.lookup ?? null);
|
|
4712
4758
|
core.indexRegistry.set(name, {
|
|
4713
4759
|
config: indexConfig,
|
|
4714
4760
|
language,
|
|
@@ -4719,7 +4765,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4719
4765
|
heldPartitions: null,
|
|
4720
4766
|
documentCount: 0,
|
|
4721
4767
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
4722
|
-
analysisRevision: language.revision
|
|
4768
|
+
analysisRevision: language.revision,
|
|
4769
|
+
synonyms: synonyms?.encoded ?? null
|
|
4723
4770
|
});
|
|
4724
4771
|
await core.indexState.registerOpen(name);
|
|
4725
4772
|
if (core.durability) {
|
|
@@ -6796,7 +6843,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6796
6843
|
};
|
|
6797
6844
|
}
|
|
6798
6845
|
async function restoreIndex(indexName, data) {
|
|
6799
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
6846
|
+
const { restoreFromSnapshot } = await import('./snapshot-JFZVOGZV.mjs');
|
|
6800
6847
|
await restoreFromSnapshot(indexName, data, {
|
|
6801
6848
|
executor: core.executor,
|
|
6802
6849
|
indexRegistry: core.indexRegistry,
|
|
@@ -6816,7 +6863,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6816
6863
|
return {
|
|
6817
6864
|
snapshot(indexName) {
|
|
6818
6865
|
return withOpenIndex(indexName, async () => {
|
|
6819
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6866
|
+
const { createSnapshot } = await import('./snapshot-JFZVOGZV.mjs');
|
|
6820
6867
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6821
6868
|
});
|
|
6822
6869
|
},
|
|
@@ -6828,8 +6875,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6828
6875
|
exportDocuments(indexName) {
|
|
6829
6876
|
return withOpenIndex(indexName, async () => {
|
|
6830
6877
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6831
|
-
import('./snapshot-
|
|
6832
|
-
import('./write-
|
|
6878
|
+
import('./snapshot-JFZVOGZV.mjs'),
|
|
6879
|
+
import('./write-OJR3H4BI.mjs')
|
|
6833
6880
|
]);
|
|
6834
6881
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6835
6882
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6977,26 +7024,27 @@ async function runEngineSuggest(core, indexName, params, partitionIds) {
|
|
|
6977
7024
|
release();
|
|
6978
7025
|
}
|
|
6979
7026
|
}
|
|
6980
|
-
async function runEngineQueryStats(core, indexName, terms, partitionIds) {
|
|
7027
|
+
async function runEngineQueryStats(core, indexName, terms, partitionIds, scope) {
|
|
6981
7028
|
await enterRead(core, indexName);
|
|
6982
7029
|
const release = await core.indexState.acquire(indexName);
|
|
6983
7030
|
try {
|
|
6984
7031
|
const entry = core.requireIndex(indexName);
|
|
6985
7032
|
const manager = core.requireManager(indexName);
|
|
6986
|
-
const
|
|
6987
|
-
|
|
6988
|
-
|
|
6989
|
-
|
|
6990
|
-
|
|
6991
|
-
|
|
6992
|
-
|
|
6993
|
-
},
|
|
6994
|
-
|
|
6995
|
-
);
|
|
7033
|
+
const term = terms.join(" ");
|
|
7034
|
+
const analysis = {
|
|
7035
|
+
stopWords: manager.analysis.stopWords,
|
|
7036
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
7037
|
+
synonyms: manager.synonyms
|
|
7038
|
+
};
|
|
7039
|
+
const stats = collectQueryTermStats(manager, term, entry.language, analysis, partitionIds, {
|
|
7040
|
+
expansion: scope?.expansion === void 0 ? void 0 : { options: scope.expansion, schema: entry.config.schema },
|
|
7041
|
+
tokens: scope?.tokens
|
|
7042
|
+
});
|
|
6996
7043
|
return {
|
|
6997
7044
|
totalDocuments: stats.totalDocuments,
|
|
6998
7045
|
docFrequencies: stats.docFrequencies,
|
|
6999
|
-
totalFieldLengths: stats.totalFieldLengths
|
|
7046
|
+
totalFieldLengths: stats.totalFieldLengths,
|
|
7047
|
+
...stats.expanded.length > 0 ? { expanded: stats.expanded } : {}
|
|
7000
7048
|
};
|
|
7001
7049
|
} finally {
|
|
7002
7050
|
release();
|
|
@@ -7019,6 +7067,22 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
|
7019
7067
|
}
|
|
7020
7068
|
}
|
|
7021
7069
|
|
|
7070
|
+
// src/narsil/synonyms.ts
|
|
7071
|
+
function synonymOperations(core) {
|
|
7072
|
+
return {
|
|
7073
|
+
async updateSynonyms(indexName, synonyms) {
|
|
7074
|
+
core.guardShutdown();
|
|
7075
|
+
const absent = core.requireIndex(indexName).synonyms === null && isEmptySynonymSet(synonyms);
|
|
7076
|
+
if (absent && availableSynonymModule() === null) return;
|
|
7077
|
+
await synonymModule().replace(core, indexName, synonyms, true);
|
|
7078
|
+
},
|
|
7079
|
+
async getSynonyms(indexName) {
|
|
7080
|
+
core.guardShutdown();
|
|
7081
|
+
return rulesOfStoredSynonyms(core.requireIndex(indexName).synonyms);
|
|
7082
|
+
}
|
|
7083
|
+
};
|
|
7084
|
+
}
|
|
7085
|
+
|
|
7022
7086
|
// src/narsil/operations.ts
|
|
7023
7087
|
function createNarsilFromCore(core, config) {
|
|
7024
7088
|
const {
|
|
@@ -7173,6 +7237,7 @@ function createNarsilFromCore(core, config) {
|
|
|
7173
7237
|
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
7174
7238
|
},
|
|
7175
7239
|
...portabilityOperations(core, config, () => narsil),
|
|
7240
|
+
...synonymOperations(core),
|
|
7176
7241
|
async checkpoint(indexName) {
|
|
7177
7242
|
guardShutdown();
|
|
7178
7243
|
requireIndex(indexName);
|
|
@@ -7313,4 +7378,4 @@ async function createNarsil(config) {
|
|
|
7313
7378
|
// src/index.ts
|
|
7314
7379
|
var VERSION = "0.1.0";
|
|
7315
7380
|
|
|
7316
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareDocumentVectors, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
|
|
7381
|
+
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, resolveProjection, decodePageCursor, requireMatchingCursor, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-6VZ3Q73N.mjs';
|
|
2
2
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-Q5DC6WAB.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
4
|
-
import { tokenize } from './chunk-A3UB2MXL.mjs';
|
|
5
4
|
import { partialFieldsOf, flattenSchema } from './chunk-CVQWQVFX.mjs';
|
|
6
5
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
|
|
7
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
8
7
|
import { isWordIndexedFieldType } from './chunk-BVOJIYRV.mjs';
|
|
8
|
+
import { tokenize } from './chunk-ZIJ63TBT.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -305,13 +305,19 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
305
305
|
if (term === void 0 || term.trim().length === 0) {
|
|
306
306
|
return void 0;
|
|
307
307
|
}
|
|
308
|
-
|
|
308
|
+
const { manager } = context;
|
|
309
|
+
const analysis = { ...manager.analysis, synonyms: manager.synonyms };
|
|
310
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
|
|
311
|
+
partitions: partitionsFor(manager, context.partitionIds),
|
|
312
|
+
expansion: { options: params, schema: context.config.schema }
|
|
313
|
+
});
|
|
309
314
|
}
|
|
310
315
|
function searchOptionsFor(manager, patternWork) {
|
|
311
316
|
return {
|
|
312
317
|
bm25Params: manager.config.bm25,
|
|
313
318
|
stopWords: manager.analysis.stopWords,
|
|
314
319
|
customTokenizer: manager.analysis.customTokenizer,
|
|
320
|
+
synonyms: manager.synonyms,
|
|
315
321
|
patternWork
|
|
316
322
|
};
|
|
317
323
|
}
|
|
@@ -879,12 +885,13 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
879
885
|
const prefixLength = options?.prefixLength ?? 0;
|
|
880
886
|
const completedToken = options?.completedToken;
|
|
881
887
|
const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
|
|
888
|
+
const synonyms = options?.synonyms;
|
|
882
889
|
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
883
890
|
const matchesQuery = (fieldToken) => {
|
|
884
891
|
const known = matchByFieldToken.get(fieldToken);
|
|
885
892
|
if (known !== void 0) return known;
|
|
886
893
|
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
887
|
-
const matches = stemmedQueryTokens.some(
|
|
894
|
+
const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
|
|
888
895
|
(stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
|
|
889
896
|
);
|
|
890
897
|
matchByFieldToken.set(fieldToken, matches);
|
|
@@ -947,7 +954,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
947
954
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
948
955
|
};
|
|
949
956
|
}
|
|
950
|
-
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
|
|
957
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
951
958
|
if (!params.highlight) return;
|
|
952
959
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
953
960
|
stem: true,
|
|
@@ -972,6 +979,7 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
972
979
|
}
|
|
973
980
|
}
|
|
974
981
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
982
|
+
const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
975
983
|
for (const hit of hits) {
|
|
976
984
|
const highlights = {};
|
|
977
985
|
const projected = hit.document;
|
|
@@ -991,7 +999,8 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
991
999
|
completedToken,
|
|
992
1000
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
993
1001
|
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
994
|
-
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
|
|
1002
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
|
|
1003
|
+
synonyms
|
|
995
1004
|
});
|
|
996
1005
|
}
|
|
997
1006
|
}
|
|
@@ -1504,7 +1513,7 @@ async function executeQuery(request, callerContext) {
|
|
|
1504
1513
|
if (params.highlight) {
|
|
1505
1514
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
1506
1515
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
1507
|
-
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
|
|
1516
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
|
|
1508
1517
|
}
|
|
1509
1518
|
const elapsed = now() - startTime;
|
|
1510
1519
|
return {
|
|
@@ -581,4 +581,4 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
|
|
|
581
581
|
return { ...params, vector: resolved };
|
|
582
582
|
}
|
|
583
583
|
|
|
584
|
-
export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, requireDocumentObject, requireMatchingDocId, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
584
|
+
export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, rejectSnapshot, requireDocumentObject, requireMatchingDocId, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-YCJLVNWR.mjs';
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-
|
|
3
|
-
import { VERSION, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-EMXAYPTQ.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-3TTYK5A7.mjs';
|
|
3
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, HEADER_SIZE, packChecksummedEnvelope } from './chunk-A4G7JSYO.mjs';
|
|
5
4
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
5
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-XLJ2S2BH.mjs';
|
|
6
6
|
import { flattenSchema, requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -62,6 +62,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
62
62
|
if (config.indexUuid !== void 0) {
|
|
63
63
|
metadata.indexUuid = config.indexUuid;
|
|
64
64
|
}
|
|
65
|
+
if (config.synonyms !== void 0) {
|
|
66
|
+
metadata.synonyms = config.synonyms;
|
|
67
|
+
}
|
|
65
68
|
if (config.heldPartitions !== void 0) {
|
|
66
69
|
metadata.heldPartitions = config.heldPartitions;
|
|
67
70
|
}
|
|
@@ -93,7 +96,8 @@ function metadataConfigOf(entry) {
|
|
|
93
96
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
94
97
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
95
98
|
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
96
|
-
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {}
|
|
99
|
+
...entry.config.patternValueLimit !== void 0 ? { patternValueLimit: entry.config.patternValueLimit } : {},
|
|
100
|
+
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
|
|
97
101
|
};
|
|
98
102
|
}
|
|
99
103
|
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
3
|
+
|
|
4
|
+
// src/languages/registry.ts
|
|
5
|
+
var languages = /* @__PURE__ */ new Map();
|
|
6
|
+
languages.set("english", english);
|
|
7
|
+
function registerLanguage(module) {
|
|
8
|
+
languages.set(module.name, module);
|
|
9
|
+
}
|
|
10
|
+
function getLanguage(name) {
|
|
11
|
+
const lang = languages.get(name);
|
|
12
|
+
if (!lang) {
|
|
13
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
14
|
+
}
|
|
15
|
+
return lang;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// src/core/id-generator.ts
|
|
19
|
+
function generateId() {
|
|
20
|
+
const bytes = new Uint8Array(16);
|
|
21
|
+
crypto.getRandomValues(bytes);
|
|
22
|
+
const now = Date.now();
|
|
23
|
+
bytes[0] = Math.floor(now / 2 ** 40);
|
|
24
|
+
bytes[1] = Math.floor(now / 2 ** 32) % 256;
|
|
25
|
+
bytes[2] = Math.floor(now / 2 ** 24) % 256;
|
|
26
|
+
bytes[3] = Math.floor(now / 2 ** 16) % 256;
|
|
27
|
+
bytes[4] = Math.floor(now / 2 ** 8) % 256;
|
|
28
|
+
bytes[5] = now % 256;
|
|
29
|
+
bytes[6] = bytes[6] & 15 | 112;
|
|
30
|
+
bytes[8] = bytes[8] & 63 | 128;
|
|
31
|
+
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
32
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export { generateId, getLanguage, registerLanguage };
|