@delali/narsil 0.3.1-next.103 → 0.3.1-next.107
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-RHG4O77A.mjs → chunk-J3QB4LBD.mjs} +1 -0
- package/dist/browser/{chunk-6DHYBN7M.mjs → chunk-JJKVYLL3.mjs} +3 -2
- package/dist/browser/{chunk-CRNQAT2Q.mjs → chunk-NNQPZPTB.mjs} +5 -5
- package/dist/browser/{chunk-MM7WIDI5.mjs → chunk-OKZGAJEF.mjs} +111 -10
- package/dist/browser/{chunk-AQZDOKDT.mjs → chunk-OL7S2KLL.mjs} +2 -2
- package/dist/browser/{chunk-74OXU3R2.mjs → chunk-PA56PRWX.mjs} +2 -2
- package/dist/browser/{chunk-YMDH3JID.mjs → chunk-WZUSMF7A.mjs} +13 -13
- package/dist/browser/{durability-wiring-SGY7CZHD.mjs → durability-wiring-HFQULBSA.mjs} +6 -6
- package/dist/browser/{module-QJ55JCJ6.mjs → module-VROKZ7IZ.mjs} +20 -16
- package/dist/browser/{restore-queue-LPKB6QJR.mjs → restore-queue-EU77FRCV.mjs} +1 -1
- package/dist/browser/{runtime-7ZAHCHWR.mjs → runtime-JQYPNOK6.mjs} +1 -1
- package/dist/browser/{snapshot-R7GY6PAG.mjs → snapshot-G77LQBZX.mjs} +7 -7
- package/dist/browser/{write-GFFRUCJD.mjs → write-RUC64JIH.mjs} +5 -5
- package/dist/{capabilities-ByQRMOx-.d.ts → capabilities-ByKPha_q.d.ts} +2 -2
- package/dist/{chunk-PC2G76VW.mjs → chunk-2MNJ2HX7.mjs} +2 -2
- package/dist/{chunk-EMRMGYRE.mjs → chunk-3BRFOMGX.mjs} +2 -2
- package/dist/{chunk-LPQOEU5M.mjs → chunk-5FPMDU54.mjs} +38 -9
- package/dist/{chunk-BWJVNZPR.mjs → chunk-6JUOMMHQ.mjs} +1 -0
- package/dist/{chunk-LKOB5COA.mjs → chunk-77SO7G55.mjs} +1 -1
- package/dist/{chunk-JTNDFNDP.mjs → chunk-7IBX2LRA.mjs} +12 -28
- package/dist/{chunk-UTG5YNXV.mjs → chunk-AEF2O7B4.mjs} +1 -1
- package/dist/{chunk-JZVARCRZ.mjs → chunk-BRIOLMIX.mjs} +2 -2
- package/dist/{chunk-YVDYMXR2.mjs → chunk-CKNOGER5.mjs} +22 -22
- package/dist/{chunk-5WT7BKQZ.mjs → chunk-CYHQNM3T.mjs} +1 -1
- package/dist/{chunk-U4436DZG.mjs → chunk-DNKM67DF.mjs} +2 -2
- package/dist/chunk-DUWYU4D5.mjs +19 -0
- package/dist/{chunk-ZKWYWW4Y.mjs → chunk-FABI2JA7.mjs} +1 -1
- package/dist/{chunk-VI4R6W25.mjs → chunk-FEOLYA2Y.mjs} +6 -6
- package/dist/{chunk-BQ2JSVAG.mjs → chunk-G6YATIRT.mjs} +113 -10
- package/dist/chunk-HAQUJYMN.mjs +344 -0
- package/dist/{chunk-TZRWRIZ6.mjs → chunk-HNAQNAFS.mjs} +3 -3
- package/dist/{chunk-64MXRJZ7.mjs → chunk-IA3H54X4.mjs} +2 -2
- package/dist/{chunk-TXV375YK.mjs → chunk-IA6I5U36.mjs} +1 -1
- package/dist/{chunk-SDFIFGOL.mjs → chunk-IZLM2OOO.mjs} +4 -4
- package/dist/{chunk-2RIRGOUL.mjs → chunk-JSSGO4B5.mjs} +1 -1
- package/dist/{chunk-5JNA5SKS.mjs → chunk-KOCHOOS5.mjs} +177 -8
- package/dist/{chunk-XKLUOODW.mjs → chunk-M5EZVXIM.mjs} +2 -2
- package/dist/{chunk-YEEQHT4V.mjs → chunk-MJIJG4YT.mjs} +2 -2
- package/dist/{chunk-75XIEF6H.mjs → chunk-MZW4V4DS.mjs} +1 -1
- package/dist/{chunk-IQYUFGQ6.mjs → chunk-NRARRUKN.mjs} +7 -7
- package/dist/{chunk-S7FL6UAH.mjs → chunk-OKZPYJZX.mjs} +3 -3
- package/dist/{chunk-HMPYTBBN.mjs → chunk-QA6F5MML.mjs} +6 -6
- package/dist/{chunk-ZM7KU4YW.mjs → chunk-QALAFGF5.mjs} +1 -1
- package/dist/{chunk-46D6JQIG.mjs → chunk-RI5VNFAI.mjs} +2 -2
- package/dist/{chunk-PNEYIHA4.mjs → chunk-RPIE4QCL.mjs} +0 -1
- package/dist/{chunk-XRTFWD3E.mjs → chunk-UM6Z7AIP.mjs} +3 -3
- package/dist/{chunk-YYQIODMF.mjs → chunk-VHPVBJOI.mjs} +2 -2
- package/dist/{chunk-HMFNFVP7.mjs → chunk-VJS4JFBQ.mjs} +2 -1
- package/dist/{chunk-RFVUDM5E.mjs → chunk-VVDQNCOG.mjs} +2 -15
- package/dist/{chunk-OEYI3KRB.mjs → chunk-VVFKFHWE.mjs} +3 -3
- package/dist/{chunk-AHVWSZTY.mjs → chunk-VWFFCDOL.mjs} +6 -6
- package/dist/{chunk-SM7S3HAP.mjs → chunk-WCJJCBR4.mjs} +9 -9
- package/dist/{chunk-EWZLIYHV.mjs → chunk-WCNN2PLX.mjs} +1 -1
- package/dist/{chunk-OM362BRO.mjs → chunk-WXVTQ6NO.mjs} +3 -3
- package/dist/chunk-ZXZ26WSO.mjs +14 -0
- package/dist/{client-BOStAkc5.d.ts → client-DI-xIXgR.d.ts} +2 -2
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-H6SGzbPH.d.ts → config-BZ_e3X8w.d.ts} +8 -5
- package/dist/{config-Cjvzbs_j.d.ts → config-DL_bDDD8.d.ts} +1 -1
- package/dist/direct-executor-BPBD2PBK.mjs +25 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +5 -4
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +3 -2
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +4 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +1818 -2425
- package/dist/{durability-wiring-2LOZIIJN.mjs → durability-wiring-AZL4EFLO.mjs} +27 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-rC0vjjY-.d.ts → engine-D_SYDsId.d.ts} +1 -1
- package/dist/{events-DqOIg5xa.d.ts → events-C7_dboUr.d.ts} +1 -0
- package/dist/{index-Bkdt8nDw.d.ts → index-Cl5qQauj.d.ts} +6 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +19 -18
- package/dist/{module-H37P44S6.mjs → module-6EY7GW2W.mjs} +25 -21
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{rerank-K7C4WZFG.mjs → rerank-WVFFZV76.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-KVCG36SI.mjs → restore-queue-DDXFDP7U.mjs} +1 -1
- package/dist/{runtime-6Y4ZYXOG.mjs → runtime-AVUUCJWQ.mjs} +5 -5
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +30 -17
- package/dist/{snapshot-YQ2CIFOZ.mjs → snapshot-IDAM65JO.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-CwPTz-tP.d.ts → tasks-BhjQZ8Qf.d.ts} +1 -1
- package/dist/{thread-5IYXJDWO.mjs → thread-LMSYLCPI.mjs} +16 -14
- package/dist/{types-2qoitTXg.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/{types-Cz5okLml.d.ts → types-DvncmvuQ.d.ts} +11 -0
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -6
- package/dist/{write-7UCWA3GX.mjs → write-3LIFIGOL.mjs} +21 -20
- package/package.json +7 -7
- package/dist/chunk-747OHOUA.mjs +0 -11
- package/dist/chunk-BOFBLJFZ.mjs +0 -209
- package/dist/direct-executor-QG7LEP6P.mjs +0 -24
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { PROCESS_LIMIT_REFRESH_INTERVAL_MS } from './chunk-ZXZ26WSO.mjs';
|
|
2
|
+
import { totalmem, freemem } from 'os';
|
|
2
3
|
import { getHeapSpaceStatistics, getHeapStatistics } from 'v8';
|
|
3
4
|
|
|
4
|
-
// src/runtime/heap-statistics.ts
|
|
5
5
|
var MEGABYTE_BYTES = 1048576;
|
|
6
6
|
var IN_MEGABYTES = /^--max-old-space-size=(\d+)$/;
|
|
7
7
|
var AS_PERCENTAGE = /^--max-old-space-size-percentage=(\d+(?:\.\d+)?)$/;
|
|
@@ -72,9 +72,9 @@ function readOldGenerationStatistics() {
|
|
|
72
72
|
usedBytes += space.space_used_size;
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
-
const
|
|
76
|
-
if (!Number.isFinite(usedBytes) || !Number.isFinite(
|
|
77
|
-
return { usedBytes, limitBytes, youngUsedBytes: Number.isFinite(youngUsedBytes) ? youngUsedBytes : 0 };
|
|
75
|
+
const limitBytes2 = configuredLimitBytes() ?? heapSizeLimitBytes() - (configuredYoungGenerationBytes() ?? youngCapacityBytes);
|
|
76
|
+
if (!Number.isFinite(usedBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
77
|
+
return { usedBytes, limitBytes: limitBytes2, youngUsedBytes: Number.isFinite(youngUsedBytes) ? youngUsedBytes : 0 };
|
|
78
78
|
} catch {
|
|
79
79
|
return null;
|
|
80
80
|
}
|
|
@@ -83,14 +83,43 @@ function readHeapStatistics() {
|
|
|
83
83
|
try {
|
|
84
84
|
const stats = getHeapStatistics();
|
|
85
85
|
const usedBytes = stats.used_heap_size;
|
|
86
|
-
const
|
|
87
|
-
if (!Number.isFinite(usedBytes) || !Number.isFinite(
|
|
86
|
+
const limitBytes2 = stats.heap_size_limit;
|
|
87
|
+
if (!Number.isFinite(usedBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
88
88
|
const available = stats.total_available_size;
|
|
89
89
|
const availableBytes = Number.isFinite(available) && available >= 0 ? available : null;
|
|
90
|
-
return { usedBytes, limitBytes, availableBytes, configuredLimitBytes: configuredLimitBytes() };
|
|
90
|
+
return { usedBytes, limitBytes: limitBytes2, availableBytes, configuredLimitBytes: configuredLimitBytes() };
|
|
91
|
+
} catch {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
var limitBytes = 0;
|
|
96
|
+
var limitReadAt = Number.NEGATIVE_INFINITY;
|
|
97
|
+
function readProcessLimit(residentBytes) {
|
|
98
|
+
const hostBytes = totalmem();
|
|
99
|
+
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
100
|
+
if (Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes) return constrained;
|
|
101
|
+
if (process.platform !== "linux") return hostBytes;
|
|
102
|
+
const availableBytes = freemem();
|
|
103
|
+
if (!Number.isFinite(availableBytes) || availableBytes < 0) return hostBytes;
|
|
104
|
+
return Math.min(hostBytes, residentBytes + availableBytes);
|
|
105
|
+
}
|
|
106
|
+
function processLimitBytes(residentBytes) {
|
|
107
|
+
const now = performance.now();
|
|
108
|
+
if (now - limitReadAt >= PROCESS_LIMIT_REFRESH_INTERVAL_MS) {
|
|
109
|
+
limitBytes = readProcessLimit(residentBytes);
|
|
110
|
+
limitReadAt = now;
|
|
111
|
+
}
|
|
112
|
+
return limitBytes;
|
|
113
|
+
}
|
|
114
|
+
function readResidentMemory() {
|
|
115
|
+
try {
|
|
116
|
+
const residentBytes = process.memoryUsage.rss();
|
|
117
|
+
const limitBytes2 = processLimitBytes(residentBytes);
|
|
118
|
+
if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes2) || limitBytes2 <= 0) return null;
|
|
119
|
+
return { residentBytes, limitBytes: limitBytes2 };
|
|
91
120
|
} catch {
|
|
92
121
|
return null;
|
|
93
122
|
}
|
|
94
123
|
}
|
|
95
124
|
|
|
96
|
-
export { readHeapStatistics, readOldGenerationStatistics };
|
|
125
|
+
export { processLimitBytes, readHeapStatistics, readOldGenerationStatistics, readResidentMemory };
|
|
@@ -68,6 +68,7 @@ var ErrorCodes = {
|
|
|
68
68
|
REPLICATION_ENTRY_INVALID: "REPLICATION_ENTRY_INVALID",
|
|
69
69
|
REPLICATION_INSYNC_REMOVAL_FAILED: "REPLICATION_INSYNC_REMOVAL_FAILED",
|
|
70
70
|
REPLICATION_ROLLBACK_FAILED: "REPLICATION_ROLLBACK_FAILED",
|
|
71
|
+
REPLICATION_OUTCOME_UNKNOWN: "REPLICATION_OUTCOME_UNKNOWN",
|
|
71
72
|
REPLICATION_LOG_FULL: "REPLICATION_LOG_FULL",
|
|
72
73
|
REPLICATION_ENTRY_CORRUPT: "REPLICATION_ENTRY_CORRUPT",
|
|
73
74
|
REPLICATION_SNAPSHOT_CORRUPT: "REPLICATION_SNAPSHOT_CORRUPT",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/schema/validator/field-type.ts
|
|
5
5
|
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
@@ -1,19 +1,18 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
2
|
-
import { generateId, getLanguage } from './chunk-
|
|
3
|
-
import { readHeapStatistics } from './chunk-
|
|
4
|
-
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-
|
|
5
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
6
|
-
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-FEOLYA2Y.mjs';
|
|
2
|
+
import { generateId, getLanguage } from './chunk-QALAFGF5.mjs';
|
|
3
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-5FPMDU54.mjs';
|
|
4
|
+
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-2MNJ2HX7.mjs';
|
|
5
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-WXVTQ6NO.mjs';
|
|
6
|
+
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-VVFKFHWE.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { isWordIndexedBase } from './chunk-
|
|
9
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
10
|
-
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-
|
|
8
|
+
import { isWordIndexedBase } from './chunk-77SO7G55.mjs';
|
|
9
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-RI5VNFAI.mjs';
|
|
10
|
+
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-AEF2O7B4.mjs';
|
|
11
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
12
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
12
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-IA6I5U36.mjs';
|
|
13
13
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
15
15
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
16
|
-
import { totalmem } from 'os';
|
|
17
16
|
|
|
18
17
|
// src/core/synonyms/platform.ts
|
|
19
18
|
function platformSynonymSearch() {
|
|
@@ -1393,21 +1392,6 @@ function createFrozenInvertedReader(table, views, counts) {
|
|
|
1393
1392
|
}
|
|
1394
1393
|
};
|
|
1395
1394
|
}
|
|
1396
|
-
function processLimitBytes() {
|
|
1397
|
-
const hostBytes = totalmem();
|
|
1398
|
-
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
1399
|
-
return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
|
|
1400
|
-
}
|
|
1401
|
-
function readResidentMemory() {
|
|
1402
|
-
try {
|
|
1403
|
-
const residentBytes = process.memoryUsage.rss();
|
|
1404
|
-
const limitBytes = processLimitBytes();
|
|
1405
|
-
if (!Number.isFinite(residentBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) return null;
|
|
1406
|
-
return { residentBytes, limitBytes };
|
|
1407
|
-
} catch {
|
|
1408
|
-
return null;
|
|
1409
|
-
}
|
|
1410
|
-
}
|
|
1411
1395
|
|
|
1412
1396
|
// src/core/partition/frozen/live-counts.ts
|
|
1413
1397
|
var UINT32_BYTES = 4;
|
|
@@ -3534,4 +3518,4 @@ function createDirectExecutor(options) {
|
|
|
3534
3518
|
};
|
|
3535
3519
|
}
|
|
3536
3520
|
|
|
3537
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents,
|
|
3521
|
+
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { createNarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/tokenizer/constants.ts
|
|
5
5
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
2
2
|
import { existsSync } from 'fs';
|
|
3
3
|
import { createRequire } from 'module';
|
|
4
4
|
import { join, dirname } from 'path';
|
|
5
5
|
|
|
6
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
6
|
+
var NATIVE_CORE_ABI_VERSION = 6;
|
|
7
7
|
var BINARY_FILE_NAME = "narsil-core.node";
|
|
8
8
|
var MUSL_DIRECTORY = "musl";
|
|
9
9
|
var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
2
|
-
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
3
|
-
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, queryBindingOf, executeQuery, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-NRARRUKN.mjs';
|
|
2
|
+
import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-3BRFOMGX.mjs';
|
|
3
|
+
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-CYHQNM3T.mjs';
|
|
4
4
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
5
|
-
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-
|
|
6
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-
|
|
7
|
-
import { generateId, getLanguage } from './chunk-
|
|
8
|
-
import { readHeapStatistics } from './chunk-
|
|
5
|
+
import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-7IBX2LRA.mjs';
|
|
6
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-FEOLYA2Y.mjs';
|
|
7
|
+
import { generateId, getLanguage } from './chunk-QALAFGF5.mjs';
|
|
8
|
+
import { readHeapStatistics } from './chunk-5FPMDU54.mjs';
|
|
9
9
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
10
10
|
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-LXWDQLTW.mjs';
|
|
11
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
12
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
13
|
-
import { getNestedValue } from './chunk-
|
|
14
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
11
|
+
import { requireSearchBackendSettings } from './chunk-BRIOLMIX.mjs';
|
|
12
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-2MNJ2HX7.mjs';
|
|
13
|
+
import { getNestedValue } from './chunk-WXVTQ6NO.mjs';
|
|
14
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, schemaFieldsOf, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-VVFKFHWE.mjs';
|
|
15
15
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
16
|
-
import { isSingleStringFieldType } from './chunk-
|
|
16
|
+
import { isSingleStringFieldType } from './chunk-77SO7G55.mjs';
|
|
17
17
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
18
18
|
import { createRequestId, isValidWorkerResponse } from './chunk-CQUMNIIG.mjs';
|
|
19
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
19
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-IA6I5U36.mjs';
|
|
20
20
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
21
21
|
import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
22
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
22
|
+
import { ErrorCodes, NarsilError } from './chunk-6JUOMMHQ.mjs';
|
|
23
23
|
import { encode, decode } from '@msgpack/msgpack';
|
|
24
24
|
import * as zlib from 'zlib';
|
|
25
25
|
|
|
@@ -4026,7 +4026,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
4026
4026
|
// src/engine/core.ts
|
|
4027
4027
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
4028
4028
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
4029
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
4029
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-AVUUCJWQ.mjs')).resolveRerankSettings(config);
|
|
4030
4030
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
4031
4031
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
4032
4032
|
const vectorCopyPolicy = {
|
|
@@ -4037,7 +4037,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4037
4037
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4038
4038
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4039
4039
|
}
|
|
4040
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4040
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-AZL4EFLO.mjs');
|
|
4041
4041
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4042
4042
|
const executor = createDirectExecutor({
|
|
4043
4043
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4334,7 +4334,7 @@ async function shutdownEngine(core) {
|
|
|
4334
4334
|
}
|
|
4335
4335
|
}
|
|
4336
4336
|
if (core.rerankAdapters.size > 0) {
|
|
4337
|
-
await (await import('./runtime-
|
|
4337
|
+
await (await import('./runtime-AVUUCJWQ.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
4338
4338
|
}
|
|
4339
4339
|
await orchestrator.shutdown();
|
|
4340
4340
|
await executor.shutdown();
|
|
@@ -6951,14 +6951,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6951
6951
|
return {
|
|
6952
6952
|
snapshot(indexName) {
|
|
6953
6953
|
return withOpenIndex(indexName, async () => {
|
|
6954
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6954
|
+
const { createSnapshot } = await import('./snapshot-IDAM65JO.mjs');
|
|
6955
6955
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6956
6956
|
});
|
|
6957
6957
|
},
|
|
6958
6958
|
async restore(indexName, data) {
|
|
6959
6959
|
core.guardShutdown();
|
|
6960
6960
|
validateIndexName(indexName);
|
|
6961
|
-
const restored = import('./restore-queue-
|
|
6961
|
+
const restored = import('./restore-queue-DDXFDP7U.mjs').then(
|
|
6962
6962
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6963
6963
|
);
|
|
6964
6964
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6967,8 +6967,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6967
6967
|
exportDocuments(indexName) {
|
|
6968
6968
|
return withOpenIndex(indexName, async () => {
|
|
6969
6969
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6970
|
-
import('./snapshot-
|
|
6971
|
-
import('./write-
|
|
6970
|
+
import('./snapshot-IDAM65JO.mjs'),
|
|
6971
|
+
import('./write-3LIFIGOL.mjs')
|
|
6972
6972
|
]);
|
|
6973
6973
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6974
6974
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -7020,7 +7020,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
7020
7020
|
const manager = core.requireManager(indexName);
|
|
7021
7021
|
const scoped = scopedParams(params, options);
|
|
7022
7022
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
7023
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
7023
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-AVUUCJWQ.mjs');
|
|
7024
7024
|
const resolvedParams = await resolveVectorText(
|
|
7025
7025
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
7026
7026
|
entry.embeddingAdapter,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QSK35UC2.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/workers/constants.ts
|
|
5
5
|
var WORKER_YOUNG_GENERATION_MB = 24;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-
|
|
2
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-QA6F5MML.mjs';
|
|
2
|
+
import { requireValidFieldTypes } from './chunk-VVFKFHWE.mjs';
|
|
3
3
|
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
4
4
|
import { decode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ReplicationMessageTypes } from './chunk-RPIE4QCL.mjs';
|
|
2
|
+
|
|
3
|
+
// src/distribution/transport/reply-deadline.ts
|
|
4
|
+
function replyDeadlineMs(messageType, config) {
|
|
5
|
+
if (messageType === ReplicationMessageTypes.ENTRY || messageType === ReplicationMessageTypes.ENTRY_BATCH) {
|
|
6
|
+
return config.replicationTimeout;
|
|
7
|
+
}
|
|
8
|
+
return config.requestTimeout;
|
|
9
|
+
}
|
|
10
|
+
function timeoutsOf(config) {
|
|
11
|
+
return {
|
|
12
|
+
connectTimeout: config.connectTimeout,
|
|
13
|
+
requestTimeout: config.requestTimeout,
|
|
14
|
+
replicationTimeout: config.replicationTimeout,
|
|
15
|
+
snapshotTimeout: config.snapshotTimeout
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export { replyDeadlineMs, timeoutsOf };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { TransportError, TransportErrorCodes } from './chunk-
|
|
2
|
+
import { TransportError, TransportErrorCodes } from './chunk-RPIE4QCL.mjs';
|
|
3
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-2MNJ2HX7.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-
|
|
4
|
-
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-
|
|
3
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-WXVTQ6NO.mjs';
|
|
4
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-VVFKFHWE.mjs';
|
|
5
5
|
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-E74YLAIG.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-
|
|
7
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-77SO7G55.mjs';
|
|
8
|
+
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-AEF2O7B4.mjs';
|
|
9
9
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
13
13
|
|
|
14
14
|
// src/core/partition/partial-words/platform.ts
|
|
15
15
|
function platformPartialWordSearch() {
|
|
@@ -2,11 +2,11 @@ import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled }
|
|
|
2
2
|
import { unpackLevels, readTrailer, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
3
3
|
import { markSimdUnavailable } from './chunk-KOKM4PQN.mjs';
|
|
4
4
|
import { MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, OSQ_STEPS, OSQ_GRID, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_PLACEMENT_QUERY_BITS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_QUERY_BITS, NATIVE_ATTACH_RETRY_MS, GRAPH_LOCK_WAIT_DEADLINE_MS, OSQ_LAMBDA, MAX_DOC_ID_TABLE_BYTES } from './chunk-QSK35UC2.mjs';
|
|
5
|
-
import { loadNativeCore } from './chunk-
|
|
6
|
-
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, DEFAULT_EF_SEARCH, FARTHEST_REJECTED_LINKS_KEPT } from './chunk-
|
|
5
|
+
import { loadNativeCore } from './chunk-BRIOLMIX.mjs';
|
|
6
|
+
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, DEFAULT_EF_SEARCH, NEAREST_LINK_FALLBACK_HOSTS, FARTHEST_REJECTED_LINKS_KEPT } from './chunk-VJS4JFBQ.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
8
|
import { threadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
9
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
9
|
+
import { NarsilError, ErrorCodes } from './chunk-6JUOMMHQ.mjs';
|
|
10
10
|
|
|
11
11
|
// src/vector/shared-buffers/growable.ts
|
|
12
12
|
function sharedMemoryAvailable() {
|
|
@@ -107,6 +107,7 @@ function createSharedGraphHandles(shape) {
|
|
|
107
107
|
locks: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
108
108
|
tombstones: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
109
109
|
incoming: perOrdinal(4, VECTOR_STORE_INITIAL_CAPACITY),
|
|
110
|
+
nearestLinkDue: perOrdinal(1, VECTOR_STORE_INITIAL_CAPACITY),
|
|
110
111
|
heldLocks: new Int32Array(createFixedBuffer(heldWords * 4), 0, heldWords)
|
|
111
112
|
};
|
|
112
113
|
publishGraphCapacity(handles);
|
|
@@ -127,7 +128,8 @@ function publishGraphCapacity(handles) {
|
|
|
127
128
|
Math.floor(handles.upperBase.byteLength / 4),
|
|
128
129
|
Math.floor(handles.locks.byteLength / 4),
|
|
129
130
|
handles.tombstones.byteLength,
|
|
130
|
-
Math.floor(handles.incoming.byteLength / 4)
|
|
131
|
+
Math.floor(handles.incoming.byteLength / 4),
|
|
132
|
+
handles.nearestLinkDue.byteLength
|
|
131
133
|
);
|
|
132
134
|
raiseHeaderWord(handles.header, GRAPH_SLOT_CAPACITY, slotCapacity);
|
|
133
135
|
raiseHeaderWord(handles.header, GRAPH_UPPER_CAPACITY, Math.floor(handles.upper.byteLength / 4));
|
|
@@ -604,12 +606,14 @@ function openAdjacency(handles, m, mMax0) {
|
|
|
604
606
|
level0: fixedView(handles.level0, Int32Array),
|
|
605
607
|
upperBase: fixedView(handles.upperBase, Int32Array),
|
|
606
608
|
upper: fixedView(handles.upper, Int32Array),
|
|
607
|
-
incoming: fixedView(handles.incoming, Int32Array)
|
|
609
|
+
incoming: fixedView(handles.incoming, Int32Array),
|
|
610
|
+
nearestLinkDue: fixedView(handles.nearestLinkDue, Uint8Array)
|
|
608
611
|
};
|
|
609
612
|
}
|
|
610
613
|
function rebindNodes(adj) {
|
|
611
614
|
adj.nodeLevels = fixedView(adj.handles.nodeLevels, Uint8Array);
|
|
612
615
|
adj.incoming = fixedView(adj.handles.incoming, Int32Array);
|
|
616
|
+
adj.nearestLinkDue = fixedView(adj.handles.nearestLinkDue, Uint8Array);
|
|
613
617
|
adj.level0 = fixedView(adj.handles.level0, Int32Array);
|
|
614
618
|
adj.upperBase = fixedView(adj.handles.upperBase, Int32Array);
|
|
615
619
|
}
|
|
@@ -647,6 +651,7 @@ function ensureAdjacencyCapacity(adj, needed) {
|
|
|
647
651
|
growBufferToExactly(handles.locks, capacity * 4);
|
|
648
652
|
growBufferToExactly(handles.tombstones, capacity);
|
|
649
653
|
growBufferToExactly(handles.incoming, capacity * 4);
|
|
654
|
+
growBufferToExactly(handles.nearestLinkDue, capacity);
|
|
650
655
|
growBufferToExactly(handles.nodeLevels, capacity);
|
|
651
656
|
publishGraphCapacity(handles);
|
|
652
657
|
rebindNodes(adj);
|
|
@@ -686,6 +691,7 @@ function createNode(adj, ord, maxLayer) {
|
|
|
686
691
|
adj.upperBase[ord] = maxLayer >= 1 ? allocateUpperBlock(adj, maxLayer) + 1 : 0;
|
|
687
692
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
688
693
|
raiseSlots(adj, ord + 1);
|
|
694
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
689
695
|
Atomics.store(adj.nodeLevels, ord, maxLayer + 1);
|
|
690
696
|
}
|
|
691
697
|
function deleteNode(adj, ord) {
|
|
@@ -694,11 +700,13 @@ function deleteNode(adj, ord) {
|
|
|
694
700
|
adj.upperBase[ord] = 0;
|
|
695
701
|
adj.level0[ord * adj.level0Stride] = 0;
|
|
696
702
|
Atomics.store(adj.incoming, ord, 0);
|
|
703
|
+
Atomics.store(adj.nearestLinkDue, ord, 0);
|
|
697
704
|
}
|
|
698
705
|
function resetAdjacency(adj) {
|
|
699
706
|
rebindNodes(adj);
|
|
700
707
|
rebindUpper(adj);
|
|
701
708
|
adj.incoming.fill(0);
|
|
709
|
+
adj.nearestLinkDue.fill(0);
|
|
702
710
|
adj.nodeLevels.fill(0);
|
|
703
711
|
adj.level0.fill(0);
|
|
704
712
|
adj.upperBase.fill(0);
|
|
@@ -715,7 +723,7 @@ function hasNode(adj, ord) {
|
|
|
715
723
|
}
|
|
716
724
|
function graphBytes(adj) {
|
|
717
725
|
const handles = adj.handles;
|
|
718
|
-
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.heldLocks.byteLength;
|
|
726
|
+
return handles.header.byteLength + handles.nodeLevels.byteLength + handles.level0.byteLength + handles.upperBase.byteLength + handles.upper.byteLength + handles.locks.byteLength + handles.tombstones.byteLength + handles.incoming.byteLength + handles.nearestLinkDue.byteLength + handles.heldLocks.byteLength;
|
|
719
727
|
}
|
|
720
728
|
|
|
721
729
|
// src/vector/hnsw/adjacency/incoming.ts
|
|
@@ -729,6 +737,7 @@ function noteLinked(adj, ord) {
|
|
|
729
737
|
}
|
|
730
738
|
function noteUnlinked(adj, ord) {
|
|
731
739
|
if (ord < 0 || !reachNode(adj, ord)) return 0;
|
|
740
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
732
741
|
for (; ; ) {
|
|
733
742
|
const seen = Atomics.load(adj.incoming, ord);
|
|
734
743
|
if (seen <= 0) return 0;
|
|
@@ -740,7 +749,9 @@ function releaseUnlessLast(adj, ord) {
|
|
|
740
749
|
for (; ; ) {
|
|
741
750
|
const seen = Atomics.load(adj.incoming, ord);
|
|
742
751
|
if (seen <= 1) return false;
|
|
743
|
-
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1)
|
|
752
|
+
if (Atomics.compareExchange(adj.incoming, ord, seen, seen - 1) !== seen) continue;
|
|
753
|
+
Atomics.store(adj.nearestLinkDue, ord, 1);
|
|
754
|
+
return true;
|
|
744
755
|
}
|
|
745
756
|
}
|
|
746
757
|
function holdsNodeAt(adj, ord) {
|
|
@@ -1473,6 +1484,7 @@ function graphMemory(graph) {
|
|
|
1473
1484
|
graph.locks,
|
|
1474
1485
|
graph.tombstones,
|
|
1475
1486
|
graph.incoming,
|
|
1487
|
+
graph.nearestLinkDue,
|
|
1476
1488
|
graph.heldLocks.buffer
|
|
1477
1489
|
];
|
|
1478
1490
|
if (!regions.every(isSharedMemory)) return null;
|
|
@@ -1485,6 +1497,7 @@ function graphMemory(graph) {
|
|
|
1485
1497
|
locks: fixedView(graph.locks, Int32Array),
|
|
1486
1498
|
tombstones: fixedView(graph.tombstones, Uint8Array),
|
|
1487
1499
|
incoming: fixedView(graph.incoming, Int32Array),
|
|
1500
|
+
nearestLinkDue: fixedView(graph.nearestLinkDue, Uint8Array),
|
|
1488
1501
|
heldLocks: graph.heldLocks
|
|
1489
1502
|
};
|
|
1490
1503
|
}
|
|
@@ -1841,6 +1854,95 @@ function writeKeepingLastLinks(state, ord, sorted, kept, held) {
|
|
|
1841
1854
|
writeNeighbors(adjacency, ord, 0, kept.ords, kept.size);
|
|
1842
1855
|
}
|
|
1843
1856
|
|
|
1857
|
+
// src/vector/hnsw/nearest-links.ts
|
|
1858
|
+
var NOT_YET_FOUND = -2;
|
|
1859
|
+
function nearer(a, b) {
|
|
1860
|
+
return a.distance < b.distance || a.distance === b.distance && a.ord < b.ord;
|
|
1861
|
+
}
|
|
1862
|
+
function listOf(state, ord) {
|
|
1863
|
+
const adjacency = state.adjacency;
|
|
1864
|
+
const base = layerBase(adjacency, ord, 0);
|
|
1865
|
+
if (base === ABSENT) return null;
|
|
1866
|
+
return adjacency.level0.subarray(base + 1, base + 1 + adjacency.level0[base]);
|
|
1867
|
+
}
|
|
1868
|
+
function nearestInList(pass, ord) {
|
|
1869
|
+
const known = pass.nearest[ord];
|
|
1870
|
+
if (known !== NOT_YET_FOUND) return known;
|
|
1871
|
+
let best = null;
|
|
1872
|
+
for (const neighbour of listOf(pass.state, ord) ?? []) {
|
|
1873
|
+
if (neighbour === ord || !isLive(pass.state, neighbour)) continue;
|
|
1874
|
+
const scored = { ord: neighbour, distance: pass.distance(ord, neighbour) };
|
|
1875
|
+
if (best === null || nearer(scored, best)) best = scored;
|
|
1876
|
+
}
|
|
1877
|
+
const nearest = best === null ? -1 : best.ord;
|
|
1878
|
+
pass.nearest[ord] = nearest;
|
|
1879
|
+
return nearest;
|
|
1880
|
+
}
|
|
1881
|
+
function fallbackHostsFor(pass, ord, nearest) {
|
|
1882
|
+
const around = [];
|
|
1883
|
+
for (const neighbour of listOf(pass.state, nearest) ?? []) {
|
|
1884
|
+
if (neighbour === ord || !isLive(pass.state, neighbour)) continue;
|
|
1885
|
+
around.push({ ord: neighbour, distance: pass.distance(ord, neighbour) });
|
|
1886
|
+
}
|
|
1887
|
+
around.sort((a, b) => nearer(a, b) ? -1 : nearer(b, a) ? 1 : 0);
|
|
1888
|
+
return around.slice(0, NEAREST_LINK_FALLBACK_HOSTS).map((scored) => scored.ord);
|
|
1889
|
+
}
|
|
1890
|
+
function replaceableSlot(pass, host, list) {
|
|
1891
|
+
let farthest = -1;
|
|
1892
|
+
let farthestNode = null;
|
|
1893
|
+
for (let i = 0; i < list.length; i++) {
|
|
1894
|
+
const neighbour = list[i];
|
|
1895
|
+
if (incomingCount(pass.state.adjacency, neighbour) < 2) continue;
|
|
1896
|
+
if (isLive(pass.state, neighbour) && nearestInList(pass, neighbour) === host) continue;
|
|
1897
|
+
const scored = { ord: neighbour, distance: pass.distance(host, neighbour) };
|
|
1898
|
+
if (farthestNode === null || nearer(farthestNode, scored)) {
|
|
1899
|
+
farthest = i;
|
|
1900
|
+
farthestNode = scored;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
return farthest;
|
|
1904
|
+
}
|
|
1905
|
+
function linkFrom(pass, host, ord) {
|
|
1906
|
+
const adjacency = pass.state.adjacency;
|
|
1907
|
+
const base = layerBase(adjacency, host, 0);
|
|
1908
|
+
if (base === ABSENT) return false;
|
|
1909
|
+
const list = adjacency.level0.subarray(base + 1, base + 1 + adjacency.level0[base]);
|
|
1910
|
+
if (list.includes(ord)) return true;
|
|
1911
|
+
if (list.length < pass.state.Mmax0) {
|
|
1912
|
+
adjacency.level0[base + list.length + 1] = ord;
|
|
1913
|
+
adjacency.level0[base] = list.length + 1;
|
|
1914
|
+
noteLinked(adjacency, ord);
|
|
1915
|
+
pass.nearest[host] = NOT_YET_FOUND;
|
|
1916
|
+
return true;
|
|
1917
|
+
}
|
|
1918
|
+
const slot = replaceableSlot(pass, host, list);
|
|
1919
|
+
if (slot === -1) return false;
|
|
1920
|
+
const replaced = list[slot];
|
|
1921
|
+
list[slot] = ord;
|
|
1922
|
+
noteUnlinked(adjacency, replaced);
|
|
1923
|
+
noteLinked(adjacency, ord);
|
|
1924
|
+
pass.nearest[host] = NOT_YET_FOUND;
|
|
1925
|
+
return true;
|
|
1926
|
+
}
|
|
1927
|
+
function linkFromNearest(state) {
|
|
1928
|
+
const adjacency = state.adjacency;
|
|
1929
|
+
const slots = adjacencySlots(adjacency);
|
|
1930
|
+
if (slots > 0) reachNode(adjacency, slots - 1);
|
|
1931
|
+
const pass = {
|
|
1932
|
+
state,
|
|
1933
|
+
distance: nodeDistanceFunction(state, state.buildMetric),
|
|
1934
|
+
nearest: new Int32Array(slots).fill(NOT_YET_FOUND)
|
|
1935
|
+
};
|
|
1936
|
+
for (let ord = 0; ord < slots; ord++) {
|
|
1937
|
+
if (Atomics.load(adjacency.nearestLinkDue, ord) !== 1) continue;
|
|
1938
|
+
Atomics.store(adjacency.nearestLinkDue, ord, 0);
|
|
1939
|
+
if (!isLive(state, ord)) continue;
|
|
1940
|
+
const nearest = nearestInList(pass, ord);
|
|
1941
|
+
if (nearest === -1 || linkFrom(pass, nearest, ord)) continue;
|
|
1942
|
+
for (const host of fallbackHostsFor(pass, ord, nearest)) if (linkFrom(pass, host, ord)) break;
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1844
1946
|
// src/vector/hnsw/connect-unreached.ts
|
|
1845
1947
|
var NOT_REACHED_THROUGH_A_NODE = -1;
|
|
1846
1948
|
function reach(walk, node, through) {
|
|
@@ -1881,7 +1983,7 @@ function holdsLinkOutsideTheWalk(state, walk, host) {
|
|
|
1881
1983
|
for (let i = 1; i <= count; i++) if (reachedThroughAnotherNode(walk, host, adjacency.level0[base + i])) return true;
|
|
1882
1984
|
return false;
|
|
1883
1985
|
}
|
|
1884
|
-
function
|
|
1986
|
+
function listOf2(state, ord) {
|
|
1885
1987
|
const adjacency = state.adjacency;
|
|
1886
1988
|
const base = layerBase(adjacency, ord, 0);
|
|
1887
1989
|
if (base === ABSENT) return new Int32Array(0);
|
|
@@ -1900,7 +2002,7 @@ function nearestHost(state, walk, orphan, nearby, test) {
|
|
|
1900
2002
|
}
|
|
1901
2003
|
};
|
|
1902
2004
|
for (const host of nearby.list) consider(host);
|
|
1903
|
-
for (const host of
|
|
2005
|
+
for (const host of listOf2(state, orphan))
|
|
1904
2006
|
if (host >= 0 && host < walk.reached.length && nearby.holds[host] !== 1) consider(host);
|
|
1905
2007
|
if (best !== -1) return best;
|
|
1906
2008
|
for (let host = 0; host < walk.reached.length; host++) consider(host);
|
|
@@ -1982,6 +2084,7 @@ function connectThroughEitherCore(state, nearby) {
|
|
|
1982
2084
|
const field = nativeFieldFor(state);
|
|
1983
2085
|
if (field !== null && nativeConnect(state, field, state.buildMetric, nearby)) return;
|
|
1984
2086
|
noteFallback("connect");
|
|
2087
|
+
linkFromNearest(state);
|
|
1985
2088
|
connectUnreached(state, nearby);
|
|
1986
2089
|
}
|
|
1987
2090
|
function connectAfterRemoval(state, nearby) {
|
|
@@ -2007,7 +2110,7 @@ function baseNeighboursOfRemoved(state, removed) {
|
|
|
2007
2110
|
const leaving = new Set(removed);
|
|
2008
2111
|
const nearby = [];
|
|
2009
2112
|
for (const ord of removed) {
|
|
2010
|
-
for (const neighbour of
|
|
2113
|
+
for (const neighbour of listOf2(state, ord)) if (!leaving.has(neighbour)) nearby.push(neighbour);
|
|
2011
2114
|
}
|
|
2012
2115
|
return Int32Array.from(nearby);
|
|
2013
2116
|
}
|