@delali/narsil 0.3.1-next.162 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-
|
|
2
|
-
import { flattenSchema } from './chunk-
|
|
1
|
+
import { VERSION, GROUP_PATTERN_SOURCE, loadGzip } from './chunk-F4MIGBBP.mjs';
|
|
2
|
+
import { flattenSchema } from './chunk-ZDDJVR7X.mjs';
|
|
3
3
|
import { MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-VUJCSQG5.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
5
5
|
import { encode } from '@msgpack/msgpack';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { createSurfaceRegistry, SegmentRefusal, encodeSurfaceTable, encodePartialSpellingsOf, encodeStringBlob, mergeTextAt, generateId, encodeExternalIdTableData, buildPartialWordTableFrom, wrapFrozenTokenTable, encodeDocumentTableData, topLevelMapKeys, mergeStoredFields, prepareDocumentText, recordValueSize, patternIndexOf, encodeFieldIndexes, mergeFieldRegions, sameFields, partialSpellingsOfDocuments, storedDocumentsOf, encodeFrozenTokenTableData, mergeKeywordSources, GrowingSegment, LivePostings, createSpellingTable, createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex } from './chunk-
|
|
2
|
-
import { storedText, bitsetIterator } from './chunk-
|
|
1
|
+
import { createSurfaceRegistry, SegmentRefusal, encodeSurfaceTable, encodePartialSpellingsOf, encodeStringBlob, mergeTextAt, generateId, encodeExternalIdTableData, buildPartialWordTableFrom, wrapFrozenTokenTable, encodeDocumentTableData, topLevelMapKeys, mergeStoredFields, prepareDocumentText, recordValueSize, patternIndexOf, encodeFieldIndexes, mergeFieldRegions, sameFields, partialSpellingsOfDocuments, storedDocumentsOf, encodeFrozenTokenTableData, mergeKeywordSources, GrowingSegment, LivePostings, createSpellingTable, createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex } from './chunk-F4MIGBBP.mjs';
|
|
2
|
+
import { storedText, bitsetIterator } from './chunk-SK3DPAI4.mjs';
|
|
3
3
|
import { encodeDocument } from './chunk-YOTP2FPE.mjs';
|
|
4
|
-
import { schemaFieldsOf, getNestedValue, isWordIndexedBase, validateDocument, validateDocumentStrict } from './chunk-
|
|
4
|
+
import { schemaFieldsOf, getNestedValue, isWordIndexedBase, validateDocument, validateDocumentStrict } from './chunk-ZDDJVR7X.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
6
6
|
|
|
7
7
|
// src/engine/orchestration/eligibility.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-
|
|
2
|
-
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-
|
|
1
|
+
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-F6BJYCF2.mjs';
|
|
2
|
+
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-F4MIGBBP.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/engine-hooks.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-
|
|
2
|
-
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-
|
|
1
|
+
import { createNumericIndex, createBooleanIndex, createEnumIndex, createLoadedGeoIndex, rebuildSizeIndexes, requirePatternSearch, patternIndexOf } from './chunk-F4MIGBBP.mjs';
|
|
2
|
+
import { getFlatSchema, schemaFieldsOf, getNestedValue } from './chunk-ZDDJVR7X.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/partition/pattern-snapshot.ts
|
|
5
5
|
function serializePatternIndexes(state, schema) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-
|
|
1
|
+
import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-WFQ6YQZA.mjs';
|
|
2
2
|
import { reactNativeHost } from './chunk-PVMEHBQO.mjs';
|
|
3
|
-
import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, compareCodePoints, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS
|
|
3
|
+
import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, MIN_M, compareCodePoints, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS } from './chunk-HRDMIQ63.mjs';
|
|
4
4
|
import { MAX_FILE_BYTES } from './chunk-VUJCSQG5.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
6
6
|
|
|
@@ -487,6 +487,8 @@ var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
|
487
487
|
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
488
488
|
var MAX_INDEX_NAME_LENGTH = 255;
|
|
489
489
|
var MAX_DOC_ID_LENGTH = 512;
|
|
490
|
+
var DEFAULT_MAX_PARTITIONS = 1;
|
|
491
|
+
var MAX_PARTITIONS = 65536;
|
|
490
492
|
var EXPORT_CHUNK_BYTES = 1048576;
|
|
491
493
|
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
492
494
|
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
@@ -581,14 +583,16 @@ function growBufferToExactly(buffer, bytes) {
|
|
|
581
583
|
if (buffer.byteLength >= bytes) return;
|
|
582
584
|
resizeBuffer(buffer, bytes, bytes);
|
|
583
585
|
}
|
|
586
|
+
function requireBytesWithinLimit(bytes, maxBytes) {
|
|
587
|
+
if (bytes <= maxBytes) return;
|
|
588
|
+
throw new NarsilError(
|
|
589
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
590
|
+
`A vector structure needs ${bytes} bytes where it may hold at most ${maxBytes}`,
|
|
591
|
+
{ bytes, maxBytes }
|
|
592
|
+
);
|
|
593
|
+
}
|
|
584
594
|
function resizeBuffer(buffer, bytes, target) {
|
|
585
|
-
|
|
586
|
-
throw new NarsilError(
|
|
587
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
588
|
-
`A vector structure needs ${bytes} bytes where it may hold at most ${buffer.maxByteLength}`,
|
|
589
|
-
{ bytes, maxBytes: buffer.maxByteLength }
|
|
590
|
-
);
|
|
591
|
-
}
|
|
595
|
+
requireBytesWithinLimit(bytes, buffer.maxByteLength);
|
|
592
596
|
if (buffer instanceof ArrayBuffer) {
|
|
593
597
|
buffer.resize(target);
|
|
594
598
|
return;
|
|
@@ -3491,6 +3495,16 @@ function markTombstone(state, ord) {
|
|
|
3491
3495
|
Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
3492
3496
|
}
|
|
3493
3497
|
}
|
|
3498
|
+
function reviveTombstone(state, ord) {
|
|
3499
|
+
if (!nodeExists(state, ord) || !isTombstoned(state, ord)) return;
|
|
3500
|
+
lockGraphShared(state.locks);
|
|
3501
|
+
try {
|
|
3502
|
+
clearTombstone(state, ord);
|
|
3503
|
+
} finally {
|
|
3504
|
+
unlockGraphShared(state.locks);
|
|
3505
|
+
}
|
|
3506
|
+
state.linkWrites += 1;
|
|
3507
|
+
}
|
|
3494
3508
|
function compactTombstones(state, mayFree2) {
|
|
3495
3509
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
3496
3510
|
countIncomingBeforeWriting(state, true);
|
|
@@ -4733,6 +4747,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4733
4747
|
if (ord !== void 0) markTombstone(state, ord);
|
|
4734
4748
|
},
|
|
4735
4749
|
markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
|
|
4750
|
+
reviveOrdinal: (ordinal) => reviveTombstone(state, ordinal),
|
|
4736
4751
|
holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
|
|
4737
4752
|
holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
|
|
4738
4753
|
has: (docId) => {
|
|
@@ -5612,6 +5627,30 @@ function decodeVectorIndexParts(raw) {
|
|
|
5612
5627
|
return raw.map(decodeVectorIndexPart);
|
|
5613
5628
|
}
|
|
5614
5629
|
|
|
5630
|
+
// src/vector/vector-index/upper-list-bound.ts
|
|
5631
|
+
var UPPER_LISTS_PER_NODE_FACTOR = 4;
|
|
5632
|
+
var UPPER_LISTS_ALLOWANCE = 64;
|
|
5633
|
+
function requireBoundedUpperLists(levelsOfParts, m, what) {
|
|
5634
|
+
let nodes = 0;
|
|
5635
|
+
let upperLists = 0;
|
|
5636
|
+
for (const levels of levelsOfParts) {
|
|
5637
|
+
for (let at = 0; at < levels.length; at++) {
|
|
5638
|
+
const level2 = levels[at];
|
|
5639
|
+
if (level2 === 0) continue;
|
|
5640
|
+
nodes++;
|
|
5641
|
+
upperLists += level2 - 1;
|
|
5642
|
+
}
|
|
5643
|
+
}
|
|
5644
|
+
const spread = m - 1;
|
|
5645
|
+
if (upperLists * spread <= UPPER_LISTS_PER_NODE_FACTOR * nodes + UPPER_LISTS_ALLOWANCE * spread) return;
|
|
5646
|
+
const limit = Math.floor(UPPER_LISTS_PER_NODE_FACTOR * nodes / spread) + UPPER_LISTS_ALLOWANCE;
|
|
5647
|
+
throw new NarsilError(
|
|
5648
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5649
|
+
`${what} holds ${upperLists} upper-layer lists over ${nodes} nodes, while a graph of ${nodes} nodes at m ${m} holds at most ${limit}`,
|
|
5650
|
+
{ upperLists, nodes, m, limit }
|
|
5651
|
+
);
|
|
5652
|
+
}
|
|
5653
|
+
|
|
5615
5654
|
// src/vector/vector-index/checkpoint-payload.ts
|
|
5616
5655
|
var VECTOR_FILE_PAYLOAD_VERSION = 1;
|
|
5617
5656
|
var VECTOR_GRAPH_PAYLOAD_VERSION = 1;
|
|
@@ -5674,15 +5713,15 @@ function calibratedMeanLengthOf(name, value) {
|
|
|
5674
5713
|
}
|
|
5675
5714
|
return value;
|
|
5676
5715
|
}
|
|
5677
|
-
function decodeNumberedGraph(raw
|
|
5716
|
+
function decodeNumberedGraph(raw) {
|
|
5678
5717
|
const name = "vector graph";
|
|
5679
5718
|
if (!isRecord2(raw)) invalid2(name, "each graph must be a map");
|
|
5680
5719
|
const entryPoint = raw.entryPoint ?? null;
|
|
5681
5720
|
if (entryPoint !== null && !isIntegerUpTo(entryPoint, 4294967295))
|
|
5682
5721
|
invalid2(name, "entryPoint must be a uint32 or nil");
|
|
5683
5722
|
if (!isIntegerUpTo(raw.maxLayer, 255)) invalid2(name, "maxLayer must be a uint8");
|
|
5684
|
-
if (!isIntegerUpTo(raw.m, MAX_M) || raw.m <
|
|
5685
|
-
invalid2(name, `m must be a whole number from ${
|
|
5723
|
+
if (!isIntegerUpTo(raw.m, MAX_M) || raw.m < MIN_M) {
|
|
5724
|
+
invalid2(name, `m must be a whole number from ${MIN_M} to ${MAX_M}`);
|
|
5686
5725
|
}
|
|
5687
5726
|
if (!isIntegerUpTo(raw.efConstruction, 65535)) invalid2(name, "efConstruction must be a uint16");
|
|
5688
5727
|
if (!isMetric(raw.metric)) invalid2(name, "metric must name a supported metric", { metric: raw.metric });
|
|
@@ -5713,7 +5752,7 @@ function decodeVectorGraphPayload(raw) {
|
|
|
5713
5752
|
return {
|
|
5714
5753
|
v: VECTOR_GRAPH_PAYLOAD_VERSION,
|
|
5715
5754
|
fieldName: raw.fieldName,
|
|
5716
|
-
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph
|
|
5755
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
|
|
5717
5756
|
};
|
|
5718
5757
|
}
|
|
5719
5758
|
function decodeVectorGraphPartPayload(raw) {
|
|
@@ -5725,7 +5764,7 @@ function decodeVectorGraphPartPayload(raw) {
|
|
|
5725
5764
|
return {
|
|
5726
5765
|
v: VECTOR_GRAPH_PART_PAYLOAD_VERSION,
|
|
5727
5766
|
fieldName: raw.fieldName,
|
|
5728
|
-
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph
|
|
5767
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
|
|
5729
5768
|
};
|
|
5730
5769
|
}
|
|
5731
5770
|
function sameGraphHeader(a, b) {
|
|
@@ -5767,10 +5806,21 @@ function assembleGraphParts(parts) {
|
|
|
5767
5806
|
}
|
|
5768
5807
|
requireOneGraphPerVector(part, p);
|
|
5769
5808
|
}
|
|
5809
|
+
for (let g = 0; g < partLists.length; g++) {
|
|
5810
|
+
const list = partLists[g];
|
|
5811
|
+
requireBoundedUpperLists(
|
|
5812
|
+
list.map((graph) => graph.levels),
|
|
5813
|
+
first.graphs[g].m,
|
|
5814
|
+
`Graph ${g} of the vector graph parts of field "${first.fieldName}"`
|
|
5815
|
+
);
|
|
5816
|
+
}
|
|
5770
5817
|
const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
|
|
5771
5818
|
return { fieldName: first.fieldName, graphs };
|
|
5772
5819
|
}
|
|
5773
5820
|
function graphPartsOfEarlierPayload(payload) {
|
|
5821
|
+
payload.graphs.forEach((graph, g) => {
|
|
5822
|
+
requireBoundedUpperLists([graph.levels], graph.m, `Graph ${g} of the vector graph of field "${payload.fieldName}"`);
|
|
5823
|
+
});
|
|
5774
5824
|
return {
|
|
5775
5825
|
fieldName: payload.fieldName,
|
|
5776
5826
|
graphs: payload.graphs.map((graph) => ({ partSize: Math.max(graph.levels.length, 1), parts: [graph] }))
|
|
@@ -6724,4 +6774,4 @@ function holdWritesAndBackgroundWork(read) {
|
|
|
6724
6774
|
function adoptBackgroundScheduler(_scheduler) {
|
|
6725
6775
|
}
|
|
6726
6776
|
|
|
6727
|
-
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_FILTER_THRESHOLD, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_RANK_CONSTANT, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DISK_STORAGE_BLOCK_BYTES, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, EXPORT_CHUNK_BYTES, FNV_OFFSET_BASIS, FNV_PRIME, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_OVERSAMPLE, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SHUTDOWN_TIMEOUT_MS, STATISTICS_BROADCAST_INTERVAL_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, WORKER_YOUNG_GENERATION_MB, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, adoptSharedTurns, allLiveDocIds, answerFromOneState, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindSearchSignal, bindVectorCoreHost, bindVectorHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, countWrite, coverOrdinal, createCentroidSums, createDocIdMap, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, detectRuntime, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUtf16, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, nodeThreadId, noteThreadSlotGone, noteThreadSlotLive, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requirePinnedState, requireSearchActive, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedStopOfSignal, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signalOfSearch, signatureOf, simdModuleLoaded, slotByteOffset, spawnNodeWorker, splitWorkerBudget, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, stopOfSignal, storePinsOf, threadSlotGone, threadSlotOfWorker, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnIsSpent, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, waitOutsideCountedWrite, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn, yieldTurn };
|
|
6777
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS, BATCH_CHUNK_SIZE, BM25_DISPLAYED_NAME_LENGTH, BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, CURSOR_BINDING_BUFFER_BYTES, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_FILTER_THRESHOLD, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PARTITIONS, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_RANK_CONSTANT, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_SUGGEST_LIMIT, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DISK_STORAGE_BLOCK_BYTES, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, EXPORT_CHUNK_BYTES, FNV_OFFSET_BASIS, FNV_PRIME, FOREIGN_STATS_TTL_MS, FUSION_TIE_INSERTION_LIMIT, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HYBRID_RESCORE_WINDOW, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_EXPORT_LINE_BYTES, MAX_HYBRID_DEPTH, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LINEAR_ALPHA, MAX_PARTITIONS, MAX_PHRASE_WORDS, MAX_PREFIX_LENGTH, MAX_QUERY_PHRASES, MAX_RENUMBERED_SEARCH_ATTEMPTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_OVERSAMPLE, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SCAN_CHUNK_ORDINALS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, RESULT_WINDOW, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, SHUTDOWN_TIMEOUT_MS, STATISTICS_BROADCAST_INTERVAL_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, WORKER_YOUNG_GENERATION_MB, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, adoptSharedTurns, allLiveDocIds, answerFromOneState, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindSearchSignal, bindVectorCoreHost, bindVectorHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, changeIndexOutsideExternalWaits, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, countWrite, coverOrdinal, createCentroidSums, createDocIdMap, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, detectRuntime, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUtf16, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, indexChangeOutsideExternalWaits, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, nodeThreadId, noteThreadSlotGone, noteThreadSlotLive, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireBoundedUpperLists, requireBytesWithinLimit, requirePinnedState, requireSearchActive, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedStopOfSignal, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signalOfSearch, signatureOf, simdModuleLoaded, slotByteOffset, spawnNodeWorker, splitWorkerBudget, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, stopOfSignal, storePinsOf, threadSlotGone, threadSlotOfWorker, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnIsSpent, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, waitOutsideCountedWrite, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn, yieldTurn };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-
|
|
1
|
+
import { REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, collectYoungGeneration, collectAndReturnPages, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE } from './chunk-I7DTUOK2.mjs';
|
|
2
2
|
|
|
3
3
|
// src/runtime/thread-heap.ts
|
|
4
4
|
var reservedBytes = 0;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-
|
|
1
|
+
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-ZDDJVR7X.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-TQ26BKBH.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/bitset.ts
|
|
@@ -16,6 +16,11 @@ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
|
|
|
16
16
|
|
|
17
17
|
// src/schema/validator/shared.ts
|
|
18
18
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
19
|
+
var VECTOR_TYPE_PATTERN = /^vector\[(.*)]$/;
|
|
20
|
+
function vectorDimensionOf(fieldType) {
|
|
21
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
22
|
+
return match === null ? null : Number.parseInt(match[1], 10);
|
|
23
|
+
}
|
|
19
24
|
function isGeopointOrVectorType(fieldType) {
|
|
20
25
|
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
21
26
|
}
|
|
@@ -57,15 +62,15 @@ function invalidType(path, type, message) {
|
|
|
57
62
|
}
|
|
58
63
|
function parseBase(baseName, path, type) {
|
|
59
64
|
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
60
|
-
const vectorMatch =
|
|
65
|
+
const vectorMatch = VECTOR_TYPE_PATTERN.exec(baseName);
|
|
61
66
|
if (vectorMatch === null) {
|
|
62
67
|
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
63
68
|
}
|
|
64
|
-
const dimension = Number.
|
|
65
|
-
if (dimension
|
|
69
|
+
const dimension = vectorDimensionOf(baseName) ?? Number.NaN;
|
|
70
|
+
if (!(dimension > 0 && dimension <= MAX_STORED_VECTOR_DIMENSION)) {
|
|
66
71
|
throw new NarsilError(
|
|
67
72
|
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
68
|
-
`Vector field "${path}" has ${
|
|
73
|
+
`Vector field "${path}" has ${vectorMatch[1]} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
69
74
|
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
70
75
|
);
|
|
71
76
|
}
|
|
@@ -218,7 +223,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
218
223
|
);
|
|
219
224
|
}
|
|
220
225
|
for (let i = 0; i < arr.length; i++) {
|
|
221
|
-
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
226
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(Math.fround(arr[i]))) {
|
|
222
227
|
throw new NarsilError(
|
|
223
228
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
224
229
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -802,7 +807,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
|
|
|
802
807
|
table.indexMap.set(fieldName, idx);
|
|
803
808
|
return idx;
|
|
804
809
|
}
|
|
805
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
806
810
|
function getNestedValue(obj, path) {
|
|
807
811
|
if (obj === null || typeof obj !== "object") return void 0;
|
|
808
812
|
if (!path.includes(".")) return Object.hasOwn(obj, path) ? obj[path] : void 0;
|
|
@@ -838,8 +842,7 @@ function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
|
838
842
|
return false;
|
|
839
843
|
}
|
|
840
844
|
function parseVectorDimension(fieldType) {
|
|
841
|
-
|
|
842
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
845
|
+
return vectorDimensionOf(fieldType);
|
|
843
846
|
}
|
|
844
847
|
function tokenizeOptions(options) {
|
|
845
848
|
return {
|
|
@@ -847,7 +850,8 @@ function tokenizeOptions(options) {
|
|
|
847
850
|
removeStopWords: true,
|
|
848
851
|
collectSurfaces: options?.collectSurfaces === true,
|
|
849
852
|
stopWordOverride: options?.stopWordOverride,
|
|
850
|
-
customTokenizer: options?.customTokenizer
|
|
853
|
+
customTokenizer: options?.customTokenizer,
|
|
854
|
+
textSource: "document"
|
|
851
855
|
};
|
|
852
856
|
}
|
|
853
857
|
function getFlatSchema(state, schema) {
|
|
@@ -862,7 +866,7 @@ var loaded = null;
|
|
|
862
866
|
var loading = null;
|
|
863
867
|
function loadMessagePack() {
|
|
864
868
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
865
|
-
loading ??= import('./msgpack-leaf-
|
|
869
|
+
loading ??= import('./msgpack-leaf-GTKLF6GJ.mjs').then(
|
|
866
870
|
(code) => {
|
|
867
871
|
code.bindMsgpackHost({
|
|
868
872
|
BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER,
|
|
@@ -895,4 +899,4 @@ function requireLoadedMessagePack() {
|
|
|
895
899
|
return loaded;
|
|
896
900
|
}
|
|
897
901
|
|
|
898
|
-
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, withSortableOption };
|
|
902
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorDimensionOf, withSortableOption };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadSlotGone, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn } from './chunk-
|
|
2
|
-
export { NATIVE_OK, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback } from './chunk-
|
|
1
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_LOCK_WAIT_DEADLINE_MS, GRAPH_LOCK_WAIT_SLICE_MS, GRAPH_RETIRED, IN_MEMORY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, MAX_DOC_ID_LENGTH, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, MAX_WASM_PAGES, MIN_OVERSAMPLE, NATIVE_ATTACH_RETRY_MS, NATIVE_THREAD_SEARCH_SLOTS, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, REFUSAL_COLLECTION_MIN_INTERVAL_MS, REFUSAL_COLLECTION_TIME_MULTIPLE, REFUSAL_YOUNG_COLLECTION_MIN_INTERVAL_MS, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, UI_THREAD_BACKGROUND_SLICE_DURING_WRITES_MS, UI_THREAD_HOP_BEFORE_TURN_WITHIN_MS, UI_THREAD_TURN_BUDGET_MS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, VECTOR_BLOCK_MAX_BYTES, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_SCRATCH_SLOTS, VECTOR_STORE_INITIAL_CAPACITY, VECTOR_WORKER_COPIES_ALLOWED, WASM_PAGE_BYTES, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, backgroundTurnIsSpent, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, collectAndReturnPages, collectYoungGeneration, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, holdWritesAndBackgroundWork, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireBoundedUpperLists, requireBytesWithinLimit, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadSlotGone, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, turnsAreBudgeted, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBackgroundTurn, yieldBuildTurn } from './chunk-I7DTUOK2.mjs';
|
|
2
|
+
export { NATIVE_OK, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback } from './chunk-WFQ6YQZA.mjs';
|
|
3
3
|
import './chunk-PVMEHBQO.mjs';
|
|
4
4
|
export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_EF_SEARCH, EARLIER_RELEASE_MIN_M, FARTHEST_REJECTED_LINKS_KEPT, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, MIN_M, NEAREST_LINK_FALLBACK_HOSTS, PLACEMENT_ROOM_REQUESTS, compareCodePoints } from './chunk-HRDMIQ63.mjs';
|
|
5
5
|
export { MAX_FILE_BYTES } from './chunk-VUJCSQG5.mjs';
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// src/analysis/constants.ts
|
|
2
|
+
var MAX_TOKEN_POSITION = 4294967295;
|
|
3
|
+
|
|
4
|
+
// src/analysis/custom-token-host.ts
|
|
5
|
+
var ErrorCodes;
|
|
6
|
+
var NarsilError;
|
|
7
|
+
var withUnpairedSurrogatesReplaced;
|
|
8
|
+
function bindCustomTokenHost(host) {
|
|
9
|
+
({ ErrorCodes, NarsilError, withUnpairedSurrogatesReplaced } = host);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// src/analysis/custom-token-results.ts
|
|
13
|
+
function refuseTokenizerResult(source, message, details) {
|
|
14
|
+
const code = source === "document" ? ErrorCodes.DOC_VALIDATION_FAILED : ErrorCodes.CONFIG_INVALID;
|
|
15
|
+
throw new NarsilError(code, `The custom tokenizer ${message}`, details);
|
|
16
|
+
}
|
|
17
|
+
function kindOf(value) {
|
|
18
|
+
if (value === null) return "null";
|
|
19
|
+
return Array.isArray(value) ? "array" : typeof value;
|
|
20
|
+
}
|
|
21
|
+
function checkedTokenizerEntry(entry, at, source) {
|
|
22
|
+
const { token, position } = entry !== null && typeof entry === "object" ? entry : {};
|
|
23
|
+
if (typeof token !== "string") {
|
|
24
|
+
refuseTokenizerResult(
|
|
25
|
+
source,
|
|
26
|
+
`returned a token of type ${kindOf(token)} at entry ${at}, where a token is a string`,
|
|
27
|
+
{ entry: at, tokenType: kindOf(token) }
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
if (typeof position !== "number" || !Number.isInteger(position) || position < 0 || position > MAX_TOKEN_POSITION) {
|
|
31
|
+
const shown = typeof position === "number" ? String(position) : kindOf(position);
|
|
32
|
+
refuseTokenizerResult(
|
|
33
|
+
source,
|
|
34
|
+
`returned the position ${shown} at entry ${at}, where a position is a whole number from 0 to ${MAX_TOKEN_POSITION}`,
|
|
35
|
+
{ entry: at, position: shown }
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
return { token: withUnpairedSurrogatesReplaced(token), position };
|
|
39
|
+
}
|
|
40
|
+
function holdsPositionsOutOfOrder(entries) {
|
|
41
|
+
for (let index = 1; index < entries.length; index++) {
|
|
42
|
+
if (entries[index].position < entries[index - 1].position) return true;
|
|
43
|
+
}
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
function customTokensOf(tokenizer, text, source) {
|
|
47
|
+
const entries = tokenizer.tokenize(text);
|
|
48
|
+
if (!Array.isArray(entries)) {
|
|
49
|
+
refuseTokenizerResult(source, `returned ${kindOf(entries)}, where it returns a list of tokens`, {
|
|
50
|
+
received: kindOf(entries)
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
const checked = new Array(entries.length);
|
|
54
|
+
for (let at = 0; at < checked.length; at++) checked[at] = checkedTokenizerEntry(entries[at], at, source);
|
|
55
|
+
if (!holdsPositionsOutOfOrder(checked)) return checked;
|
|
56
|
+
return checked.sort((first, second) => first.position - second.position);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export { bindCustomTokenHost, customTokensOf };
|
|
@@ -33,10 +33,10 @@ function validateVector(vector, targetField, expectedDimensions) {
|
|
|
33
33
|
);
|
|
34
34
|
}
|
|
35
35
|
for (let i = 0; i < vector.length; i++) {
|
|
36
|
-
if (Number.
|
|
36
|
+
if (!Number.isFinite(vector[i])) {
|
|
37
37
|
throw new NarsilError(
|
|
38
38
|
ErrorCodes.EMBEDDING_FAILED,
|
|
39
|
-
`Adapter returned vector with
|
|
39
|
+
`Adapter returned vector with ${vector[i]} at index ${i} for field "${targetField}"`,
|
|
40
40
|
{ field: targetField, index: i }
|
|
41
41
|
);
|
|
42
42
|
}
|