@delali/narsil 0.3.1-next.155 → 0.3.1-next.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +45 -0
- package/dist/{analysis-change-3TFIHT5E.mjs → analysis-change-5BQOQ4BF.mjs} +25 -18
- package/dist/{react-native/chunks/analysis-change-entry-JLL42PH6.mjs → analysis-change-entry-TXX6JSMX.mjs} +1 -1
- package/dist/browser/{analysis-change-TC36BU47.mjs → analysis-change-435CUEPZ.mjs} +8 -7
- package/dist/{analysis-change-entry-KA6JAO4X.mjs → browser/analysis-change-entry-HZXKPPGV.mjs} +1 -1
- package/dist/browser/{chunk-A5KUEGDY.mjs → chunk-2CFI5VVV.mjs} +2 -2
- package/dist/browser/{chunk-BXM5XOCA.mjs → chunk-2TI4SSUP.mjs} +2 -2
- package/dist/browser/{chunk-CJSYBQ4J.mjs → chunk-3SSRJMFN.mjs} +1 -1
- package/dist/browser/{chunk-SMEQ36NT.mjs → chunk-3V7DNNZL.mjs} +458 -622
- package/dist/browser/{chunk-RRO3EV4C.mjs → chunk-4NGKC4H4.mjs} +7 -7
- package/dist/browser/{chunk-ABCXENU5.mjs → chunk-6LWDPLTB.mjs} +1 -1
- package/dist/browser/{chunk-FNIFPFB7.mjs → chunk-7U3WVJUP.mjs} +2 -2
- package/dist/browser/{chunk-KKAMCABR.mjs → chunk-BHFQATSC.mjs} +1 -1
- package/dist/browser/{chunk-IKDKX5VQ.mjs → chunk-CRE3G7F3.mjs} +3 -3
- package/dist/browser/{chunk-56W3HQVJ.mjs → chunk-EA63E3D2.mjs} +4 -4
- package/dist/browser/{chunk-K5AXJBLL.mjs → chunk-IZXVW2ZJ.mjs} +3 -3
- package/dist/browser/{chunk-RND5CLX3.mjs → chunk-K4Q6KX3R.mjs} +16 -16
- package/dist/browser/{chunk-XY4VVBBO.mjs → chunk-KZWSX6BH.mjs} +2 -2
- package/dist/browser/{chunk-3UIEB4QF.mjs → chunk-MGI4BCXC.mjs} +2 -2
- package/dist/browser/chunk-MOG4OVDK.mjs +51 -0
- package/dist/browser/{chunk-RQBUNZQW.mjs → chunk-NQAME3RK.mjs} +2 -2
- package/dist/browser/{chunk-LFIWRCEA.mjs → chunk-NXQG7DRA.mjs} +29 -6
- package/dist/browser/{chunk-JVBP5EYO.mjs → chunk-SSTPBZZX.mjs} +7 -7
- package/dist/browser/{chunk-JIJXMI42.mjs → chunk-STOWQFT3.mjs} +1 -1
- package/dist/browser/{chunk-26A5JFT2.mjs → chunk-TK63WFXK.mjs} +2 -1
- package/dist/browser/{chunk-4EWUGZYZ.mjs → chunk-U4YDIBUT.mjs} +2 -2
- package/dist/browser/{chunk-K4GCSZLI.mjs → chunk-VLJQW4J6.mjs} +2985 -2985
- package/dist/browser/{chunk-SUDNZKPL.mjs → chunk-WI4MR75E.mjs} +1 -1
- package/dist/browser/{chunk-5TCXK5SB.mjs → chunk-WN464PTL.mjs} +1 -1
- package/dist/browser/{cold-module-NEUGHUPA.mjs → cold-module-2EHCIV4K.mjs} +15 -14
- package/dist/browser/{durability-wiring-CVGPRO6D.mjs → durability-wiring-A7BVG2DW.mjs} +20 -19
- package/dist/browser/{engine-hooks-YXMPHZKZ.mjs → engine-hooks-E4UD67JD.mjs} +7 -6
- package/dist/browser/geo-index-HHWGMNIS.mjs +1386 -0
- package/dist/browser/{import-OM6QAW45.mjs → import-DXYB6756.mjs} +8 -7
- package/dist/browser/{module-E33X4AOK.mjs → module-NIYMEA5R.mjs} +11 -10
- package/dist/browser/{module-3HNFCBJ5.mjs → module-TCT66FWA.mjs} +6 -6
- package/dist/browser/{phrase-module-ZXXU7S3B.mjs → phrase-module-HAT3TLZN.mjs} +1 -1
- package/dist/browser/{restore-hooks-IF2ZROUP.mjs → restore-hooks-YPN4ZFCY.mjs} +10 -9
- package/dist/browser/{restore-queue-ZBS7CJA6.mjs → restore-queue-G3RQGL3Y.mjs} +3 -3
- package/dist/browser/{retention-ASWUHAYI.mjs → retention-ZWVMNC3Q.mjs} +9 -8
- package/dist/browser/{snapshot-PN7ZW6Q4.mjs → snapshot-VEQNIE6C.mjs} +24 -23
- package/dist/browser/{start-ZIRB7MQ2.mjs → start-BM442C6R.mjs} +11 -10
- package/dist/browser/{write-HMDYCTV2.mjs → write-2EVMZWDE.mjs} +12 -11
- package/dist/{chunk-2MANFZJ3.mjs → chunk-25KDNIZZ.mjs} +1 -1
- package/dist/{chunk-3HHQLEEQ.mjs → chunk-2UV4QBHW.mjs} +7 -7
- package/dist/{chunk-BY6ZDGYS.mjs → chunk-3HYC4JRF.mjs} +1 -1
- package/dist/{chunk-HW5TNXBO.mjs → chunk-46KY2PJD.mjs} +7 -6
- package/dist/{chunk-SY3EWKQY.mjs → chunk-4KKXNJM6.mjs} +1 -1
- package/dist/{chunk-VQEUR6T6.mjs → chunk-5VV4R6P7.mjs} +3040 -3040
- package/dist/chunk-6HTGOXIK.mjs +875 -0
- package/dist/{chunk-RSVULW4Q.mjs → chunk-6I6AYAXS.mjs} +4 -4
- package/dist/chunk-7GFE3PEY.mjs +40 -0
- package/dist/{chunk-3R4UZWQG.mjs → chunk-7LXHHCXZ.mjs} +15 -15
- package/dist/{chunk-JUKYYXSP.mjs → chunk-7V5IUP2T.mjs} +1 -31
- package/dist/{chunk-4BYWRYPM.mjs → chunk-A4L7YTKX.mjs} +52 -89
- package/dist/{chunk-WQRN64Q6.mjs → chunk-BU4SAIC4.mjs} +1 -1
- package/dist/{chunk-5FVU47TB.mjs → chunk-BWZ3P4MU.mjs} +3 -3
- package/dist/{chunk-LC3FDWTX.mjs → chunk-CITFCZKF.mjs} +3 -2
- package/dist/{chunk-PCRA7HTI.mjs → chunk-CSLGFPFQ.mjs} +1 -1
- package/dist/{chunk-VHBNF4MA.mjs → chunk-DP5C7YFQ.mjs} +2 -2
- package/dist/{chunk-3HB3PEC6.mjs → chunk-DWEPMFJE.mjs} +5 -5
- package/dist/{chunk-KMUNHGBQ.mjs → chunk-EFBDHH5P.mjs} +29 -28
- package/dist/{chunk-DCNKVEDB.mjs → chunk-FAKFUDZZ.mjs} +5 -6
- package/dist/chunk-FB6463IN.mjs +1499 -0
- package/dist/{chunk-KFLK5NA2.mjs → chunk-FCJ553HS.mjs} +1 -1
- package/dist/{chunk-SG3JBNHY.mjs → chunk-FJKKCV4D.mjs} +1 -1
- package/dist/{chunk-H6VIGE62.mjs → chunk-FNGAM5B7.mjs} +10 -9
- package/dist/{chunk-VGF7CFRP.mjs → chunk-FRBFVQFN.mjs} +5 -4
- package/dist/{chunk-QH3YIQP4.mjs → chunk-FYOVBEMM.mjs} +2 -2
- package/dist/chunk-G6HRSGOU.mjs +74 -0
- package/dist/{chunk-DUOLGBTE.mjs → chunk-HFPMMRQT.mjs} +3 -3
- package/dist/{chunk-ZHCUZZE5.mjs → chunk-HHBUCIUY.mjs} +1 -1
- package/dist/{chunk-ZTW5WTGG.mjs → chunk-I2KPOLOZ.mjs} +3 -3
- package/dist/{chunk-L7ZD4L3K.mjs → chunk-I53G4JAU.mjs} +7 -6
- package/dist/{chunk-YEDIDCR7.mjs → chunk-I5B7RXU7.mjs} +91 -22
- package/dist/{react-native/chunks/chunk-KWRW5U65.mjs → chunk-J4IJTTEX.mjs} +1 -1
- package/dist/{chunk-S2EQU2GE.mjs → chunk-JDJBQLRO.mjs} +9 -9
- package/dist/{chunk-KPIV4RUC.mjs → chunk-JWRUUCWD.mjs} +2 -2
- package/dist/{chunk-KBKEXQZA.mjs → chunk-K5QFDYST.mjs} +1 -1
- package/dist/{chunk-DIBHI6W5.mjs → chunk-KF4EXSM4.mjs} +2 -2
- package/dist/{chunk-Q7G3BV3Y.mjs → chunk-KKQC526K.mjs} +19 -11
- package/dist/{chunk-4KRC2APY.mjs → chunk-L6V6BST3.mjs} +6 -19
- package/dist/{chunk-U4BS74YN.mjs → chunk-MQOJIO5B.mjs} +1 -1
- package/dist/chunk-ORN5BLL7.mjs +11 -0
- package/dist/{chunk-ZAUJFLEN.mjs → chunk-P5ZI2P2W.mjs} +122 -1697
- package/dist/{chunk-AILVAOLX.mjs → chunk-PWC4VB3Q.mjs} +2 -2
- package/dist/{chunk-J4IWBIVY.mjs → chunk-QOW2JPUI.mjs} +5 -5
- package/dist/{chunk-PHQNACDT.mjs → chunk-RDXO6JST.mjs} +2 -2
- package/dist/chunk-RLOES5SX.mjs +31 -0
- package/dist/{chunk-6L3XJXGU.mjs → chunk-SA5LOGO3.mjs} +1 -1
- package/dist/chunk-SQJWAIF4.mjs +12 -0
- package/dist/{react-native/chunks/chunk-PPEPTUVV.mjs → chunk-TBC3XUL2.mjs} +29 -6
- package/dist/chunk-TDIK6NTV.mjs +10 -0
- package/dist/{chunk-MT2NRTA6.mjs → chunk-TGS5UXBE.mjs} +3 -3
- package/dist/{chunk-KBDBPVLN.mjs → chunk-TZ3355VS.mjs} +1 -1
- package/dist/{chunk-Q622CTMC.mjs → chunk-TZTRKM33.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-JV6VQY2S.mjs → chunk-U55XJEIG.mjs} +2 -1
- package/dist/{chunk-Z2BE2MAM.mjs → chunk-VTTRFF2C.mjs} +28 -15
- package/dist/{chunk-KXCVQWOR.mjs → chunk-WX4LAO6M.mjs} +3 -3
- package/dist/{chunk-BDRIVYOK.mjs → chunk-XAXKV65G.mjs} +1 -1
- package/dist/{chunk-UZRZKNNZ.mjs → chunk-XXHN7J46.mjs} +1 -1
- package/dist/{chunk-H4KHPLHJ.mjs → chunk-YTWZVGCB.mjs} +1 -1
- package/dist/{chunk-QUJHMPTP.mjs → chunk-Z2KAFCRT.mjs} +2 -2
- package/dist/{chunk-SMVH6JBE.mjs → chunk-ZKP7G7TV.mjs} +1 -1
- package/dist/{cold-module-7XXCEGTZ.mjs → cold-module-6MO2XJF2.mjs} +29 -23
- package/dist/direct-executor-FTMYY3TI.mjs +34 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution.mjs +57 -51
- package/dist/{durability-wiring-NCKUC5EO.mjs → durability-wiring-VIF5MOSY.mjs} +41 -34
- package/dist/{engine-hooks-DHCMOADY.mjs → engine-hooks-EGUGNZYE.mjs} +24 -17
- package/dist/geo-index-MRH5JMIR.mjs +1387 -0
- package/dist/{import-NBFXFCSG.mjs → import-XKDPJQDP.mjs} +24 -17
- package/dist/index.browser.mjs +6 -5
- package/dist/index.d.ts +127 -2
- package/dist/index.mjs +23 -16
- package/dist/index.react-native.mjs +12 -10
- package/dist/invalidation/filesystem.mjs +2 -2
- package/dist/{long-query-thread-J4DVRM4A.mjs → long-query-thread-YA4IAYSY.mjs} +25 -20
- package/dist/{module-DWKPBSMN.mjs → module-FQ4JWNQJ.mjs} +26 -19
- package/dist/{module-UN76TF7I.mjs → module-N55W4Q5C.mjs} +9 -9
- package/dist/{paged-partition-archive-DPNWVSK2.mjs → paged-partition-archive-PXW5XAYU.mjs} +32 -26
- package/dist/partial-words.mjs +8 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +30 -21
- package/dist/{phrase-module-ZI4WSQVH.mjs → phrase-module-XW5MKWP5.mjs} +3 -2
- package/dist/query-suggestions/merge-worker.mjs +25 -18
- package/dist/query-suggestions.browser.mjs +12 -11
- package/dist/query-suggestions.mjs +34 -27
- package/dist/react-native/chunks/{analysis-change-6BPDPH6K.mjs → analysis-change-LT5QBQ3H.mjs} +13 -11
- package/dist/{browser/analysis-change-entry-4MH3CZXH.mjs → react-native/chunks/analysis-change-entry-SCVWK6OU.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-7PXRWYUU.mjs → chunk-2PRHILX3.mjs} +53 -19
- package/dist/react-native/chunks/{chunk-C3PTWYKT.mjs → chunk-3JWDECAZ.mjs} +4 -4
- package/dist/{chunk-R6A773WK.mjs → react-native/chunks/chunk-3WI35N75.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-CUH5F57W.mjs → chunk-4JYO3JD2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-3JIBF3XT.mjs → chunk-DNXOMPAQ.mjs} +1 -1
- package/dist/react-native/chunks/chunk-DXRSEWA2.mjs +12 -0
- package/dist/{chunk-YTJCGKM4.mjs → react-native/chunks/chunk-EJ33LCHV.mjs} +29 -6
- package/dist/react-native/chunks/{chunk-EIOYM5TO.mjs → chunk-ENLMYKSX.mjs} +3024 -3054
- package/dist/react-native/chunks/chunk-FHPW7P2O.mjs +40 -0
- package/dist/{chunk-4UMZ7KQL.mjs → react-native/chunks/chunk-FTZHMRD2.mjs} +2 -1
- package/dist/react-native/chunks/{chunk-OL3WDV53.mjs → chunk-G43IPIDG.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-4Y53T4YE.mjs → chunk-GXX3DQXP.mjs} +5 -19
- package/dist/react-native/chunks/{chunk-GIZTJXYW.mjs → chunk-KV46CT4D.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HUOYZJ3Q.mjs → chunk-MIP4SOR6.mjs} +1419 -711
- package/dist/react-native/chunks/{chunk-ALOIBXWN.mjs → chunk-MUWD2H2S.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-4LCEN2IU.mjs → chunk-NB3LP33V.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VXGE4ZXL.mjs → chunk-PDRWWPKV.mjs} +17 -17
- package/dist/react-native/chunks/{chunk-N2WUOWLL.mjs → chunk-POPDAGH6.mjs} +7 -7
- package/dist/react-native/chunks/{chunk-GJFXPHXI.mjs → chunk-QUKQNI3H.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-KTTAU63H.mjs → chunk-SK2PWYDH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-7GRCBBME.mjs → chunk-TP6KCKER.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-KJIG57UD.mjs → chunk-UQ4OAXKU.mjs} +7 -7
- package/dist/react-native/chunks/{chunk-PEHS2UO3.mjs → chunk-VWUDIOT6.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-JLIIWJUV.mjs → chunk-YZW4CHM6.mjs} +50 -11
- package/dist/react-native/chunks/{chunk-QJXTNPLQ.mjs → chunk-ZLWVW734.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-TYIRMPD3.mjs → chunk-ZXZFN7FI.mjs} +1 -1
- package/dist/react-native/chunks/{cold-module-5PHF6DZR.mjs → cold-module-KVZYUGRY.mjs} +19 -17
- package/dist/react-native/chunks/{durability-wiring-KO565FZC.mjs → durability-wiring-34OYO32E.mjs} +27 -25
- package/dist/react-native/chunks/{engine-hooks-MVRRHH6B.mjs → engine-hooks-M7JV46NH.mjs} +13 -11
- package/dist/react-native/chunks/geo-index-PX3BIHAH.mjs +1387 -0
- package/dist/react-native/chunks/{import-YVZA4V4V.mjs → import-H6AQREWI.mjs} +14 -12
- package/dist/react-native/chunks/{module-ZL5ZG6YB.mjs → module-LBZPGOWH.mjs} +15 -13
- package/dist/react-native/chunks/{module-H5MVPSP2.mjs → module-TTEVPRCW.mjs} +8 -8
- package/dist/react-native/chunks/{phrase-module-RTVKEWJT.mjs → phrase-module-QK22HSQ2.mjs} +1 -1
- package/dist/react-native/chunks/{restore-hooks-SILFPU6B.mjs → restore-hooks-IBGIY3DR.mjs} +17 -15
- package/dist/react-native/chunks/{restore-queue-HSYP2AEC.mjs → restore-queue-ZZX54AH7.mjs} +3 -3
- package/dist/react-native/chunks/{retention-7I7UAWYK.mjs → retention-EW4TTDFB.mjs} +18 -16
- package/dist/react-native/chunks/{snapshot-EZYB6SLF.mjs → snapshot-REYVIN3X.mjs} +31 -29
- package/dist/react-native/chunks/{start-SHVZ72I5.mjs → start-BQK6KN3R.mjs} +18 -16
- package/dist/react-native/chunks/{write-JUESXHDL.mjs → write-FQAYCZF5.mjs} +16 -14
- package/dist/react-native/partial-words.mjs +7 -6
- package/dist/react-native/pattern.mjs +1 -1
- package/dist/react-native/query-suggestions.mjs +19 -17
- package/dist/react-native/sync.mjs +7 -7
- package/dist/react-native/synonyms.mjs +92 -24
- package/dist/{restore-hooks-HZ3ILAMO.mjs → restore-hooks-CDERDEF4.mjs} +28 -21
- package/dist/{restore-queue-JVGENGGE.mjs → restore-queue-7NNSTROA.mjs} +3 -3
- package/dist/{retention-HROPV2QU.mjs → retention-QSVBBNRM.mjs} +28 -21
- package/dist/server.mjs +45 -39
- package/dist/{snapshot-4HS2G2UM.mjs → snapshot-RH4NSKIZ.mjs} +44 -38
- package/dist/{start-UTWHGH42.mjs → start-MCKCOGIW.mjs} +31 -24
- package/dist/sync/filesystem.mjs +3 -3
- package/dist/sync.browser.mjs +5 -5
- package/dist/sync.mjs +5 -5
- package/dist/synonyms.mjs +7 -6
- package/dist/{thread-6MOXY2KD.mjs → thread-O4SBJJJS.mjs} +24 -19
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +17 -6
- package/dist/{write-GTTWWRRG.mjs → write-3FK7NG3Y.mjs} +26 -19
- package/package.json +15 -8
- package/dist/chunk-DJHHF2L5.mjs +0 -17
- package/dist/chunk-HNVVEQP7.mjs +0 -44
- package/dist/direct-executor-G5TK7NLA.mjs +0 -29
- package/dist/browser/{chunk-4WCVMOU3.mjs → chunk-XAWNZ4G2.mjs} +10 -10
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { compareCodePoints } from './chunk-
|
|
2
|
-
import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-
|
|
1
|
+
import { withUnpairedSurrogatesReplaced, compareCodePoints } from './chunk-EJ33LCHV.mjs';
|
|
2
|
+
import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-FTZHMRD2.mjs';
|
|
3
3
|
import { createNarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
|
|
4
4
|
|
|
5
5
|
// src/core/tokenizer/constants.ts
|
|
@@ -14,7 +14,19 @@ var MEDIUM_DEVICE_MEMORY_GB = 4;
|
|
|
14
14
|
var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
|
|
15
15
|
var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
|
|
16
16
|
var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
|
|
17
|
+
var NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS = 256;
|
|
18
|
+
var NATIVE_ANALYSIS_HELD_TERMS = 16384;
|
|
17
19
|
var DEFAULT_MIN_TOKEN_LENGTH = 1;
|
|
20
|
+
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
21
|
+
|
|
22
|
+
// src/core/tokenizer/native-hook.ts
|
|
23
|
+
var installed = null;
|
|
24
|
+
function installNativeAnalysis(analysis) {
|
|
25
|
+
installed = analysis;
|
|
26
|
+
}
|
|
27
|
+
function nativeAnalysis() {
|
|
28
|
+
return installed;
|
|
29
|
+
}
|
|
18
30
|
|
|
19
31
|
// src/core/tokenizer/normalize.ts
|
|
20
32
|
var DOTTED_CAPITAL_I = "\u0130";
|
|
@@ -156,21 +168,21 @@ function evictOldestEntries(count) {
|
|
|
156
168
|
}
|
|
157
169
|
}
|
|
158
170
|
}
|
|
171
|
+
function transformInTypeScript(raw, language, stem, removeDiacritics) {
|
|
172
|
+
let normalized = raw;
|
|
173
|
+
if (language.normalizer) normalized = language.normalizer(normalized);
|
|
174
|
+
if (removeDiacritics) normalized = stripDiacritics(normalized);
|
|
175
|
+
if (stem && language.stemmer) normalized = language.stemmer(normalized);
|
|
176
|
+
return normalized;
|
|
177
|
+
}
|
|
159
178
|
function transformToken(raw, language, stem, removeDiacritics) {
|
|
160
179
|
const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
|
|
161
180
|
let bucket = getCacheBucket(language, flags);
|
|
162
181
|
const cached = bucket.get(raw);
|
|
163
182
|
if (cached !== void 0) return cached;
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
}
|
|
168
|
-
if (removeDiacritics) {
|
|
169
|
-
normalized = stripDiacritics(normalized);
|
|
170
|
-
}
|
|
171
|
-
if (stem && language.stemmer) {
|
|
172
|
-
normalized = language.stemmer(normalized);
|
|
173
|
-
}
|
|
183
|
+
const normalized = withUnpairedSurrogatesReplaced(
|
|
184
|
+
nativeAnalysis()?.transform(raw, language, stem, removeDiacritics) ?? transformInTypeScript(raw, language, stem, removeDiacritics)
|
|
185
|
+
);
|
|
174
186
|
if (normalizationCacheSize >= maxCacheSize) {
|
|
175
187
|
evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
|
|
176
188
|
bucket = getCacheBucket(language, flags);
|
|
@@ -332,7 +344,6 @@ function producesSurfaceForms(language, options) {
|
|
|
332
344
|
if (!options?.collectSurfaces || options.customTokenizer) return false;
|
|
333
345
|
return (options.stem ?? true) && language.stemmer !== null;
|
|
334
346
|
}
|
|
335
|
-
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
336
347
|
function resolveStopWords(language, override) {
|
|
337
348
|
if (!override) return language.stopWords;
|
|
338
349
|
if (override instanceof Set) return override;
|
|
@@ -362,6 +373,18 @@ function stripPossessive(token) {
|
|
|
362
373
|
}
|
|
363
374
|
return token;
|
|
364
375
|
}
|
|
376
|
+
function customTokensOf(tokenizer, text) {
|
|
377
|
+
const entries = tokenizer.tokenize(text);
|
|
378
|
+
let repaired = null;
|
|
379
|
+
for (let at = 0; at < entries.length; at++) {
|
|
380
|
+
const entry = entries[at];
|
|
381
|
+
const token = withUnpairedSurrogatesReplaced(entry.token);
|
|
382
|
+
if (token === entry.token) continue;
|
|
383
|
+
repaired ??= entries.slice();
|
|
384
|
+
repaired[at] = { token, position: entry.position };
|
|
385
|
+
}
|
|
386
|
+
return repaired ?? entries;
|
|
387
|
+
}
|
|
365
388
|
function visitTokens(text, language, options, visit) {
|
|
366
389
|
const {
|
|
367
390
|
stem = true,
|
|
@@ -372,9 +395,12 @@ function visitTokens(text, language, options, visit) {
|
|
|
372
395
|
stopWordOverride
|
|
373
396
|
} = options ?? {};
|
|
374
397
|
if (customTokenizer) {
|
|
375
|
-
for (const entry of customTokenizer
|
|
398
|
+
for (const entry of customTokensOf(customTokenizer, text)) {
|
|
399
|
+
visit(entry.token, entry.position, void 0, entry.token);
|
|
400
|
+
}
|
|
376
401
|
return;
|
|
377
402
|
}
|
|
403
|
+
if (nativeAnalysis()?.visitTokens(text, language, options, visit) === true) return;
|
|
378
404
|
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
379
405
|
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
380
406
|
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
@@ -391,11 +417,17 @@ function visitTokens(text, language, options, visit) {
|
|
|
391
417
|
visit(processed, position, void 0, part);
|
|
392
418
|
continue;
|
|
393
419
|
}
|
|
394
|
-
const
|
|
395
|
-
visit(processed, position,
|
|
420
|
+
const shown = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
421
|
+
visit(processed, position, surfaceOf(shown, processed), part);
|
|
396
422
|
}
|
|
397
423
|
}
|
|
424
|
+
function surfaceOf(shown, term) {
|
|
425
|
+
if (shown === term) return void 0;
|
|
426
|
+
const surface = withUnpairedSurrogatesReplaced(shown);
|
|
427
|
+
return surface === term ? void 0 : surface;
|
|
428
|
+
}
|
|
398
429
|
function visitSurfaceSpellings(text, language, visit) {
|
|
430
|
+
if (nativeAnalysis()?.visitSpellings(text, language, visit) === true) return;
|
|
399
431
|
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
400
432
|
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
401
433
|
const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
|
|
@@ -403,12 +435,14 @@ function visitSurfaceSpellings(text, language, visit) {
|
|
|
403
435
|
for (const part of rawParts) {
|
|
404
436
|
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
405
437
|
if (candidate.length < minLength) continue;
|
|
406
|
-
visit(
|
|
438
|
+
visit(
|
|
439
|
+
removeDiacritics ? transformToken(candidate, language, false, true) : withUnpairedSurrogatesReplaced(candidate)
|
|
440
|
+
);
|
|
407
441
|
}
|
|
408
442
|
}
|
|
409
443
|
function tokenize(text, language, options) {
|
|
410
444
|
if (options?.customTokenizer) {
|
|
411
|
-
const customResult = options.customTokenizer
|
|
445
|
+
const customResult = customTokensOf(options.customTokenizer, text);
|
|
412
446
|
return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
|
|
413
447
|
}
|
|
414
448
|
const tokens = [];
|
|
@@ -492,4 +526,4 @@ function codePointOrder(strings) {
|
|
|
492
526
|
return order;
|
|
493
527
|
}
|
|
494
528
|
|
|
495
|
-
export { clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, getNormalizationCacheSize, resetNormalizationCache, resolveStopWords, tokenize, transformToken, visitSurfaceSpellings, visitTokens };
|
|
529
|
+
export { NATIVE_ANALYSIS_FIRST_TOKEN_RECORDS, NATIVE_ANALYSIS_HELD_TERMS, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, getNormalizationCacheSize, installNativeAnalysis, resetNormalizationCache, resolveStopWords, tokenize, transformToken, visitSurfaceSpellings, visitTokens };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-
|
|
1
|
+
import { littleEndianBytes, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, KEYWORD_SEGMENT_PAYLOAD_VERSION, uint64Bytes } from './chunk-QUKQNI3H.mjs';
|
|
2
2
|
import { SEGMENT_FILE_FRAMING_SLACK_BYTES, SEGMENT_TOKEN_TABLE_SHARE_OF_FILE } from './chunk-F753DA54.mjs';
|
|
3
3
|
import { segmentKey, MAX_SEGMENTS_PER_PARTITION } from './chunk-6WA3C3DV.mjs';
|
|
4
|
-
import { selectionKeeps, packChecksummedEnvelope, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
5
|
-
import { messagePackUpperBoundBytes } from './chunk-
|
|
4
|
+
import { selectionKeeps, packChecksummedEnvelope, segmentArraysOf, mergeSegmentContents } from './chunk-MIP4SOR6.mjs';
|
|
5
|
+
import { messagePackUpperBoundBytes } from './chunk-GXX3DQXP.mjs';
|
|
6
6
|
import { crc32, openChecksummedEnvelope, HEADER_SIZE } from './chunk-YH7QY657.mjs';
|
|
7
7
|
import { MAX_FILE_BYTES } from './chunk-6VBXNFLE.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
|
|
@@ -771,7 +771,7 @@ async function writePagedGroup(state, group) {
|
|
|
771
771
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
772
772
|
const id = state.nextSegmentId;
|
|
773
773
|
const key = segmentKey(request.indexName, request.partitionId, id);
|
|
774
|
-
const { writePagedSegment } = await import('./module-
|
|
774
|
+
const { writePagedSegment } = await import('./module-LBZPGOWH.mjs');
|
|
775
775
|
const written = await writePagedSegment({
|
|
776
776
|
directory,
|
|
777
777
|
key,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { compareCodePoints } from './chunk-
|
|
1
|
+
import { compareCodePoints } from './chunk-EJ33LCHV.mjs';
|
|
2
2
|
|
|
3
3
|
// src/core/ordering/fold-table.ts
|
|
4
4
|
var SINGLE_FOLDS = "1t,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2j,lj;b,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;y,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3d;1,1;2,1;2,1;2,-7g;2,5u;1,1;2,1;2,5q;1,1;2,5p;1,5p;1,1;3,27;1,5m;1,5n;1,1;2,5p;1,5r;2,5v;1,5t;1,1;4,5v;1,5x;2,5y;1,1;2,1;2,1;2,62;1,1;2,62;3,1;2,62;1,1;2,61;1,61;1,1;2,1;2,63;1,1;4,1;8,2;1,1;2,2;1,1;2,2;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,2;1,1;2,1;2,-2p;1,-1k;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-3m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;8,8bv;1,1;2,-4j;1,8bs;3,1;2,-5f;1,1x;1,1z;1,1;2,1;2,1;2,1;2,1;6v,38;17,1;2,1;4,1;9,38;7,12;2,11;1,11;1,11;2,1s;2,1r;1,1r;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;2,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;n,1;d,8;1,-u;1,-p;4,-f;1,-m;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1i;1,-1c;3,-1o;1,-1s;2,1;2,-7;1,1;3,-3m;1,-3m;1,-3m;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,28;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1d,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,f;1,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;28a,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;1,5ls;2,5ls;6,5ls;mj,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1oj,-4su;1,-4st;1,-4sk;1,-4si;1,-4si;1,-4sj;1,-4sc;1,-4ro;1,r7n;1,1;7,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;1,-2bk;3,-2bk;1,-2bk;1,-2bk;8x,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;7,-1m;5,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;a,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;b,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;c,-8;2,-8;2,-8;2,-8;9,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;1,-8;21,-8;1,-8;1,-22;1,-22;3,-5j9;a,-2e;1,-2e;1,-2e;1,-2e;d,-8;1,-8;1,-2s;1,-2s;d,-8;1,-8;1,-34;1,-34;1,-7;c,-3k;1,-3k;1,-3i;1,-3i;8b,-5st;4,-6gv;1,-6di;7,s;1a,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;1,g;k,1;mr,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1,q;1f5,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1,1c;1d,1;2,-8af;1,-2xy;1,-89z;3,1;2,1;2,1;2,-8bg;1,-8al;1,-8bj;1,-8bi;2,1;3,1;9,-8cf;1,-8cf;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;9,1;2,1;5,1;nym,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;k,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;3s,1;2,1;2,1;2,1;2,1;2,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;b,1;2,1;2,-r9g;1,1;2,1;2,1;2,1;2,1;5,1;2,-wmg;3,1;2,1;4,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wn8;1,-wnj;1,-wnf;1,-wn5;1,-wn8;2,-wlu;1,-wmi;1,-wlx;1,ps;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-1c;1,-wn7;1,-raw;1,1;2,1;2,-wo7;1,1;2,1;2,1;2,1;2,1;2,1;2,1;2,1;2,-wu9;p,1;or,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;1,-tzk;ggy,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;xy,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;3t,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;1,14;4d,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1,13;1,13;1,13;1,13;1,13;2,13;1,13;1d7,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;1,1s;4e,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;27v,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;gw1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1,w;1t,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;1,r;o5k,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;1,y;";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, graphPartKey, vectorFileKey } from './chunk-6WA3C3DV.mjs';
|
|
2
1
|
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-DVDX4SDR.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, graphPartKey, vectorFileKey } from './chunk-6WA3C3DV.mjs';
|
|
3
3
|
import { openChecksummedEnvelope, HEADER_SIZE } from './chunk-YH7QY657.mjs';
|
|
4
|
-
import { requireValidFieldTypes } from './chunk-
|
|
4
|
+
import { requireValidFieldTypes } from './chunk-DNXOMPAQ.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-6VBXNFLE.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MAX_DOCUMENT_ENCODING_DEPTH, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES } from './chunk-6VBXNFLE.mjs';
|
|
2
2
|
import { isPlainObject, isWordIndexedBase, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, parseFieldType, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS } from './chunk-C5G6TJ7A.mjs';
|
|
3
|
-
import { compareCodePoints } from './chunk-
|
|
3
|
+
import { compareCodePoints } from './chunk-EJ33LCHV.mjs';
|
|
4
4
|
import { NarsilError, ErrorCodes } from './chunk-DZ7H7D2C.mjs';
|
|
5
5
|
import { Encoder } from '@msgpack/msgpack';
|
|
6
6
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// src/geo/constants.ts
|
|
2
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
3
|
+
var MAX_LATITUDE_DEGREES = 90;
|
|
4
|
+
var MAX_LONGITUDE_DEGREES = 180;
|
|
5
|
+
var MIN_POLYGON_POINTS = 3;
|
|
6
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
7
|
+
var FULL_TURN_DEGREES = 360;
|
|
8
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
9
|
+
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
10
|
+
var MAX_COVERING_CELLS = 16;
|
|
11
|
+
|
|
12
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS };
|
|
@@ -16,17 +16,40 @@ function exceedsCodePoints(value, maximum) {
|
|
|
16
16
|
return false;
|
|
17
17
|
}
|
|
18
18
|
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
var REPLACEMENT_CHARACTER = "\uFFFD";
|
|
20
|
+
function firstUnpairedSurrogate(value) {
|
|
21
21
|
for (let i = 0; i < value.length; i++) {
|
|
22
22
|
const unit = value.charCodeAt(i);
|
|
23
23
|
if (unit < 55296 || unit >= 57344) continue;
|
|
24
|
-
if (unit >= 56320) return
|
|
24
|
+
if (unit >= 56320) return i;
|
|
25
25
|
const low = value.charCodeAt(i + 1);
|
|
26
|
-
if (!(low >= 56320 && low < 57344)) return
|
|
26
|
+
if (!(low >= 56320 && low < 57344)) return i;
|
|
27
27
|
i++;
|
|
28
28
|
}
|
|
29
|
-
return
|
|
29
|
+
return -1;
|
|
30
|
+
}
|
|
31
|
+
function holdsUnpairedSurrogate(value) {
|
|
32
|
+
return SURROGATE_UNIT.test(value) && firstUnpairedSurrogate(value) >= 0;
|
|
33
|
+
}
|
|
34
|
+
function withUnpairedSurrogatesReplaced(value) {
|
|
35
|
+
const first = firstUnpairedSurrogate(value);
|
|
36
|
+
if (first < 0) return value;
|
|
37
|
+
let repaired = value.slice(0, first);
|
|
38
|
+
for (let i = first; i < value.length; i++) {
|
|
39
|
+
const unit = value.charCodeAt(i);
|
|
40
|
+
if (unit < 55296 || unit >= 57344) {
|
|
41
|
+
repaired += value[i];
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
const low = value.charCodeAt(i + 1);
|
|
45
|
+
if (unit < 56320 && low >= 56320 && low < 57344) {
|
|
46
|
+
repaired += value.slice(i, i + 2);
|
|
47
|
+
i++;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
repaired += REPLACEMENT_CHARACTER;
|
|
51
|
+
}
|
|
52
|
+
return repaired;
|
|
30
53
|
}
|
|
31
54
|
function codePointCount(value) {
|
|
32
55
|
let counted = 0;
|
|
@@ -53,4 +76,4 @@ function compareCodePoints(a, b) {
|
|
|
53
76
|
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
54
77
|
}
|
|
55
78
|
|
|
56
|
-
export { codePointCount, compareCodePoints, exceedsCodePoints, holdsUnpairedSurrogate };
|
|
79
|
+
export { codePointCount, compareCodePoints, exceedsCodePoints, holdsUnpairedSurrogate, withUnpairedSurrogatesReplaced };
|