@delali/narsil 0.3.1-next.162 → 0.3.1-next.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +24 -8
  2. package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
  3. package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
  4. package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
  5. package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
  6. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  7. package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
  8. package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
  9. package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
  10. package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
  11. package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
  12. package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
  13. package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
  14. package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
  15. package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
  16. package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
  17. package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
  18. package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
  19. package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
  20. package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
  21. package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
  22. package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
  23. package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
  24. package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
  25. package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
  26. package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
  27. package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
  28. package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
  29. package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
  30. package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
  31. package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
  32. package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
  33. package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
  34. package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
  35. package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
  36. package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
  37. package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
  38. package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
  39. package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
  40. package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
  41. package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
  42. package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
  43. package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
  44. package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
  45. package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
  46. package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
  47. package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
  48. package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
  49. package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
  50. package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
  51. package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
  52. package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
  53. package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
  54. package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
  55. package/dist/client.d.ts +8 -8
  56. package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
  57. package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
  58. package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
  59. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  60. package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
  61. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  62. package/dist/distribution/coordinator/etcd.mjs +1 -1
  63. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  64. package/dist/distribution/transport/grpc.mjs +1 -1
  65. package/dist/distribution/transport/tcp.mjs +1 -1
  66. package/dist/distribution.d.ts +7 -7
  67. package/dist/distribution.mjs +39 -39
  68. package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
  69. package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
  70. package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
  71. package/dist/index.browser.mjs +3 -3
  72. package/dist/index.d.ts +6 -6
  73. package/dist/index.mjs +7 -7
  74. package/dist/index.react-native.mjs +6 -6
  75. package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
  76. package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
  77. package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
  78. package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
  79. package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
  80. package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
  81. package/dist/partial-words.browser.mjs +3 -3
  82. package/dist/partial-words.mjs +7 -7
  83. package/dist/pattern.browser.mjs +2 -2
  84. package/dist/pattern.mjs +2 -2
  85. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  86. package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
  87. package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
  88. package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
  89. package/dist/query-suggestions/merge-worker.mjs +10 -10
  90. package/dist/query-suggestions.browser.mjs +3 -3
  91. package/dist/query-suggestions.d.ts +2 -2
  92. package/dist/query-suggestions.mjs +8 -8
  93. package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
  94. package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
  95. package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
  96. package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
  97. package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
  98. package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
  99. package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
  100. package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
  101. package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
  103. package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
  105. package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
  106. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  107. package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
  108. package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
  109. package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
  110. package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
  111. package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
  112. package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
  113. package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
  114. package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
  115. package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
  116. package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
  117. package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
  118. package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
  119. package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
  120. package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
  121. package/dist/react-native/partial-words.mjs +6 -6
  122. package/dist/react-native/pattern.mjs +3 -3
  123. package/dist/react-native/query-suggestions.mjs +6 -6
  124. package/dist/react-native/sync.mjs +6 -6
  125. package/dist/react-native/synonyms.mjs +6 -6
  126. package/dist/react.d.ts +9 -9
  127. package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
  128. package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
  129. package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
  130. package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
  131. package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
  132. package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
  133. package/dist/server.d.ts +10 -10
  134. package/dist/server.mjs +26 -26
  135. package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
  136. package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
  137. package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
  138. package/dist/sync/filesystem.d.ts +3 -3
  139. package/dist/sync/filesystem.mjs +15 -15
  140. package/dist/sync/indexeddb.browser.mjs +1 -1
  141. package/dist/sync/indexeddb.d.ts +3 -3
  142. package/dist/sync/indexeddb.mjs +2 -2
  143. package/dist/sync.browser.mjs +3 -3
  144. package/dist/sync.d.ts +4 -4
  145. package/dist/sync.mjs +7 -7
  146. package/dist/synonyms.browser.mjs +3 -3
  147. package/dist/synonyms.mjs +7 -7
  148. package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
  149. package/dist/thread-OWXVP2ZA.mjs +30 -0
  150. package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
  151. package/dist/vector/search-worker.d.ts +1 -1
  152. package/dist/vector/search-worker.mjs +5 -5
  153. package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
  154. package/dist/workers/entry.mjs +13 -13
  155. package/package.json +9 -8
  156. package/dist/thread-XWDK6TAR.mjs +0 -30
  157. /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
package/README.md CHANGED
@@ -12,7 +12,7 @@ The search engine that scales with your data.
12
12
 
13
13
  Narsil is an open source search engine with full-text, vector, hybrid, and geosearch on one typed index. You can embed Narsil in your application process, where the engine searches without a network hop, or deploy it as a standalone search server with a REST API, a write-ahead log, and bulk NDJSON ingest. Both forms contain the same engine, which stores indexes in one cross-language binary format (`.nrsl`), so you can load an index from one form into the other.
14
14
 
15
- The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format, so a second implementation in Go or Rust is the headline item on the [roadmap](https://github.com/assetcorp/narsil/blob/main/ROADMAP.md).
15
+ The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format, and [a Rust crate and a Go module](#other-languages) hold the same engine for programs in those languages.
16
16
 
17
17
  <!-- BENCH:headline START -->
18
18
  On BEIR SciFact, Narsil's BM25 ranking scores 0.681 nDCG@10, which is within 0.002 of the Anserini reference of 0.679. Elasticsearch scores 0.679 and OpenSearch scores 0.679 on the same queries. On SciFact, Narsil answers 4,200 keyword queries a second at its peak, and OpenSearch follows with 3,027. On DBpedia entities 1M with every engine held at 0.99 recall, Narsil answers 670 vector queries a second at its peak, while OpenSearch leads with 869. With each engine's recommended production settings on the same set, Narsil answers 1,154 vector queries a second at its peak, and OpenSearch follows with 1,111. Narsil has the highest peak throughput in 16 of the 20 keyword, vector, and hybrid comparisons in the newest recorded run: 8 of 12 with every engine on equal settings, and all 8 with each engine's recommended production settings. The harness recorded these figures on 2026-09-28 in run `20260928T002255Z`. GCP c3-standard-8 hardware in us-central1-a, us-east1-b, and us-west1-a hosted the engines. You'll find every dataset, every engine's settings, and the full method in [`BENCHMARKS.md`](https://github.com/assetcorp/narsil/blob/main/BENCHMARKS.md).
@@ -33,6 +33,7 @@ Try it in your browser at [narsil.sondelali.com/demo](https://narsil.sondelali.c
33
33
  - [Distribution](#distribution)
34
34
  - [Search quality](#search-quality)
35
35
  - [Runtime support](#runtime-support)
36
+ - [Other languages](#other-languages)
36
37
  - [License](#license)
37
38
 
38
39
  ## Install
@@ -43,7 +44,7 @@ pnpm add @delali/narsil
43
44
 
44
45
  Narsil works in Node.js 22 or newer, and in Bun, Deno, and browsers. The [Runtime support](#runtime-support) table covers each runtime.
45
46
 
46
- On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core as an optional dependency, so the engine searches vector graphs through it. If you omit optional dependencies, Narsil searches those graphs through WebAssembly, with the same results, as it always does in a browser.
47
+ On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core in C as an optional dependency, so the engine analyses text and answers keyword and vector searches through it. If you omit optional dependencies, Narsil does that work through WebAssembly and TypeScript, with the same results, as it always does in a browser.
47
48
 
48
49
  ## Add Narsil with a coding agent
49
50
 
@@ -102,15 +103,15 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
102
103
 
103
104
  **Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
104
105
 
105
- **Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter of your own. A query can [rerank](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) its top hits with a cross-encoder, through a Cohere-compatible rerank service, a local Transformers.js model, or an adapter of your own.
106
+ **Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter that you write. A query can [rerank](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) its top hits with a cross-encoder, through a Cohere-compatible rerank service, a local Transformers.js model, or an adapter that you write.
106
107
 
107
108
  **Geosearch.** [Geo filters](https://github.com/assetcorp/narsil/blob/main/docs/geosearch.md) match documents by radius, using Haversine or Vincenty distance, or by polygon containment. You can combine a geo filter with every other query feature.
108
109
 
109
- **Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend of your own. With durability on, the engine keeps a write-ahead log with periodic checkpoints and recovers on its own after a crash. You can stream a whole index into a portable snapshot archive, or export its documents as JSON Lines to move the index across releases. With [device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md), a phone, a laptop, or a browser tab keeps a copy of one index from a server, which it searches and writes while it is offline.
110
+ **Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend that you write. With durability on, the engine keeps a write-ahead log with periodic checkpoints and recovers automatically after a crash. You can stream a whole index into a portable snapshot archive, or export its documents as JSON Lines to move the index across releases. With [device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md), a phone, a laptop, or a browser tab keeps a copy of one index from a server, which it searches and writes while it is offline.
110
111
 
111
112
  **Scale.** The engine routes each document to a [partition](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) by a deterministic hash. When you call `rebalance()`, the engine reshapes the partitions online and buffers incoming writes in a write-ahead queue. Once an index holds 1,000 documents, the engine serves keyword queries from worker copies on half its worker threads, while the HTTP server receives requests on the threads that hold those copies.
112
113
 
113
- **Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover 107 languages as separate entry points, including 20 African languages.
114
+ **Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover the languages that the guide lists, including Swahili, Yoruba, Hausa, and other African languages, and each module is a separate entry point.
114
115
 
115
116
  ## Documentation
116
117
 
@@ -127,10 +128,13 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
127
128
  | [Reranking](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) | Reordering the top hits with a cross-encoder, the bundled rerank adapters, and custom ones |
128
129
  | [Persistence and durability](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) | Storage backends, the write-ahead log, checkpoints, recovery, snapshots, document exports, and indexes from an earlier release |
129
130
  | [Partitions and workers](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) | Partition routing, online rebalancing, worker copies, and multi-instance invalidation |
130
- | [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) | The 107 language modules, analysis revisions and rebuilds, and named tokenizers and stop words |
131
+ | [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) | Every language module, analysis revisions and rebuilds, and named tokenizers and stop words |
131
132
  | [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) | Wrapping an engine in a REST API, every route it serves, and long-running tasks |
132
133
  | [Cluster mode](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) | Multi-node indexes: nodes and roles, replication, routed writes, distributed searches and reads, and the calls that fail in a cluster |
133
134
  | [Client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) | Reaching a server from a browser or Node, following a task, and the codes it raises |
135
+ | [Python client](https://github.com/assetcorp/narsil/blob/main/docs/python-client.md) | Reaching a server or a cluster from Python, in blocking and async forms, and the failover rules |
136
+ | [Go client](https://github.com/assetcorp/narsil/blob/main/docs/go-client.md) | Reaching a server or a cluster from Go, streaming snapshots, and the failover rules |
137
+ | [React Native](https://github.com/assetcorp/narsil/blob/main/docs/react-native.md) | The engine inside an iOS or Android app, where the index lives, snapshots, memory, and errors |
134
138
  | [Device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md) | A copy of one index on a device that searches and writes offline, the sync routes, conflicts, and the snapshot |
135
139
  | [Popular query suggestions](https://github.com/assetcorp/narsil/blob/main/docs/query-suggestions.md) | Whole searches that people ran and clicked, counted per person, built into a suggestions index, and the privacy trade-offs |
136
140
  | [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over a client or an engine in the page, one shared request per key, and loading a corpus from a component |
@@ -149,11 +153,11 @@ The [specification](https://github.com/assetcorp/narsil/tree/main/packages/spec)
149
153
 
150
154
  ## Distribution
151
155
 
152
- `@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition, and it can change the replication factor and the partition count of an index. A node with a durability directory keeps its id there across restarts, and the etcd coordinator connects over TLS with a user and a password. A node that the HTTP server fronts reports the health of the cluster and its own Prometheus metrics. The [cluster dashboard example](https://github.com/assetcorp/narsil/tree/main/packages/ts/examples/cluster-dashboard) starts three nodes against etcd and shows the state of each partition while you cut the network links. The [cluster guide](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) covers the API, including the packages that the etcd coordinator and the gRPC transport each need. For the contract that every implementation must uphold, read [`packages/spec/distribution`](https://github.com/assetcorp/narsil/tree/main/packages/spec/distribution).
156
+ `@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition, and it can change the replication factor and the partition count of an index. A node with a durability directory keeps its id there across restarts, and the etcd coordinator connects over TLS with a user and a password. A node that the HTTP server fronts reports the health of the cluster and the node's Prometheus metrics. The [cluster dashboard example](https://github.com/assetcorp/narsil/tree/main/packages/ts/examples/cluster-dashboard) starts three nodes against etcd and shows the state of each partition while you cut the network links. The [cluster guide](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) covers the API, including the packages that the etcd coordinator and the gRPC transport each need. For the contract that every implementation must uphold, read [`packages/spec/distribution`](https://github.com/assetcorp/narsil/tree/main/packages/spec/distribution).
153
157
 
154
158
  ## Search quality
155
159
 
156
- We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English in its own way. Narsil has the top nDCG@10 of the three.
160
+ We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English differently. Narsil has the top nDCG@10 of the three.
157
161
 
158
162
  **What these metrics mean:**
159
163
 
@@ -179,6 +183,18 @@ Narsil also works as a search server. On the BEIR datasets, we compare the serve
179
183
 
180
184
  The [browser example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/browser/README.md) shows an embedded engine with IndexedDB persistence, while the [server app example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/server-app/README.md) shows the same interface over the client SDK and the React hooks.
181
185
 
186
+ ## Other languages
187
+
188
+ | Package | What it does |
189
+ | --- | --- |
190
+ | [`delali-narsil`](https://github.com/assetcorp/narsil/tree/main/packages/rust) | The Rust crate creates, changes, and searches an index inside a Rust program. |
191
+ | [`github.com/assetcorp/narsil/packages/go/engine`](https://github.com/assetcorp/narsil/tree/main/packages/go/engine) | The Go engine creates, changes, and searches an index inside a Go program. |
192
+ | [`github.com/assetcorp/narsil/packages/go`](https://github.com/assetcorp/narsil/tree/main/packages/go) | The Go client calls a Narsil server, or every node of a cluster, from Go. |
193
+ | [`narsil-client`](https://github.com/assetcorp/narsil/tree/main/packages/python) | The Python client calls a Narsil server, or every node of a cluster, from Python. |
194
+ | [`@delali/narsil-react-native`](https://github.com/assetcorp/narsil/tree/main/packages/react-native) | The driver puts the engine inside an iOS or Android app. |
195
+
196
+ The Rust crate and the Go engine save every file in the same layouts as this package, so this package can open an index that either one saves, and either one can open an index that this package saves. Only this package holds the HTTP server and cluster mode.
197
+
182
198
  ## License
183
199
 
184
200
  Apache-2.0
@@ -206,6 +206,11 @@ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
206
206
 
207
207
  // src/schema/validator/shared.ts
208
208
  var VECTOR_PATTERN = /^vector\[(\d+)]$/;
209
+ var VECTOR_TYPE_PATTERN = /^vector\[(.*)]$/;
210
+ function vectorDimensionOf(fieldType) {
211
+ const match = VECTOR_PATTERN.exec(fieldType);
212
+ return match === null ? null : Number.parseInt(match[1], 10);
213
+ }
209
214
  function isGeopointOrVectorType(fieldType) {
210
215
  return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
211
216
  }
@@ -247,15 +252,15 @@ function invalidType(path, type, message) {
247
252
  }
248
253
  function parseBase(baseName, path, type) {
249
254
  if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
250
- const vectorMatch = VECTOR_PATTERN.exec(baseName);
255
+ const vectorMatch = VECTOR_TYPE_PATTERN.exec(baseName);
251
256
  if (vectorMatch === null) {
252
257
  invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
253
258
  }
254
- const dimension = Number.parseInt(vectorMatch[1], 10);
255
- if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
259
+ const dimension = vectorDimensionOf(baseName) ?? Number.NaN;
260
+ if (!(dimension > 0 && dimension <= MAX_STORED_VECTOR_DIMENSION)) {
256
261
  throw new NarsilError(
257
262
  ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
258
- `Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
263
+ `Vector field "${path}" has ${vectorMatch[1]} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
259
264
  { field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
260
265
  );
261
266
  }
@@ -486,7 +491,7 @@ function validateVector(path, value, expectedDimension) {
486
491
  );
487
492
  }
488
493
  for (let i = 0; i < arr.length; i++) {
489
- if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
494
+ if (typeof arr[i] !== "number" || !Number.isFinite(Math.fround(arr[i]))) {
490
495
  throw new NarsilError(
491
496
  ErrorCodes.DOC_VALIDATION_FAILED,
492
497
  `Field "${path}" vector element at index ${i} is not a valid number`,
@@ -1122,7 +1127,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
1122
1127
  table.indexMap.set(fieldName, idx);
1123
1128
  return idx;
1124
1129
  }
1125
- var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
1126
1130
  function getNestedValue(obj, path) {
1127
1131
  if (obj === null || typeof obj !== "object") return void 0;
1128
1132
  if (!path.includes(".")) return Object.hasOwn(obj, path) ? obj[path] : void 0;
@@ -1158,8 +1162,7 @@ function textFieldsChanged(oldDoc, newDoc, fields) {
1158
1162
  return false;
1159
1163
  }
1160
1164
  function parseVectorDimension(fieldType) {
1161
- const match = VECTOR_PATTERN2.exec(fieldType);
1162
- return match ? Number.parseInt(match[1], 10) : null;
1165
+ return vectorDimensionOf(fieldType);
1163
1166
  }
1164
1167
  function tokenizeOptions(options) {
1165
1168
  return {
@@ -1167,7 +1170,8 @@ function tokenizeOptions(options) {
1167
1170
  removeStopWords: true,
1168
1171
  collectSurfaces: options?.collectSurfaces === true,
1169
1172
  stopWordOverride: options?.stopWordOverride,
1170
- customTokenizer: options?.customTokenizer
1173
+ customTokenizer: options?.customTokenizer,
1174
+ textSource: "document"
1171
1175
  };
1172
1176
  }
1173
1177
  function getFlatSchema(state, schema) {
@@ -1225,4 +1229,4 @@ function requireLoadedMessagePack() {
1225
1229
  return loaded;
1226
1230
  }
1227
1231
 
1228
- export { ANTIMERIDIAN_SPAN_DEGREES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, ClientErrorCodes, DEFAULT_EF_SEARCH, DEG_TO_RAD, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, EARLIER_RELEASE_MIN_M, EARTH_RADIUS_METERS, ErrorCodes, FARTHEST_REJECTED_LINKS_KEPT, FULL_TURN_DEGREES, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOCUMENT_NESTING_DEPTH, MAX_FILE_BYTES, MAX_LATITUDE_DEGREES, MAX_LAYER_CAP, MAX_LONGITUDE_DEGREES, MAX_M, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_M, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, NATIVE_CRC32_MIN_BYTES, NEAREST_LINK_FALLBACK_HOSTS, NarsilError, PLACEMENT_ROOM_REQUESTS, PROTOTYPE_POLLUTION_KEYS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, ServerErrorCodes, VALID_HNSW_METRICS, VECTOR_PATTERN, codePointCount, compareCodePoints, createNarsilError, defaultSortMode, describeError, exceedsCodePoints, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, holdsUnpairedSurrogate, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isNarsilError, isPlainObject, isQuantizationMode, isSingleStringFieldType, isSortMode, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, sortModeOf, textFieldsChanged, toComparableSortValue, tokenizeOptions, truncateSortString, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, withSortableOption, withUnpairedSurrogatesReplaced };
1232
+ export { ANTIMERIDIAN_SPAN_DEGREES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, ClientErrorCodes, DEFAULT_EF_SEARCH, DEG_TO_RAD, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, EARLIER_RELEASE_MIN_M, EARTH_RADIUS_METERS, ErrorCodes, FARTHEST_REJECTED_LINKS_KEPT, FULL_TURN_DEGREES, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOCUMENT_NESTING_DEPTH, MAX_FILE_BYTES, MAX_LATITUDE_DEGREES, MAX_LAYER_CAP, MAX_LONGITUDE_DEGREES, MAX_M, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_M, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, NATIVE_CRC32_MIN_BYTES, NEAREST_LINK_FALLBACK_HOSTS, NarsilError, PLACEMENT_ROOM_REQUESTS, PROTOTYPE_POLLUTION_KEYS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, ServerErrorCodes, VALID_HNSW_METRICS, VECTOR_PATTERN, codePointCount, compareCodePoints, createNarsilError, defaultSortMode, describeError, exceedsCodePoints, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, holdsUnpairedSurrogate, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isNarsilError, isPlainObject, isQuantizationMode, isSingleStringFieldType, isSortMode, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, sortModeOf, textFieldsChanged, toComparableSortValue, tokenizeOptions, truncateSortString, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorDimensionOf, withSortableOption, withUnpairedSurrogatesReplaced };
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes, requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-MZHQ65VI.mjs';
1
+ import { NarsilError, ErrorCodes, requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-2Q45HI5H.mjs';
2
2
 
3
3
  // src/core/stored-values/build.ts
4
4
  var ORDINAL_LIMIT = 4294967295;