@delali/narsil 0.3.1-next.163 → 0.3.1-next.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +20 -19
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ The search engine that scales with your data.
|
|
|
12
12
|
|
|
13
13
|
Narsil is an open source search engine with full-text, vector, hybrid, and geosearch on one typed index. You can embed Narsil in your application process, where the engine searches without a network hop, or deploy it as a standalone search server with a REST API, a write-ahead log, and bulk NDJSON ingest. Both forms contain the same engine, which stores indexes in one cross-language binary format (`.nrsl`), so you can load an index from one form into the other.
|
|
14
14
|
|
|
15
|
-
The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format,
|
|
15
|
+
The engine partitions large indexes across workers and merges the results of the partitions into a single ranked list. This TypeScript package is the reference implementation of the `.nrsl` format, and [a Rust crate and a Go module](#other-languages) hold the same engine for programs in those languages.
|
|
16
16
|
|
|
17
17
|
<!-- BENCH:headline START -->
|
|
18
18
|
On BEIR SciFact, Narsil's BM25 ranking scores 0.681 nDCG@10, which is within 0.002 of the Anserini reference of 0.679. Elasticsearch scores 0.679 and OpenSearch scores 0.679 on the same queries. On SciFact, Narsil answers 4,200 keyword queries a second at its peak, and OpenSearch follows with 3,027. On DBpedia entities 1M with every engine held at 0.99 recall, Narsil answers 670 vector queries a second at its peak, while OpenSearch leads with 869. With each engine's recommended production settings on the same set, Narsil answers 1,154 vector queries a second at its peak, and OpenSearch follows with 1,111. Narsil has the highest peak throughput in 16 of the 20 keyword, vector, and hybrid comparisons in the newest recorded run: 8 of 12 with every engine on equal settings, and all 8 with each engine's recommended production settings. The harness recorded these figures on 2026-09-28 in run `20260928T002255Z`. GCP c3-standard-8 hardware in us-central1-a, us-east1-b, and us-west1-a hosted the engines. You'll find every dataset, every engine's settings, and the full method in [`BENCHMARKS.md`](https://github.com/assetcorp/narsil/blob/main/BENCHMARKS.md).
|
|
@@ -33,6 +33,7 @@ Try it in your browser at [narsil.sondelali.com/demo](https://narsil.sondelali.c
|
|
|
33
33
|
- [Distribution](#distribution)
|
|
34
34
|
- [Search quality](#search-quality)
|
|
35
35
|
- [Runtime support](#runtime-support)
|
|
36
|
+
- [Other languages](#other-languages)
|
|
36
37
|
- [License](#license)
|
|
37
38
|
|
|
38
39
|
## Install
|
|
@@ -43,7 +44,7 @@ pnpm add @delali/narsil
|
|
|
43
44
|
|
|
44
45
|
Narsil works in Node.js 22 or newer, and in Bun, Deno, and browsers. The [Runtime support](#runtime-support) table covers each runtime.
|
|
45
46
|
|
|
46
|
-
On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core as an optional dependency, so the engine
|
|
47
|
+
On an arm64 or x64 machine under macOS, Linux, or Windows, the package manager also installs Narsil's native search core in C as an optional dependency, so the engine analyses text and answers keyword and vector searches through it. If you omit optional dependencies, Narsil does that work through WebAssembly and TypeScript, with the same results, as it always does in a browser.
|
|
47
48
|
|
|
48
49
|
## Add Narsil with a coding agent
|
|
49
50
|
|
|
@@ -102,15 +103,15 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
102
103
|
|
|
103
104
|
**Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
|
|
104
105
|
|
|
105
|
-
**Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter
|
|
106
|
+
**Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter that you write. A query can [rerank](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) its top hits with a cross-encoder, through a Cohere-compatible rerank service, a local Transformers.js model, or an adapter that you write.
|
|
106
107
|
|
|
107
108
|
**Geosearch.** [Geo filters](https://github.com/assetcorp/narsil/blob/main/docs/geosearch.md) match documents by radius, using Haversine or Vincenty distance, or by polygon containment. You can combine a geo filter with every other query feature.
|
|
108
109
|
|
|
109
|
-
**Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend
|
|
110
|
+
**Storage.** [Persistence adapters](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) store indexes on the filesystem, in IndexedDB, in memory, or in a backend that you write. With durability on, the engine keeps a write-ahead log with periodic checkpoints and recovers automatically after a crash. You can stream a whole index into a portable snapshot archive, or export its documents as JSON Lines to move the index across releases. With [device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md), a phone, a laptop, or a browser tab keeps a copy of one index from a server, which it searches and writes while it is offline.
|
|
110
111
|
|
|
111
112
|
**Scale.** The engine routes each document to a [partition](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) by a deterministic hash. When you call `rebalance()`, the engine reshapes the partitions online and buffers incoming writes in a write-ahead queue. Once an index holds 1,000 documents, the engine serves keyword queries from worker copies on half its worker threads, while the HTTP server receives requests on the threads that hold those copies.
|
|
112
113
|
|
|
113
|
-
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover
|
|
114
|
+
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover the languages that the guide lists, including Swahili, Yoruba, Hausa, and other African languages, and each module is a separate entry point.
|
|
114
115
|
|
|
115
116
|
## Documentation
|
|
116
117
|
|
|
@@ -127,10 +128,13 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
127
128
|
| [Reranking](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) | Reordering the top hits with a cross-encoder, the bundled rerank adapters, and custom ones |
|
|
128
129
|
| [Persistence and durability](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) | Storage backends, the write-ahead log, checkpoints, recovery, snapshots, document exports, and indexes from an earlier release |
|
|
129
130
|
| [Partitions and workers](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) | Partition routing, online rebalancing, worker copies, and multi-instance invalidation |
|
|
130
|
-
| [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) |
|
|
131
|
+
| [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) | Every language module, analysis revisions and rebuilds, and named tokenizers and stop words |
|
|
131
132
|
| [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) | Wrapping an engine in a REST API, every route it serves, and long-running tasks |
|
|
132
133
|
| [Cluster mode](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) | Multi-node indexes: nodes and roles, replication, routed writes, distributed searches and reads, and the calls that fail in a cluster |
|
|
133
134
|
| [Client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) | Reaching a server from a browser or Node, following a task, and the codes it raises |
|
|
135
|
+
| [Python client](https://github.com/assetcorp/narsil/blob/main/docs/python-client.md) | Reaching a server or a cluster from Python, in blocking and async forms, and the failover rules |
|
|
136
|
+
| [Go client](https://github.com/assetcorp/narsil/blob/main/docs/go-client.md) | Reaching a server or a cluster from Go, streaming snapshots, and the failover rules |
|
|
137
|
+
| [React Native](https://github.com/assetcorp/narsil/blob/main/docs/react-native.md) | The engine inside an iOS or Android app, where the index lives, snapshots, memory, and errors |
|
|
134
138
|
| [Device sync](https://github.com/assetcorp/narsil/blob/main/docs/device-sync.md) | A copy of one index on a device that searches and writes offline, the sync routes, conflicts, and the snapshot |
|
|
135
139
|
| [Popular query suggestions](https://github.com/assetcorp/narsil/blob/main/docs/query-suggestions.md) | Whole searches that people ran and clicked, counted per person, built into a suggestions index, and the privacy trade-offs |
|
|
136
140
|
| [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over a client or an engine in the page, one shared request per key, and loading a corpus from a component |
|
|
@@ -149,11 +153,11 @@ The [specification](https://github.com/assetcorp/narsil/tree/main/packages/spec)
|
|
|
149
153
|
|
|
150
154
|
## Distribution
|
|
151
155
|
|
|
152
|
-
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition, and it can change the replication factor and the partition count of an index. A node with a durability directory keeps its id there across restarts, and the etcd coordinator connects over TLS with a user and a password. A node that the HTTP server fronts reports the health of the cluster and
|
|
156
|
+
`@delali/narsil/distribution` holds Narsil's multi-node cluster mode: nodes and roles, replication, coordinator adapters for etcd and in-process testing, TCP and gRPC transports with mutual TLS, and distributed query routing. A cluster node can create, drop, clear, write, update, search, list, count, and suggest across every partition, and it can change the replication factor and the partition count of an index. A node with a durability directory keeps its id there across restarts, and the etcd coordinator connects over TLS with a user and a password. A node that the HTTP server fronts reports the health of the cluster and the node's Prometheus metrics. The [cluster dashboard example](https://github.com/assetcorp/narsil/tree/main/packages/ts/examples/cluster-dashboard) starts three nodes against etcd and shows the state of each partition while you cut the network links. The [cluster guide](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) covers the API, including the packages that the etcd coordinator and the gRPC transport each need. For the contract that every implementation must uphold, read [`packages/spec/distribution`](https://github.com/assetcorp/narsil/tree/main/packages/spec/distribution).
|
|
153
157
|
|
|
154
158
|
## Search quality
|
|
155
159
|
|
|
156
|
-
We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English
|
|
160
|
+
We measure ranking quality on the [BEIR](https://github.com/beir-cellar/beir) SciFact corpus, which holds 5,183 documents and 300 queries with human relevance judgements. We compare Narsil with Orama and MiniSearch in one process. All three use the same 33 Lucene English stop words and default BM25 parameters, although each one stems English differently. Narsil has the top nDCG@10 of the three.
|
|
157
161
|
|
|
158
162
|
**What these metrics mean:**
|
|
159
163
|
|
|
@@ -179,6 +183,18 @@ Narsil also works as a search server. On the BEIR datasets, we compare the serve
|
|
|
179
183
|
|
|
180
184
|
The [browser example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/browser/README.md) shows an embedded engine with IndexedDB persistence, while the [server app example](https://github.com/assetcorp/narsil/blob/main/packages/ts/examples/server-app/README.md) shows the same interface over the client SDK and the React hooks.
|
|
181
185
|
|
|
186
|
+
## Other languages
|
|
187
|
+
|
|
188
|
+
| Package | What it does |
|
|
189
|
+
| --- | --- |
|
|
190
|
+
| [`delali-narsil`](https://github.com/assetcorp/narsil/tree/main/packages/rust) | The Rust crate creates, changes, and searches an index inside a Rust program. |
|
|
191
|
+
| [`github.com/assetcorp/narsil/packages/go/engine`](https://github.com/assetcorp/narsil/tree/main/packages/go/engine) | The Go engine creates, changes, and searches an index inside a Go program. |
|
|
192
|
+
| [`github.com/assetcorp/narsil/packages/go`](https://github.com/assetcorp/narsil/tree/main/packages/go) | The Go client calls a Narsil server, or every node of a cluster, from Go. |
|
|
193
|
+
| [`narsil-client`](https://github.com/assetcorp/narsil/tree/main/packages/python) | The Python client calls a Narsil server, or every node of a cluster, from Python. |
|
|
194
|
+
| [`@delali/narsil-react-native`](https://github.com/assetcorp/narsil/tree/main/packages/react-native) | The driver puts the engine inside an iOS or Android app. |
|
|
195
|
+
|
|
196
|
+
The Rust crate and the Go engine save every file in the same layouts as this package, so this package can open an index that either one saves, and either one can open an index that this package saves. Only this package holds the HTTP server and cluster mode.
|
|
197
|
+
|
|
182
198
|
## License
|
|
183
199
|
|
|
184
200
|
Apache-2.0
|
|
@@ -206,6 +206,11 @@ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
|
|
|
206
206
|
|
|
207
207
|
// src/schema/validator/shared.ts
|
|
208
208
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
209
|
+
var VECTOR_TYPE_PATTERN = /^vector\[(.*)]$/;
|
|
210
|
+
function vectorDimensionOf(fieldType) {
|
|
211
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
212
|
+
return match === null ? null : Number.parseInt(match[1], 10);
|
|
213
|
+
}
|
|
209
214
|
function isGeopointOrVectorType(fieldType) {
|
|
210
215
|
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
211
216
|
}
|
|
@@ -247,15 +252,15 @@ function invalidType(path, type, message) {
|
|
|
247
252
|
}
|
|
248
253
|
function parseBase(baseName, path, type) {
|
|
249
254
|
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
250
|
-
const vectorMatch =
|
|
255
|
+
const vectorMatch = VECTOR_TYPE_PATTERN.exec(baseName);
|
|
251
256
|
if (vectorMatch === null) {
|
|
252
257
|
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
253
258
|
}
|
|
254
|
-
const dimension = Number.
|
|
255
|
-
if (dimension
|
|
259
|
+
const dimension = vectorDimensionOf(baseName) ?? Number.NaN;
|
|
260
|
+
if (!(dimension > 0 && dimension <= MAX_STORED_VECTOR_DIMENSION)) {
|
|
256
261
|
throw new NarsilError(
|
|
257
262
|
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
258
|
-
`Vector field "${path}" has ${
|
|
263
|
+
`Vector field "${path}" has ${vectorMatch[1]} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
259
264
|
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
260
265
|
);
|
|
261
266
|
}
|
|
@@ -486,7 +491,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
486
491
|
);
|
|
487
492
|
}
|
|
488
493
|
for (let i = 0; i < arr.length; i++) {
|
|
489
|
-
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
494
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(Math.fround(arr[i]))) {
|
|
490
495
|
throw new NarsilError(
|
|
491
496
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
492
497
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -1122,7 +1127,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
|
|
|
1122
1127
|
table.indexMap.set(fieldName, idx);
|
|
1123
1128
|
return idx;
|
|
1124
1129
|
}
|
|
1125
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
1126
1130
|
function getNestedValue(obj, path) {
|
|
1127
1131
|
if (obj === null || typeof obj !== "object") return void 0;
|
|
1128
1132
|
if (!path.includes(".")) return Object.hasOwn(obj, path) ? obj[path] : void 0;
|
|
@@ -1158,8 +1162,7 @@ function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
|
1158
1162
|
return false;
|
|
1159
1163
|
}
|
|
1160
1164
|
function parseVectorDimension(fieldType) {
|
|
1161
|
-
|
|
1162
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
1165
|
+
return vectorDimensionOf(fieldType);
|
|
1163
1166
|
}
|
|
1164
1167
|
function tokenizeOptions(options) {
|
|
1165
1168
|
return {
|
|
@@ -1167,7 +1170,8 @@ function tokenizeOptions(options) {
|
|
|
1167
1170
|
removeStopWords: true,
|
|
1168
1171
|
collectSurfaces: options?.collectSurfaces === true,
|
|
1169
1172
|
stopWordOverride: options?.stopWordOverride,
|
|
1170
|
-
customTokenizer: options?.customTokenizer
|
|
1173
|
+
customTokenizer: options?.customTokenizer,
|
|
1174
|
+
textSource: "document"
|
|
1171
1175
|
};
|
|
1172
1176
|
}
|
|
1173
1177
|
function getFlatSchema(state, schema) {
|
|
@@ -1225,4 +1229,4 @@ function requireLoadedMessagePack() {
|
|
|
1225
1229
|
return loaded;
|
|
1226
1230
|
}
|
|
1227
1231
|
|
|
1228
|
-
export { ANTIMERIDIAN_SPAN_DEGREES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, ClientErrorCodes, DEFAULT_EF_SEARCH, DEG_TO_RAD, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, EARLIER_RELEASE_MIN_M, EARTH_RADIUS_METERS, ErrorCodes, FARTHEST_REJECTED_LINKS_KEPT, FULL_TURN_DEGREES, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOCUMENT_NESTING_DEPTH, MAX_FILE_BYTES, MAX_LATITUDE_DEGREES, MAX_LAYER_CAP, MAX_LONGITUDE_DEGREES, MAX_M, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_M, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, NATIVE_CRC32_MIN_BYTES, NEAREST_LINK_FALLBACK_HOSTS, NarsilError, PLACEMENT_ROOM_REQUESTS, PROTOTYPE_POLLUTION_KEYS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, ServerErrorCodes, VALID_HNSW_METRICS, VECTOR_PATTERN, codePointCount, compareCodePoints, createNarsilError, defaultSortMode, describeError, exceedsCodePoints, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, holdsUnpairedSurrogate, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isNarsilError, isPlainObject, isQuantizationMode, isSingleStringFieldType, isSortMode, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, sortModeOf, textFieldsChanged, toComparableSortValue, tokenizeOptions, truncateSortString, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, withSortableOption, withUnpairedSurrogatesReplaced };
|
|
1232
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER, CHECKSUM_YIELD_CHUNK_BYTES, COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, CONNECT_RESTARTS_BEFORE_ONE_PASS, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, ClientErrorCodes, DEFAULT_EF_SEARCH, DEG_TO_RAD, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, DOCUMENT_REPLY_ENVELOPE_DEPTH, EARLIER_RELEASE_MIN_M, EARTH_RADIUS_METERS, ErrorCodes, FARTHEST_REJECTED_LINKS_KEPT, FULL_TURN_DEGREES, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_ENCODING_DEPTH, MAX_DOCUMENT_NESTING_DEPTH, MAX_FILE_BYTES, MAX_LATITUDE_DEGREES, MAX_LAYER_CAP, MAX_LONGITUDE_DEGREES, MAX_M, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MIN_M, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, NATIVE_CRC32_MIN_BYTES, NEAREST_LINK_FALLBACK_HOSTS, NarsilError, PLACEMENT_ROOM_REQUESTS, PROTOTYPE_POLLUTION_KEYS, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, ServerErrorCodes, VALID_HNSW_METRICS, VECTOR_PATTERN, codePointCount, compareCodePoints, createNarsilError, defaultSortMode, describeError, exceedsCodePoints, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, holdsUnpairedSurrogate, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isNarsilError, isPlainObject, isQuantizationMode, isSingleStringFieldType, isSortMode, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, sortModeOf, textFieldsChanged, toComparableSortValue, tokenizeOptions, truncateSortString, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorDimensionOf, withSortableOption, withUnpairedSurrogatesReplaced };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes, requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes, requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-2Q45HI5H.mjs';
|
|
2
2
|
|
|
3
3
|
// src/core/stored-values/build.ts
|
|
4
4
|
var ORDINAL_LIMIT = 4294967295;
|