@delali/narsil 0.3.1-next.95 → 0.3.1-next.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +2 -1
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/{chunk-3U636ADD.mjs → chunk-CN4I4FZU.mjs} +2 -2
  9. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  10. package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
  11. package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-NAFT3K6X.mjs} +240 -135
  12. package/dist/browser/{chunk-LULHQXCK.mjs → chunk-NNG3EEB6.mjs} +5 -4
  13. package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
  14. package/dist/browser/chunk-RELZRJNR.mjs +203 -0
  15. package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
  16. package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
  17. package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
  18. package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
  19. package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
  20. package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-J57MZK6L.mjs} +115 -61
  21. package/dist/browser/{write-ZUBDNLCH.mjs → write-L7FAQA47.mjs} +6 -4
  22. package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
  23. package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
  24. package/dist/{chunk-7QONURHW.mjs → chunk-45AVKHMJ.mjs} +2 -2
  25. package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
  26. package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
  27. package/dist/{chunk-54HYC5FT.mjs → chunk-BKB3XTMI.mjs} +6 -6
  28. package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
  29. package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
  30. package/dist/chunk-FMWRCP2C.mjs +57 -0
  31. package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
  32. package/dist/chunk-HJSN7UYG.mjs +10 -0
  33. package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
  34. package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
  35. package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
  36. package/dist/chunk-K7D4ZTSW.mjs +126 -0
  37. package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
  38. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  39. package/dist/{chunk-J7SDAOTP.mjs → chunk-L3L5YFXC.mjs} +4 -4
  40. package/dist/{chunk-Q4UTU5ZE.mjs → chunk-LH4NZNJJ.mjs} +10 -10
  41. package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
  42. package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
  43. package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
  44. package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
  45. package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
  46. package/dist/{chunk-L26LEDHO.mjs → chunk-NTRMFARG.mjs} +7 -7
  47. package/dist/chunk-NW4XDSP2.mjs +212 -0
  48. package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
  49. package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
  50. package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
  51. package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
  52. package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
  53. package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
  54. package/dist/{chunk-JFUE6F2Y.mjs → chunk-VWULPWRJ.mjs} +60 -21
  55. package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
  56. package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
  57. package/dist/{chunk-MASE2LTJ.mjs → chunk-ZG477IPV.mjs} +7 -7
  58. package/dist/chunk-ZYNAMIYX.mjs +113 -0
  59. package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
  60. package/dist/client.d.ts +9 -9
  61. package/dist/client.mjs +2 -0
  62. package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
  63. package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
  64. package/dist/direct-executor-CXZ66ACP.mjs +22 -0
  65. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  66. package/dist/distribution/coordinator/etcd.mjs +1 -1
  67. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  68. package/dist/distribution/transport/grpc.mjs +1 -1
  69. package/dist/distribution.d.ts +24 -12
  70. package/dist/distribution.mjs +177 -54
  71. package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
  72. package/dist/embeddings/openai.d.ts +2 -2
  73. package/dist/embeddings/openai.mjs +22 -133
  74. package/dist/{engine-BGt9lI1V.d.ts → engine-BGCEm9E_.d.ts} +50 -20
  75. package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
  76. package/dist/index.browser.mjs +4 -2
  77. package/dist/index.d.ts +8 -8
  78. package/dist/index.mjs +19 -18
  79. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  80. package/dist/invalidation/filesystem.d.ts +1 -1
  81. package/dist/invalidation/noop.d.ts +1 -1
  82. package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
  83. package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
  84. package/dist/partial-words.mjs +6 -6
  85. package/dist/pattern.mjs +3 -3
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  87. package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
  88. package/dist/react.d.ts +8 -8
  89. package/dist/react.mjs +2 -0
  90. package/dist/rerank-NWTKK636.mjs +60 -0
  91. package/dist/rerankers/http.d.ts +50 -0
  92. package/dist/rerankers/http.mjs +104 -0
  93. package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-JPRBUYII.mjs} +1 -1
  94. package/dist/runtime-ZKYZFJJC.mjs +177 -0
  95. package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
  96. package/dist/server.d.ts +11 -11
  97. package/dist/server.mjs +63 -22
  98. package/dist/{snapshot-YC6TS4OP.mjs → snapshot-HOGXNO5W.mjs} +138 -83
  99. package/dist/synonyms.mjs +3 -3
  100. package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
  101. package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
  102. package/dist/vector/search-worker.d.ts +6 -2
  103. package/dist/vector/search-worker.mjs +9 -7
  104. package/dist/workers/entry.mjs +5 -5
  105. package/dist/{write-M2OALSA4.mjs → write-XLIGTXBU.mjs} +21 -20
  106. package/package.json +11 -7
  107. package/dist/direct-executor-5KJGVQHM.mjs +0 -22
package/README.md CHANGED
@@ -102,7 +102,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
102
102
 
103
103
  **Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
104
104
 
105
- **Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter of your own.
105
+ **Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter of your own. A query can [rerank](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) its top hits with a cross-encoder, through a Cohere-compatible rerank service, a local Transformers.js model, or an adapter of your own.
106
106
 
107
107
  **Geosearch.** [Geo filters](https://github.com/assetcorp/narsil/blob/main/docs/geosearch.md) match documents by radius, using Haversine or Vincenty distance, or by polygon containment. You can combine a geo filter with every other query feature.
108
108
 
@@ -124,6 +124,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
124
124
  | [Hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md) | Reciprocal rank fusion and linear blending of text and vector rankings |
125
125
  | [Geosearch](https://github.com/assetcorp/narsil/blob/main/docs/geosearch.md) | Radius and polygon filters, and the two distance formulas |
126
126
  | [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) | Automatic embedding on insert and query, named adapters, the bundled ones, and custom ones |
127
+ | [Reranking](https://github.com/assetcorp/narsil/blob/main/docs/reranking.md) | Reordering the top hits with a cross-encoder, the bundled rerank adapters, and custom ones |
127
128
  | [Persistence and durability](https://github.com/assetcorp/narsil/blob/main/docs/persistence-and-durability.md) | Storage backends, the write-ahead log, checkpoints, recovery, snapshots, document exports, and indexes from an earlier release |
128
129
  | [Partitions and workers](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) | Partition routing, online rebalancing, worker copies, and multi-instance invalidation |
129
130
  | [Language support](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) | The 107 language modules, analysis revisions and rebuilds, and named tokenizers and stop words |
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-79OIa4xm.js';
1
+ import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
2
2
 
3
3
  /**
4
4
  * Where {@link createFilesystemPersistence} writes its partitions.
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-SYWGORQB.mjs';
1
+ import { NarsilError, ErrorCodes } from '../chunk-KV3AZFT3.mjs';
2
2
 
3
3
  // src/persistence/filesystem.ts
4
4
  var cachedFs = null;
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-79OIa4xm.js';
1
+ import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
2
2
 
3
3
  /**
4
4
  * Which IndexedDB database and object store
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-SYWGORQB.mjs';
1
+ import { NarsilError, ErrorCodes } from '../chunk-KV3AZFT3.mjs';
2
2
 
3
3
  // src/persistence/constants.ts
4
4
  var DEFAULT_INDEXEDDB_NAME = "narsil";
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-79OIa4xm.js';
1
+ import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
2
2
 
3
3
  /**
4
4
  * Builds a persistence adapter that keeps every partition in a map, which is
@@ -145,5 +145,31 @@ interface EmbeddingAdapter {
145
145
  /** Releases whatever the adapter holds open, such as a model or a connection pool. */
146
146
  shutdown?(): Promise<void>;
147
147
  }
148
+ /**
149
+ * Scores the top hits of a query with a second model, such as a cross-encoder
150
+ * that reads the query and each document together, so that a query can
151
+ * reorder its first-stage ranking.
152
+ *
153
+ * Register one under a name through {@link NarsilConfig.rerankAdapters} or
154
+ * {@link AdapterRegistrationOperations.registerRerankAdapter}, and give that
155
+ * name in a query's `rerank.adapter`.
156
+ *
157
+ * @public
158
+ */
159
+ interface RerankAdapter {
160
+ /**
161
+ * Scores every document against one query.
162
+ *
163
+ * @param query - The text that the model compares each document with.
164
+ * @param documents - The text of each document, which is an empty string
165
+ * where a document holds no text in the reranked fields.
166
+ * @param signal - Aborts a call that the engine no longer needs.
167
+ * @returns One finite score per document, in the order of `documents`,
168
+ * where a higher score marks a more relevant document.
169
+ */
170
+ rerank(query: string, documents: string[], signal?: AbortSignal): Promise<number[]>;
171
+ /** Releases whatever the adapter holds open, such as a model or a connection pool. The engine calls it once as it shuts down. */
172
+ shutdown?(): Promise<void>;
173
+ }
148
174
 
149
- export type { EmbeddingAdapter as E, InvalidationAdapter as I, PersistenceAdapter as P, InvalidationEvent as a, PartitionStatistics as b };
175
+ export type { EmbeddingAdapter as E, InvalidationAdapter as I, PersistenceAdapter as P, RerankAdapter as R, InvalidationEvent as a, PartitionStatistics as b };
@@ -1,5 +1,5 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-D3CC2NMZ.mjs';
2
- import { NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-NAFT3K6X.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-RELZRJNR.mjs';
3
3
  import { decode } from '@msgpack/msgpack';
4
4
 
5
5
  function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
@@ -0,0 +1,10 @@
1
+ // src/rerank/constants.ts
2
+ var DEFAULT_RERANK_TIMEOUT_MS = 3e4;
3
+ var MIN_RERANK_TIMEOUT_MS = 1;
4
+ var MAX_RERANK_TIMEOUT_MS = 2147483647;
5
+ var DEFAULT_MAX_RERANK_WINDOW = 100;
6
+ var MIN_RERANK_WINDOW = 1;
7
+ var MAX_RERANK_WINDOW = 1e3;
8
+ var DEFAULT_RERANK_WINDOW = 100;
9
+
10
+ export { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW, MAX_RERANK_TIMEOUT_MS, MAX_RERANK_WINDOW, MIN_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW };
@@ -1,4 +1,5 @@
1
- import { NarsilError, ErrorCodes, MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-HARBSUWP.mjs';
1
+ import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-QWES44AW.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-RELZRJNR.mjs';
2
3
 
3
4
  // src/vector/vector-index/dead-bits.ts
4
5
  var NO_LIVE_VECTOR_AT_POSITION = -1;