@delali/narsil 0.3.1-next.143 → 0.3.1-next.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/dist/adapters/filesystem.d.ts +1 -1
  2. package/dist/adapters/filesystem.mjs +101 -13
  3. package/dist/adapters/gcs.d.ts +49 -0
  4. package/dist/adapters/gcs.mjs +213 -0
  5. package/dist/adapters/indexeddb.d.ts +1 -1
  6. package/dist/adapters/indexeddb.mjs +2 -7
  7. package/dist/adapters/memory.d.ts +1 -1
  8. package/dist/adapters/memory.mjs +5 -0
  9. package/dist/adapters/s3.d.ts +53 -0
  10. package/dist/adapters/s3.mjs +263 -0
  11. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  12. package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
  13. package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
  14. package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
  15. package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
  16. package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
  17. package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
  18. package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
  19. package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
  20. package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
  21. package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
  22. package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
  23. package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
  24. package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
  25. package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
  26. package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
  27. package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
  28. package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
  29. package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
  30. package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
  31. package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
  32. package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
  33. package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
  34. package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
  35. package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
  36. package/dist/browser/import-FTMLTPGQ.mjs +9 -0
  37. package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
  38. package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
  39. package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
  40. package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
  41. package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
  42. package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
  43. package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
  44. package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
  45. package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
  46. package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
  47. package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
  48. package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
  49. package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
  50. package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
  51. package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
  52. package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
  53. package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
  54. package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
  55. package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
  56. package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
  57. package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
  58. package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
  59. package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
  60. package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
  61. package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
  62. package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
  63. package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
  64. package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
  65. package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
  66. package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
  67. package/dist/chunk-K5HSUPYS.mjs +11 -0
  68. package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
  69. package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
  70. package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
  71. package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
  72. package/dist/chunk-NWTEJ2T7.mjs +100 -0
  73. package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
  74. package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
  75. package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
  76. package/dist/chunk-PRID7CA6.mjs +104 -0
  77. package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
  78. package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
  79. package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
  80. package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
  81. package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
  82. package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
  83. package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
  84. package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
  85. package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
  86. package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
  87. package/dist/chunk-SVO6NV4D.mjs +248 -0
  88. package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
  89. package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
  90. package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
  91. package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
  92. package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
  93. package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
  94. package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
  95. package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
  96. package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
  97. package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
  98. package/dist/client.d.ts +8 -8
  99. package/dist/client.mjs +44 -0
  100. package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
  101. package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
  102. package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
  103. package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
  104. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  105. package/dist/distribution/coordinator/etcd.mjs +1 -1
  106. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  107. package/dist/distribution/transport/grpc.mjs +4 -4
  108. package/dist/distribution/transport/in-memory.mjs +2 -2
  109. package/dist/distribution/transport/tcp.mjs +3 -3
  110. package/dist/distribution.d.ts +8 -8
  111. package/dist/distribution.mjs +16219 -14535
  112. package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
  113. package/dist/embeddings/openai.d.ts +1 -1
  114. package/dist/embeddings/openai.mjs +2 -2
  115. package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
  116. package/dist/import-6KXFY6XI.mjs +35 -0
  117. package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
  118. package/dist/index.browser.mjs +4 -4
  119. package/dist/index.d.ts +7 -7
  120. package/dist/index.mjs +21 -30
  121. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  122. package/dist/invalidation/filesystem.d.ts +1 -1
  123. package/dist/invalidation/filesystem.mjs +1 -1
  124. package/dist/invalidation/noop.d.ts +1 -1
  125. package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
  126. package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
  127. package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
  128. package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
  129. package/dist/partial-words.mjs +7 -9
  130. package/dist/pattern.mjs +4 -6
  131. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  132. package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
  133. package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
  134. package/dist/react.d.ts +11 -11
  135. package/dist/react.mjs +9 -0
  136. package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
  137. package/dist/rerankers/http.d.ts +1 -1
  138. package/dist/rerankers/http.mjs +2 -2
  139. package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
  140. package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
  141. package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
  142. package/dist/server.d.ts +24 -12
  143. package/dist/server.mjs +153 -36
  144. package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
  145. package/dist/synonyms.mjs +3 -4
  146. package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
  147. package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
  148. package/dist/vector/search-worker.d.ts +2 -2
  149. package/dist/vector/search-worker.mjs +5 -6
  150. package/dist/workers/entry.mjs +7 -7
  151. package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
  152. package/openapi.json +349 -2
  153. package/package.json +27 -8
  154. package/dist/browser/import-TLB2A5LJ.mjs +0 -9
  155. package/dist/chunk-5WYVUPGD.mjs +0 -427
  156. package/dist/chunk-AV4IJKD2.mjs +0 -35
  157. package/dist/chunk-GES7U2PM.mjs +0 -68
  158. package/dist/chunk-KGOXHLLZ.mjs +0 -693
  159. package/dist/chunk-N37VMC62.mjs +0 -18
  160. package/dist/chunk-QSK35UC2.mjs +0 -35
  161. package/dist/chunk-WNBRW4Q2.mjs +0 -128
  162. package/dist/chunk-WQBCBKY3.mjs +0 -8
  163. package/dist/chunk-ZATRDAYU.mjs +0 -160
  164. package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
  165. package/dist/import-OOBUW3RR.mjs +0 -44
@@ -1,427 +0,0 @@
1
- import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KGOXHLLZ.mjs';
2
- import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
3
-
4
- // src/narsil/internals.ts
5
- var engineCores = /* @__PURE__ */ new WeakMap();
6
- var rerankWindowSources = /* @__PURE__ */ new WeakMap();
7
- function bindEngineCore(engine, core) {
8
- engineCores.set(engine, core);
9
- }
10
- function engineCoreOf(engine) {
11
- return engineCores.get(engine);
12
- }
13
- function bindRerankWindowSource(owner, read) {
14
- rerankWindowSources.set(owner, read);
15
- }
16
- function rerankWindowCapOf(owner) {
17
- const core = engineCores.get(owner);
18
- if (core !== void 0) return core.rerankSettings.maxWindow;
19
- return rerankWindowSources.get(owner)?.();
20
- }
21
-
22
- // src/engine/embed.ts
23
- function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
24
- for (const targetField of Object.keys(embeddingConfig.fields)) {
25
- const existing = resolveFieldValue(document, targetField);
26
- if (existing === void 0 || existing === null) {
27
- throw new NarsilError(
28
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
29
- adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
30
- adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
31
- );
32
- }
33
- }
34
- }
35
- function validateVector(vector, targetField, expectedDimensions) {
36
- if (!(vector instanceof Float32Array)) {
37
- throw new NarsilError(
38
- ErrorCodes.EMBEDDING_FAILED,
39
- `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
40
- { field: targetField, returnedType: typeof vector }
41
- );
42
- }
43
- if (vector.length !== expectedDimensions) {
44
- throw new NarsilError(
45
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
46
- `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
47
- { field: targetField, expected: expectedDimensions, actual: vector.length }
48
- );
49
- }
50
- for (let i = 0; i < vector.length; i++) {
51
- if (Number.isNaN(vector[i])) {
52
- throw new NarsilError(
53
- ErrorCodes.EMBEDDING_FAILED,
54
- `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
55
- { field: targetField, index: i }
56
- );
57
- }
58
- }
59
- return vector;
60
- }
61
- function resolveFieldValue(document, path) {
62
- if (!path.includes(".")) return document[path];
63
- const segments = path.split(".");
64
- let current = document;
65
- for (const segment of segments) {
66
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
67
- current = current[segment];
68
- }
69
- return current;
70
- }
71
- function setFieldValue(document, path, value) {
72
- if (!path.includes(".")) {
73
- document[path] = value;
74
- return;
75
- }
76
- const segments = path.split(".");
77
- let current = document;
78
- for (let i = 0; i < segments.length - 1; i++) {
79
- let next = current[segments[i]];
80
- if (next === null || next === void 0 || typeof next !== "object") {
81
- next = {};
82
- current[segments[i]] = next;
83
- }
84
- current = next;
85
- }
86
- current[segments[segments.length - 1]] = value;
87
- }
88
- function deleteFieldValue(document, path) {
89
- if (!path.includes(".")) {
90
- delete document[path];
91
- return;
92
- }
93
- const segments = path.split(".");
94
- let current = document;
95
- for (let i = 0; i < segments.length - 1; i++) {
96
- const next = current[segments[i]];
97
- if (next === null || next === void 0 || typeof next !== "object") return;
98
- current = next;
99
- }
100
- delete current[segments[segments.length - 1]];
101
- }
102
- function collectSourceText(document, sourceFields, targetField, documentIndex) {
103
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
104
- const parts = [];
105
- for (const source of sources) {
106
- const value = resolveFieldValue(document, source);
107
- if (value === void 0 || value === null || value === "") continue;
108
- if (typeof value !== "string") {
109
- throw new NarsilError(
110
- ErrorCodes.DOC_VALIDATION_FAILED,
111
- `Embedding source field "${source}" must be a string, got ${typeof value}`,
112
- { field: source, targetField }
113
- );
114
- }
115
- parts.push(value);
116
- }
117
- if (parts.length === 0) {
118
- const details = { field: targetField };
119
- if (documentIndex !== void 0) {
120
- details.documentIndex = documentIndex;
121
- }
122
- throw new NarsilError(
123
- ErrorCodes.EMBEDDING_NO_SOURCE,
124
- `All source fields for embedding target "${targetField}" are missing or empty`,
125
- details
126
- );
127
- }
128
- return parts.join("\n");
129
- }
130
- function wrapAdapterError(err, targetField) {
131
- if (err instanceof NarsilError) return err;
132
- const message = err instanceof Error ? err.message : String(err);
133
- return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
134
- field: targetField,
135
- cause: message
136
- });
137
- }
138
- async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
139
- const embeddedFields = /* @__PURE__ */ new Set();
140
- try {
141
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
142
- const existing = resolveFieldValue(document, targetField);
143
- if (existing !== void 0 && existing !== null) continue;
144
- const text = collectSourceText(document, sourceFields, targetField);
145
- let vector;
146
- try {
147
- const raw = await adapter.embed(text, "document", signal);
148
- vector = validateVector(raw, targetField, adapter.dimensions);
149
- } catch (err) {
150
- throw wrapAdapterError(err, targetField);
151
- }
152
- setFieldValue(document, targetField, vector);
153
- embeddedFields.add(targetField);
154
- }
155
- } catch (err) {
156
- for (const field of embeddedFields) {
157
- deleteFieldValue(document, field);
158
- }
159
- throw err;
160
- }
161
- return embeddedFields;
162
- }
163
- var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([
164
- ErrorCodes.EMBEDDING_NO_SOURCE,
165
- ErrorCodes.DOC_VALIDATION_FAILED
166
- ]);
167
- function isDocumentSourceError(err) {
168
- return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
169
- }
170
- async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
171
- const embedded = /* @__PURE__ */ new Map();
172
- const failed = /* @__PURE__ */ new Map();
173
- try {
174
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
175
- const needsEmbedding = [];
176
- for (let i = 0; i < documents.length; i++) {
177
- if (failed.has(i)) continue;
178
- const doc = documents[i];
179
- const existing = resolveFieldValue(doc, targetField);
180
- if (existing !== void 0 && existing !== null) continue;
181
- try {
182
- const text = collectSourceText(doc, sourceFields, targetField, i);
183
- needsEmbedding.push({ docIndex: i, text });
184
- } catch (err) {
185
- if (isDocumentSourceError(err)) {
186
- failed.set(i, err);
187
- } else {
188
- throw err;
189
- }
190
- }
191
- }
192
- if (needsEmbedding.length === 0) continue;
193
- const texts = needsEmbedding.map((entry) => entry.text);
194
- let vectors;
195
- try {
196
- if (adapter.embedBatch) {
197
- vectors = await adapter.embedBatch(texts, "document", signal);
198
- } else {
199
- vectors = [];
200
- const concurrency = 8;
201
- for (let start = 0; start < texts.length; start += concurrency) {
202
- const chunk = texts.slice(start, start + concurrency);
203
- const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
204
- vectors.push(...chunkResults);
205
- }
206
- }
207
- } catch (err) {
208
- throw wrapAdapterError(err, targetField);
209
- }
210
- if (vectors.length !== needsEmbedding.length) {
211
- throw new NarsilError(
212
- ErrorCodes.EMBEDDING_FAILED,
213
- `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
214
- { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
215
- );
216
- }
217
- for (let j = 0; j < needsEmbedding.length; j++) {
218
- const { docIndex } = needsEmbedding[j];
219
- const validated = validateVector(vectors[j], targetField, adapter.dimensions);
220
- setFieldValue(documents[docIndex], targetField, validated);
221
- let fieldSet = embedded.get(docIndex);
222
- if (!fieldSet) {
223
- fieldSet = /* @__PURE__ */ new Set();
224
- embedded.set(docIndex, fieldSet);
225
- }
226
- fieldSet.add(targetField);
227
- }
228
- }
229
- } catch (err) {
230
- for (const [docIndex, fields] of embedded) {
231
- for (const field of fields) {
232
- deleteFieldValue(documents[docIndex], field);
233
- }
234
- }
235
- throw err;
236
- }
237
- return { embedded, failed };
238
- }
239
-
240
- // src/engine/mutations/insert-admission.ts
241
- function providedDocId(document) {
242
- if (document === null || typeof document !== "object") return void 0;
243
- const id = document.id;
244
- return typeof id === "string" && id.length > 0 ? id : void 0;
245
- }
246
- function requireDocumentObject(document) {
247
- if (document !== null && typeof document === "object" && !Array.isArray(document)) {
248
- requireDocumentWithinSizeLimit(document);
249
- return;
250
- }
251
- const received = Array.isArray(document) ? "a list" : document === null ? "null" : `a ${typeof document}`;
252
- throw new NarsilError(
253
- ErrorCodes.DOC_VALIDATION_FAILED,
254
- `Each document must be an object, while this entry is ${received}`,
255
- {
256
- received
257
- }
258
- );
259
- }
260
- function requireMatchingDocId(docId, document) {
261
- const embeddedDocId = providedDocId(document);
262
- if (embeddedDocId === void 0 || embeddedDocId === docId) return;
263
- throw new NarsilError(
264
- ErrorCodes.DOC_VALIDATION_FAILED,
265
- `The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
266
- { docId, documentId: embeddedDocId }
267
- );
268
- }
269
- function asBatchInsertError(err) {
270
- return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
271
- }
272
- function createPendingAdmissions() {
273
- return { total: 0, byPartition: /* @__PURE__ */ new Map() };
274
- }
275
- function capsEachPartition(ctx, manager) {
276
- return ctx.capacityScope === "partition" && manager.config.partitions?.maxDocsPerPartition !== void 0;
277
- }
278
- function recordPendingAdmission(ctx, pending, manager, docId) {
279
- pending.total += 1;
280
- if (!capsEachPartition(ctx, manager)) return;
281
- const partitionId = manager.routePartition(docId);
282
- pending.byPartition.set(partitionId, (pending.byPartition.get(partitionId) ?? 0) + 1);
283
- }
284
- function admitInsert(ctx, indexName, manager, docId, pending) {
285
- const rebalancing = ctx.isRebalancing(indexName);
286
- if (rebalancing) {
287
- const bufferedState = ctx.bufferedDocState(indexName, docId);
288
- const exists = bufferedState !== void 0 ? bufferedState === "present" : manager.has(docId);
289
- if (exists) {
290
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, { docId });
291
- }
292
- }
293
- assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing);
294
- }
295
- function admitBufferedUpsert(ctx, indexName, manager, docId) {
296
- assertRoomFor(ctx, indexName, manager, docId, void 0, true);
297
- }
298
- function assertRoomFor(ctx, indexName, manager, docId, pending, rebalancing) {
299
- if (ctx.capacityScope === "partition") {
300
- if (!capsEachPartition(ctx, manager)) return;
301
- const partitionId = manager.routePartition(docId);
302
- manager.assertPartitionCapacity(partitionId, pending?.byPartition.get(partitionId) ?? 0);
303
- return;
304
- }
305
- const pendingAdmitted = pending?.total ?? 0;
306
- if (!rebalancing) {
307
- manager.assertCapacity(pendingAdmitted);
308
- return;
309
- }
310
- manager.assertCapacity(
311
- ctx.pendingRebalanceWrites(indexName) + pendingAdmitted,
312
- ctx.rebalanceTargetPartitionCount(indexName)
313
- );
314
- }
315
- function collectRequiredFieldFailures(documents, chunkStart, chunkEnd, required, chunkFailedIndexes, failed) {
316
- for (let i = chunkStart; i < chunkEnd; i++) {
317
- try {
318
- validateRequiredFields(documents[i], required);
319
- } catch (err) {
320
- chunkFailedIndexes.add(i);
321
- failed.push({ docId: providedDocId(documents[i]) ?? "", error: asBatchInsertError(err) });
322
- }
323
- }
324
- }
325
- async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signal, chunkFailedIndexes, failed) {
326
- if (entry.config.embedding === void 0) return;
327
- if (entry.embeddingAdapter === null) {
328
- for (let i = chunkStart; i < chunkEnd; i++) {
329
- if (chunkFailedIndexes.has(i)) continue;
330
- try {
331
- assertDocumentCarriesMappedVectors(
332
- documents[i],
333
- entry.config.embedding,
334
- entry.embeddingAdapterName
335
- );
336
- } catch (err) {
337
- chunkFailedIndexes.add(i);
338
- failed.push({
339
- docId: providedDocId(documents[i]) ?? "",
340
- error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
341
- });
342
- }
343
- }
344
- return;
345
- }
346
- const embeddableSlice = [];
347
- const embeddableOriginalIndexes = [];
348
- for (let i = chunkStart; i < chunkEnd; i++) {
349
- if (chunkFailedIndexes.has(i)) continue;
350
- embeddableSlice.push(documents[i]);
351
- embeddableOriginalIndexes.push(i);
352
- }
353
- if (embeddableSlice.length === 0) return;
354
- try {
355
- const embedResult = await embedBatchDocumentFields(
356
- embeddableSlice,
357
- entry.config.embedding,
358
- entry.embeddingAdapter,
359
- signal
360
- );
361
- for (const [sliceIndex, error] of embedResult.failed) {
362
- const originalIdx = embeddableOriginalIndexes[sliceIndex];
363
- chunkFailedIndexes.add(originalIdx);
364
- failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
365
- }
366
- } catch (err) {
367
- const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
368
- for (const originalIdx of embeddableOriginalIndexes) {
369
- chunkFailedIndexes.add(originalIdx);
370
- failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
371
- }
372
- }
373
- }
374
-
375
- // src/engine/resolve-vector-text.ts
376
- async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
377
- if (!params.vector) return params;
378
- if (params.vector.text === void 0 && params.vector.value === void 0) return params;
379
- if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
380
- if (params.vector.text !== void 0 && params.vector.value !== void 0) {
381
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
382
- }
383
- if (!embeddingAdapter) {
384
- throw new NarsilError(
385
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
386
- embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
387
- embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
388
- );
389
- }
390
- if (typeof params.vector.text !== "string") {
391
- throw new NarsilError(
392
- ErrorCodes.DOC_VALIDATION_FAILED,
393
- `Vector query 'text' must be a string, got ${typeof params.vector.text}`
394
- );
395
- }
396
- if (params.vector.text.length === 0) {
397
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
398
- }
399
- let raw;
400
- try {
401
- raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
402
- } catch (err) {
403
- if (err instanceof NarsilError) throw err;
404
- const message = err instanceof Error ? err.message : String(err);
405
- throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `The embedding adapter cannot embed the query: ${message}`, {
406
- cause: message
407
- });
408
- }
409
- if (!(raw instanceof Float32Array)) {
410
- throw new NarsilError(
411
- ErrorCodes.EMBEDDING_FAILED,
412
- `Adapter returned ${typeof raw} for query embedding, expected Float32Array`
413
- );
414
- }
415
- if (raw.length !== embeddingAdapter.dimensions) {
416
- throw new NarsilError(
417
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
418
- `Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
419
- { expected: embeddingAdapter.dimensions, actual: raw.length }
420
- );
421
- }
422
- const resolved = { ...params.vector, value: Array.from(raw) };
423
- delete resolved.text;
424
- return { ...params, vector: resolved };
425
- }
426
-
427
- export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, bindRerankWindowSource, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, recordPendingAdmission, requireDocumentObject, requireMatchingDocId, rerankWindowCapOf, resolveVectorText };
@@ -1,35 +0,0 @@
1
- // src/runtime/detect.ts
2
- function detectRuntimeName() {
3
- if ("Bun" in globalThis) {
4
- return "bun";
5
- }
6
- if ("Deno" in globalThis) {
7
- return "deno";
8
- }
9
- const proc = globalThis.process;
10
- if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
11
- return "node";
12
- }
13
- return "browser";
14
- }
15
- function getCpuCount() {
16
- const nav = globalThis.navigator;
17
- if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
18
- return nav.hardwareConcurrency;
19
- }
20
- return 1;
21
- }
22
- function detectRuntime() {
23
- const runtime = detectRuntimeName();
24
- return {
25
- runtime,
26
- cpuCount: getCpuCount(),
27
- supportsWorkerThreads: runtime === "node" || runtime === "bun",
28
- supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
29
- supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
30
- supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
31
- supportsBroadcastChannel: "BroadcastChannel" in globalThis
32
- };
33
- }
34
-
35
- export { detectRuntime };
@@ -1,68 +0,0 @@
1
- // src/core/fuzzy.ts
2
- var FIRST_SURROGATE = 55296;
3
- var LAST_SURROGATE = 57343;
4
- var LAST_BASIC_PLANE_CODE_POINT = 65535;
5
- function hasSurrogate(text) {
6
- for (let index = 0; index < text.length; index++) {
7
- const unit = text.charCodeAt(index);
8
- if (unit >= FIRST_SURROGATE && unit <= LAST_SURROGATE) return true;
9
- }
10
- return false;
11
- }
12
- function fuzzyPrefixOf(queryTerm, prefixLength) {
13
- let end = 0;
14
- for (let counted = 0; counted < prefixLength && end < queryTerm.length; counted++) {
15
- const codePoint = queryTerm.codePointAt(end);
16
- end += codePoint !== void 0 && codePoint > LAST_BASIC_PLANE_CODE_POINT ? 2 : 1;
17
- }
18
- return queryTerm.slice(0, end);
19
- }
20
- function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
21
- if (queryTerm === indexTerm) return true;
22
- if (tolerance <= 0) return false;
23
- if (!indexTerm.startsWith(fuzzyPrefixOf(queryTerm, prefixLength))) return false;
24
- return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
25
- }
26
- function boundedLevenshtein(a, b, tolerance) {
27
- if (tolerance < 0) return { distance: -1, withinTolerance: false, cells: 0 };
28
- if (a === b) return { distance: 0, withinTolerance: true, cells: 0 };
29
- return tabulatedLevenshtein(a, b, tolerance);
30
- }
31
- function tabulatedLevenshtein(a, b, tolerance) {
32
- if (tolerance < 0) return { distance: -1, withinTolerance: false, cells: 0 };
33
- if (hasSurrogate(a) || hasSurrogate(b)) return bandedDistance(Array.from(a), Array.from(b), tolerance);
34
- return bandedDistance(a, b, tolerance);
35
- }
36
- function bandedDistance(a, b, tolerance) {
37
- const m = a.length;
38
- const n = b.length;
39
- const beyond = tolerance + 1;
40
- if (Math.abs(m - n) > tolerance) return { distance: beyond, withinTolerance: false, cells: 0 };
41
- let previous = new Uint32Array(n + 1).fill(beyond);
42
- let current = new Uint32Array(n + 1).fill(beyond);
43
- const firstRowEnd = Math.min(n, tolerance);
44
- for (let j = 0; j <= firstRowEnd; j++) previous[j] = j;
45
- let cells = 0;
46
- for (let i = 1; i <= m; i++) {
47
- const low = Math.max(1, i - tolerance);
48
- const high = Math.min(n, i + tolerance);
49
- current[low - 1] = low === 1 ? i : beyond;
50
- let rowMin = current[low - 1];
51
- const character = a[i - 1];
52
- if (high >= low) cells += high - low + 1;
53
- for (let j = low; j <= high; j++) {
54
- const substitution = previous[j - 1] + (character === b[j - 1] ? 0 : 1);
55
- const cell = Math.min(previous[j] + 1, current[j - 1] + 1, substitution, beyond);
56
- current[j] = cell;
57
- if (cell < rowMin) rowMin = cell;
58
- }
59
- if (rowMin > tolerance) return { distance: beyond, withinTolerance: false, cells };
60
- const finished = previous;
61
- previous = current;
62
- current = finished;
63
- }
64
- const distance = previous[n];
65
- return { distance, withinTolerance: distance <= tolerance, cells };
66
- }
67
-
68
- export { boundedLevenshtein, fuzzyPrefixOf, fuzzyTermMatches, tabulatedLevenshtein };