@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-CmYJO34X.js';
3
- import { E as ErrorDetail, h as TaskStore } from './tasks-Co8HfRD7.js';
2
+ import { C as ClusterNamespace } from './config-CIG5yDyg.js';
3
+ import { E as ErrorDetail, i as TaskStore } from './tasks-CjRJRGTb.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -286,6 +286,13 @@ declare const TASK_FILTER_CAPABILITY = "tasks.filter";
286
286
  * @public
287
287
  */
288
288
  declare const REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
289
+ /**
290
+ * Marks that `POST /indexes/{name}/_change-analysis` changes the language, the
291
+ * tokeniser, or the stop words of a live index.
292
+ *
293
+ * @public
294
+ */
295
+ declare const CHANGE_ANALYSIS_CAPABILITY = "indexes.changeAnalysis";
289
296
  /**
290
297
  * Marks that `POST /indexes/{name}/_open` and `POST /indexes/{name}/_close`
291
298
  * load and release an index's server memory.
@@ -327,4 +334,4 @@ declare const BM25_TUNING_CAPABILITY = "indexes.bm25";
327
334
  */
328
335
  declare const SERVER_CAPABILITIES: readonly string[];
329
336
 
330
- export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
337
+ export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B, CHANGE_ANALYSIS_CAPABILITY as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type CorsOptions as d, type RequestContext as e, type RequestDenial as f, type ServerLimits as g };
@@ -0,0 +1,21 @@
1
+ // src/engine/analysis-change-entry.ts
2
+ function holdsChangeWork(entry) {
3
+ return entry !== void 0 && (entry.pendingAnalysis !== null || entry.durableAnalysis !== null);
4
+ }
5
+ async function abandonChangeOf(core, indexName) {
6
+ if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
7
+ const { abandonAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
8
+ await abandonAnalysisChange(core, indexName);
9
+ }
10
+ async function resumeChangeOf(core, indexName) {
11
+ if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
12
+ if (core.mutationCtx.capacityScope === "partition") return;
13
+ if (core.executor.getManager(indexName) === void 0) return;
14
+ const { resumeAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
15
+ void resumeAnalysisChange(core, indexName).catch(() => void 0);
16
+ }
17
+ async function resumeEveryChange(core) {
18
+ for (const indexName of [...core.indexRegistry.keys()]) await resumeChangeOf(core, indexName);
19
+ }
20
+
21
+ export { abandonChangeOf, resumeChangeOf, resumeEveryChange };
@@ -1,5 +1,5 @@
1
- import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-PU6SKH7O.mjs';
2
- import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
1
+ import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KGOXHLLZ.mjs';
2
+ import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
3
3
 
4
4
  // src/narsil/internals.ts
5
5
  var engineCores = /* @__PURE__ */ new WeakMap();
@@ -19,191 +19,6 @@ function rerankWindowCapOf(owner) {
19
19
  return rerankWindowSources.get(owner)?.();
20
20
  }
21
21
 
22
- // src/engine/snapshot/restore-config.ts
23
- function rejectSnapshot(message) {
24
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
25
- }
26
- function isPlainObject(value) {
27
- return value !== null && typeof value === "object" && !Array.isArray(value);
28
- }
29
- function restoredBm25(raw, reject) {
30
- if (raw === void 0) {
31
- return void 0;
32
- }
33
- if (!isPlainObject(raw)) {
34
- reject("bm25 must be an object");
35
- }
36
- const fields = restoredBm25Fields(raw.fields, reject);
37
- return {
38
- ...restoredBm25Pair(raw, "bm25", reject),
39
- ...fields !== void 0 ? { fields } : {}
40
- };
41
- }
42
- function restoredBm25Pair(raw, label, reject) {
43
- const { k1, b } = raw;
44
- if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
45
- reject(`${label} k1 must be a finite number`);
46
- }
47
- if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
48
- reject(`${label} b must be a finite number`);
49
- }
50
- return {
51
- ...k1 !== void 0 ? { k1 } : {},
52
- ...b !== void 0 ? { b } : {}
53
- };
54
- }
55
- function restoredBm25Fields(raw, reject) {
56
- if (raw === void 0) return void 0;
57
- if (!isPlainObject(raw)) reject("bm25 fields must be an object");
58
- const source = raw;
59
- const fields = /* @__PURE__ */ Object.create(null);
60
- for (const name of Object.keys(source)) {
61
- const pair = source[name];
62
- if (!isPlainObject(pair)) reject("bm25 fields must map each field to an object");
63
- fields[name] = restoredBm25Pair(pair, "bm25 field", reject);
64
- }
65
- return fields;
66
- }
67
- function restoredStopWords(name, list, reject) {
68
- if (name !== void 0 && typeof name !== "string") {
69
- reject("stopWords must be a name");
70
- }
71
- if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
72
- reject("stopWordList must be a list of words");
73
- }
74
- if (typeof name === "string" && Array.isArray(list)) {
75
- reject("stopWords and stopWordList cannot both be present");
76
- }
77
- if (typeof name === "string") {
78
- return name;
79
- }
80
- if (Array.isArray(list)) {
81
- return new Set(list);
82
- }
83
- return void 0;
84
- }
85
- function restoredPartitionConfig(raw, reject = rejectSnapshot) {
86
- if (raw === void 0) {
87
- return void 0;
88
- }
89
- if (!isPlainObject(raw)) {
90
- reject("partitionConfig must be an object");
91
- }
92
- const { maxDocsPerPartition, maxPartitions, watermark } = raw;
93
- if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
94
- reject("partitionConfig maxDocsPerPartition must be a positive integer");
95
- }
96
- if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
97
- reject("partitionConfig maxPartitions must be a positive integer");
98
- }
99
- if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
100
- reject("partitionConfig watermark must be above 0 and at most 1");
101
- }
102
- return {
103
- ...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
104
- ...typeof maxPartitions === "number" ? { maxPartitions } : {},
105
- ...typeof watermark === "number" ? { watermark } : {}
106
- };
107
- }
108
- function restoredScoring(raw, reject) {
109
- if (raw === void 0) {
110
- return void 0;
111
- }
112
- if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
113
- reject("defaultScoring must be 'local', 'dfs', or 'broadcast'");
114
- }
115
- return raw;
116
- }
117
- function restoredBoolean(raw, field, reject) {
118
- if (raw === void 0) {
119
- return void 0;
120
- }
121
- if (typeof raw !== "boolean") {
122
- reject(`${field} must be a boolean`);
123
- }
124
- return raw;
125
- }
126
- function restoredRequired(raw, reject) {
127
- if (raw === void 0) {
128
- return void 0;
129
- }
130
- if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
131
- reject("required must be a list of field paths");
132
- }
133
- return raw;
134
- }
135
- function restoredVectorPromotion(raw, reject) {
136
- if (raw === void 0) {
137
- return void 0;
138
- }
139
- if (!isPlainObject(raw) || raw.hnswConfig !== void 0 && !isPlainObject(raw.hnswConfig)) {
140
- reject("vectorPromotion must be an object");
141
- }
142
- return raw;
143
- }
144
- function restoredEmbedding(raw, reject = rejectSnapshot) {
145
- if (raw === void 0) {
146
- return void 0;
147
- }
148
- if (!isPlainObject(raw)) {
149
- reject("embedding must be an object");
150
- }
151
- const { fields, adapter } = raw;
152
- if (adapter !== void 0 && typeof adapter !== "string") {
153
- reject("embedding adapter must be a name");
154
- }
155
- if (!isPlainObject(fields)) {
156
- reject("embedding fields must be an object");
157
- }
158
- for (const value of Object.values(fields)) {
159
- const isPath = typeof value === "string";
160
- const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
161
- if (!isPath && !isPathList) {
162
- reject("embedding fields must map vector fields to source paths");
163
- }
164
- }
165
- return {
166
- fields,
167
- ...adapter !== void 0 ? { adapter } : {}
168
- };
169
- }
170
- function readSharedConfigFields(source, reject) {
171
- if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
172
- reject("tokenizer must be a name");
173
- }
174
- const stopWords = restoredStopWords(source.stopWords, source.stopWordList, reject);
175
- const bm25 = restoredBm25(source.bm25, reject);
176
- const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject);
177
- const defaultScoring = restoredScoring(source.defaultScoring, reject);
178
- const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject);
179
- const strict = restoredBoolean(source.strict, "strict", reject);
180
- const required = restoredRequired(source.required, reject);
181
- const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject);
182
- if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
183
- reject("patternValueLimit must be a number");
184
- }
185
- return {
186
- ...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
187
- ...stopWords !== void 0 ? { stopWords } : {},
188
- ...bm25 !== void 0 ? { bm25 } : {},
189
- ...surfaceForms !== void 0 ? { surfaceForms } : {},
190
- ...defaultScoring !== void 0 ? { defaultScoring } : {},
191
- ...trackPositions !== void 0 ? { trackPositions } : {},
192
- ...strict !== void 0 ? { strict } : {},
193
- ...required !== void 0 ? { required } : {},
194
- ...vectorPromotion !== void 0 ? { vectorPromotion } : {},
195
- ...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
196
- };
197
- }
198
- function restoredConfigFields(envelope, reject = rejectSnapshot) {
199
- const shared = readSharedConfigFields(envelope, reject);
200
- const partitions = restoredPartitionConfig(envelope.partitionConfig, reject);
201
- return {
202
- ...shared,
203
- ...partitions !== void 0 ? { partitions } : {}
204
- };
205
- }
206
-
207
22
  // src/engine/embed.ts
208
23
  function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
209
24
  for (const targetField of Object.keys(embeddingConfig.fields)) {
@@ -609,4 +424,4 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
609
424
  return { ...params, vector: resolved };
610
425
  }
611
426
 
612
- export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, bindRerankWindowSource, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, rejectSnapshot, requireDocumentObject, requireMatchingDocId, rerankWindowCapOf, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
427
+ export { admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, bindRerankWindowSource, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, recordPendingAdmission, requireDocumentObject, requireMatchingDocId, rerankWindowCapOf, resolveVectorText };
@@ -0,0 +1,214 @@
1
+ import { untagDocumentLine } from './chunk-BOJZGUS6.mjs';
2
+ import { chunksOf } from './chunk-CZQ33DLW.mjs';
3
+ import { restoredConfigFields, restoredEmbedding } from './chunk-L76W3IZJ.mjs';
4
+ import { requireConfigurationShape } from './chunk-IO5F7ZBE.mjs';
5
+ import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
6
+ import { flattenSchema } from './chunk-KGOXHLLZ.mjs';
7
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
8
+
9
+ // src/engine/export/lines.ts
10
+ var NEWLINE = 10;
11
+ function joined(parts, total) {
12
+ if (parts.length === 1) return parts[0];
13
+ const bytes = new Uint8Array(total);
14
+ let offset = 0;
15
+ for (const part of parts) {
16
+ bytes.set(part, offset);
17
+ offset += part.byteLength;
18
+ }
19
+ return bytes;
20
+ }
21
+ async function* exportLines(source, maxLineBytes) {
22
+ const decoder = new TextDecoder("utf-8", { fatal: true });
23
+ let parts = [];
24
+ let pending = 0;
25
+ let overflowing = false;
26
+ let line = 0;
27
+ const finishLine = () => {
28
+ line += 1;
29
+ if (overflowing) {
30
+ overflowing = false;
31
+ parts = [];
32
+ pending = 0;
33
+ return { line, refused: "too-long" };
34
+ }
35
+ const bytes = joined(parts, pending);
36
+ parts = [];
37
+ pending = 0;
38
+ try {
39
+ return { line, text: decoder.decode(bytes) };
40
+ } catch {
41
+ return { line, refused: "not-utf8" };
42
+ }
43
+ };
44
+ for await (const chunk of chunksOf(source)) {
45
+ let start = 0;
46
+ for (; ; ) {
47
+ const newline = chunk.indexOf(NEWLINE, start);
48
+ const end = newline < 0 ? chunk.byteLength : newline;
49
+ const piece = chunk.subarray(start, end);
50
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
51
+ if (piece.byteLength > 0) parts.push(piece.slice());
52
+ pending += piece.byteLength;
53
+ } else {
54
+ overflowing = true;
55
+ parts = [];
56
+ }
57
+ if (newline < 0) break;
58
+ yield finishLine();
59
+ start = newline + 1;
60
+ }
61
+ }
62
+ if (pending > 0 || overflowing) yield finishLine();
63
+ }
64
+
65
+ // src/engine/export/import.ts
66
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
67
+ var DOCUMENT_EXPORT_VERSION = 1;
68
+ function invalidHeader(message) {
69
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
70
+ }
71
+ function isRecord(value) {
72
+ return typeof value === "object" && value !== null && !Array.isArray(value);
73
+ }
74
+ function configOfHeader(line) {
75
+ if (line === void 0) invalidHeader("is missing");
76
+ if (!("text" in line)) {
77
+ invalidHeader(
78
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
79
+ );
80
+ }
81
+ let raw;
82
+ try {
83
+ raw = JSON.parse(line.text);
84
+ } catch {
85
+ invalidHeader("is not JSON");
86
+ }
87
+ if (!isRecord(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
88
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
89
+ }
90
+ const configuration = requireConfigurationShape(raw.index, "document export header");
91
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
92
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
93
+ return {
94
+ schema: configuration.schema,
95
+ language: configuration.language,
96
+ ...fields,
97
+ ...embedding === void 0 ? {} : {
98
+ embedding: {
99
+ fields: embedding.fields,
100
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
101
+ }
102
+ }
103
+ };
104
+ }
105
+ async function prepareIndex(engine, indexName, config) {
106
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
107
+ await engine.createIndex(indexName, config);
108
+ return;
109
+ }
110
+ const existing = flattenSchema(engine.getStats(indexName).schema);
111
+ const exported = flattenSchema(config.schema);
112
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
113
+ for (const field of fields) {
114
+ if (existing[field] === exported[field]) continue;
115
+ throw new NarsilError(
116
+ ErrorCodes.CONFIG_INVALID,
117
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
118
+ { indexName, field }
119
+ );
120
+ }
121
+ }
122
+ function highestReportedAt(refused) {
123
+ let highest = 0;
124
+ for (let at = 1; at < refused.length; at++) {
125
+ if (refused[at].line > refused[highest].line) highest = at;
126
+ }
127
+ return highest;
128
+ }
129
+ function refuse(state, line, code, message) {
130
+ state.result.refusedCount += 1;
131
+ const refused = state.result.refused;
132
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
133
+ refused.push({ line, code, message });
134
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
135
+ return;
136
+ }
137
+ if (line >= refused[state.highestReported].line) return;
138
+ refused[state.highestReported] = { line, code, message };
139
+ state.highestReported = highestReportedAt(refused);
140
+ }
141
+ async function flush(state) {
142
+ if (state.batch.length === 0) return;
143
+ const batch = state.batch;
144
+ state.batch = [];
145
+ state.batchIds = /* @__PURE__ */ new Set();
146
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
147
+ const inserts = [];
148
+ const updates = [];
149
+ for (const entry of batch) {
150
+ if (await state.engine.has(state.indexName, entry.docId))
151
+ updates.push({ docId: entry.docId, document: entry.document });
152
+ else inserts.push(entry.document);
153
+ }
154
+ for (const outcome of [
155
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
156
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
157
+ ]) {
158
+ if (outcome === null) continue;
159
+ state.result.imported += outcome.succeeded.length;
160
+ for (const failure of outcome.failed) {
161
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
162
+ refuse(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
163
+ }
164
+ }
165
+ }
166
+ async function takeLine(state, entry) {
167
+ if (!("text" in entry)) {
168
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
169
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
170
+ return;
171
+ }
172
+ if (entry.text.length === 0) return;
173
+ let document;
174
+ try {
175
+ document = JSON.parse(entry.text);
176
+ } catch {
177
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
178
+ return;
179
+ }
180
+ if (!isRecord(document) || typeof document.id !== "string" || document.id.length === 0) {
181
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
182
+ return;
183
+ }
184
+ try {
185
+ untagDocumentLine(entry.text, document);
186
+ } catch (error) {
187
+ if (!(error instanceof NarsilError)) throw error;
188
+ refuse(state, entry.line, error.code, error.message);
189
+ return;
190
+ }
191
+ if (state.batchIds.has(document.id)) await flush(state);
192
+ state.batch.push({ line: entry.line, docId: document.id, document });
193
+ state.batchIds.add(document.id);
194
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
195
+ }
196
+ async function readDocumentExport(engine, indexName, source) {
197
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
198
+ const header = await lines.next();
199
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
200
+ const state = {
201
+ engine,
202
+ indexName,
203
+ batch: [],
204
+ batchIds: /* @__PURE__ */ new Set(),
205
+ result: { imported: 0, refusedCount: 0, refused: [] },
206
+ highestReported: 0
207
+ };
208
+ for await (const entry of lines) await takeLine(state, entry);
209
+ await flush(state);
210
+ state.result.refused.sort((a, b) => a.line - b.line);
211
+ return state.result;
212
+ }
213
+
214
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport };
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
1
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
2
2
  import { existsSync } from 'fs';
3
3
  import { createRequire } from 'module';
4
4
  import { join, dirname } from 'path';