@delali/narsil 0.3.1-next.144 → 0.3.1-next.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/dist/adapters/filesystem.d.ts +1 -1
  2. package/dist/adapters/filesystem.mjs +101 -13
  3. package/dist/adapters/gcs.d.ts +49 -0
  4. package/dist/adapters/gcs.mjs +213 -0
  5. package/dist/adapters/indexeddb.d.ts +1 -1
  6. package/dist/adapters/indexeddb.mjs +2 -7
  7. package/dist/adapters/memory.d.ts +1 -1
  8. package/dist/adapters/memory.mjs +5 -0
  9. package/dist/adapters/s3.d.ts +53 -0
  10. package/dist/adapters/s3.mjs +263 -0
  11. package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
  12. package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
  13. package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
  14. package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
  15. package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
  16. package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
  17. package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
  18. package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
  19. package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
  20. package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
  21. package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
  22. package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
  23. package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
  24. package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
  25. package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
  26. package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
  27. package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
  28. package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
  29. package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
  30. package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
  31. package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
  32. package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
  33. package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
  34. package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
  35. package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
  36. package/dist/browser/import-FTMLTPGQ.mjs +9 -0
  37. package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
  38. package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
  39. package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
  40. package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
  41. package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
  42. package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
  43. package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
  44. package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
  45. package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
  46. package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
  47. package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
  48. package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
  49. package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
  50. package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
  51. package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
  52. package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
  53. package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
  54. package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
  55. package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
  56. package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
  57. package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
  58. package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
  59. package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
  60. package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
  61. package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
  62. package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
  63. package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
  64. package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
  65. package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
  66. package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
  67. package/dist/chunk-K5HSUPYS.mjs +11 -0
  68. package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
  69. package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
  70. package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
  71. package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
  72. package/dist/chunk-NWTEJ2T7.mjs +100 -0
  73. package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
  74. package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
  75. package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
  76. package/dist/chunk-PRID7CA6.mjs +104 -0
  77. package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
  78. package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
  79. package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
  80. package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
  81. package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
  82. package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
  83. package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
  84. package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
  85. package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
  86. package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
  87. package/dist/chunk-SVO6NV4D.mjs +248 -0
  88. package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
  89. package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
  90. package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
  91. package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
  92. package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
  93. package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
  94. package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
  95. package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
  96. package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
  97. package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
  98. package/dist/client.d.ts +8 -8
  99. package/dist/client.mjs +44 -0
  100. package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
  101. package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
  102. package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
  103. package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
  104. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  105. package/dist/distribution/coordinator/etcd.mjs +1 -1
  106. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  107. package/dist/distribution/transport/grpc.mjs +4 -4
  108. package/dist/distribution/transport/in-memory.mjs +2 -2
  109. package/dist/distribution/transport/tcp.mjs +3 -3
  110. package/dist/distribution.d.ts +8 -8
  111. package/dist/distribution.mjs +16219 -14535
  112. package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
  113. package/dist/embeddings/openai.d.ts +1 -1
  114. package/dist/embeddings/openai.mjs +2 -2
  115. package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
  116. package/dist/import-6KXFY6XI.mjs +35 -0
  117. package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
  118. package/dist/index.browser.mjs +4 -4
  119. package/dist/index.d.ts +7 -7
  120. package/dist/index.mjs +21 -30
  121. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  122. package/dist/invalidation/filesystem.d.ts +1 -1
  123. package/dist/invalidation/filesystem.mjs +1 -1
  124. package/dist/invalidation/noop.d.ts +1 -1
  125. package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
  126. package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
  127. package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
  128. package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
  129. package/dist/partial-words.mjs +7 -9
  130. package/dist/pattern.mjs +4 -6
  131. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  132. package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
  133. package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
  134. package/dist/react.d.ts +11 -11
  135. package/dist/react.mjs +9 -0
  136. package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
  137. package/dist/rerankers/http.d.ts +1 -1
  138. package/dist/rerankers/http.mjs +2 -2
  139. package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
  140. package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
  141. package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
  142. package/dist/server.d.ts +24 -12
  143. package/dist/server.mjs +153 -36
  144. package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
  145. package/dist/synonyms.mjs +3 -4
  146. package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
  147. package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
  148. package/dist/vector/search-worker.d.ts +2 -2
  149. package/dist/vector/search-worker.mjs +5 -6
  150. package/dist/workers/entry.mjs +7 -7
  151. package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
  152. package/openapi.json +349 -2
  153. package/package.json +27 -8
  154. package/dist/browser/import-TLB2A5LJ.mjs +0 -9
  155. package/dist/chunk-5WYVUPGD.mjs +0 -427
  156. package/dist/chunk-AV4IJKD2.mjs +0 -35
  157. package/dist/chunk-GES7U2PM.mjs +0 -68
  158. package/dist/chunk-KGOXHLLZ.mjs +0 -693
  159. package/dist/chunk-N37VMC62.mjs +0 -18
  160. package/dist/chunk-QSK35UC2.mjs +0 -35
  161. package/dist/chunk-WNBRW4Q2.mjs +0 -128
  162. package/dist/chunk-WQBCBKY3.mjs +0 -8
  163. package/dist/chunk-ZATRDAYU.mjs +0 -160
  164. package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
  165. package/dist/import-OOBUW3RR.mjs +0 -44
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
1
+ import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
2
2
 
3
3
  /**
4
4
  * Where {@link createFilesystemPersistence} writes its partitions.
@@ -1,6 +1,26 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-HNHCNTVG.mjs';
1
+ import { requireStreamLength, countedChunks } from '../chunk-PRID7CA6.mjs';
2
+ import '../chunk-U476IXKQ.mjs';
3
+ import { OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
4
+ import { NarsilError, ErrorCodes } from '../chunk-IQHZMCLX.mjs';
2
5
 
3
6
  // src/persistence/filesystem.ts
7
+ function unfinishedSaveName(basename) {
8
+ return `${basename}~tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
9
+ }
10
+ async function* readChunks(handle) {
11
+ try {
12
+ let remaining = (await handle.stat()).size;
13
+ while (remaining > 0) {
14
+ const buffer = new Uint8Array(Math.min(remaining, OBJECT_STORE_PART_BYTES));
15
+ const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, null);
16
+ if (bytesRead === 0) return;
17
+ remaining -= bytesRead;
18
+ yield bytesRead === buffer.byteLength ? buffer : buffer.subarray(0, bytesRead);
19
+ }
20
+ } finally {
21
+ await handle.close().catch(() => void 0);
22
+ }
23
+ }
4
24
  var cachedFs = null;
5
25
  var cachedPath = null;
6
26
  async function getFs() {
@@ -49,6 +69,25 @@ async function fsyncDirectory(dirPath, fs) {
49
69
  }
50
70
  }
51
71
  }
72
+ async function writeSyncedFile(fs, path, data, key) {
73
+ let handle = null;
74
+ try {
75
+ handle = await fs.open(path, "w");
76
+ await handle.write(data);
77
+ try {
78
+ await handle.sync();
79
+ } catch (err) {
80
+ throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for "${key}"`, {
81
+ key,
82
+ cause: err instanceof Error ? err.message : String(err)
83
+ });
84
+ }
85
+ await handle.close();
86
+ handle = null;
87
+ } finally {
88
+ if (handle !== null) await handle.close().catch(() => void 0);
89
+ }
90
+ }
52
91
  async function listDirectoryRecursive(dir, baseDir, pathMod, fs) {
53
92
  const results = [];
54
93
  let entries;
@@ -88,31 +127,80 @@ function createFilesystemPersistence(config) {
88
127
  const pathMod = await getPath();
89
128
  const dir = pathMod.dirname(resolvedPath);
90
129
  await fs.mkdir(dir, { recursive: true });
91
- const tempName = `${pathMod.basename(resolvedPath)}.tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
92
- const tempPath = pathMod.join(dir, tempName);
93
- let handle = null;
130
+ const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
94
131
  try {
95
- handle = await fs.open(tempPath, "w");
96
- await handle.write(data);
97
- try {
98
- await handle.sync();
99
- } catch (err) {
100
- throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for "${key}"`, {
132
+ await writeSyncedFile(fs, tempPath, data, key);
133
+ await fs.rename(tempPath, resolvedPath);
134
+ await fsyncDirectory(dir, fs);
135
+ } catch (err) {
136
+ await fs.unlink(tempPath).catch(() => void 0);
137
+ throw err;
138
+ }
139
+ },
140
+ async saveIfAbsent(key, data) {
141
+ const resolvedPath = await resolveAndGuard(baseDir, key);
142
+ const fs = await getFs();
143
+ const pathMod = await getPath();
144
+ const dir = pathMod.dirname(resolvedPath);
145
+ const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
146
+ let stored = false;
147
+ try {
148
+ await fs.mkdir(dir, { recursive: true });
149
+ await writeSyncedFile(fs, tempPath, data, key);
150
+ await fs.link(tempPath, resolvedPath);
151
+ stored = true;
152
+ } catch (err) {
153
+ if (err.code !== "EEXIST") {
154
+ await fs.unlink(tempPath).catch(() => void 0);
155
+ if (err instanceof NarsilError) throw err;
156
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
101
157
  key,
102
158
  cause: err instanceof Error ? err.message : String(err)
103
159
  });
104
160
  }
161
+ }
162
+ await fs.unlink(tempPath).catch(() => void 0);
163
+ await fsyncDirectory(dir, fs);
164
+ return stored;
165
+ },
166
+ async saveStream(key, data, length) {
167
+ requireStreamLength(key, length);
168
+ const resolvedPath = await resolveAndGuard(baseDir, key);
169
+ const fs = await getFs();
170
+ const pathMod = await getPath();
171
+ const dir = pathMod.dirname(resolvedPath);
172
+ await fs.mkdir(dir, { recursive: true });
173
+ const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
174
+ let handle = null;
175
+ try {
176
+ handle = await fs.open(tempPath, "w");
177
+ for await (const chunk of countedChunks(key, data, length)) await handle.write(chunk);
178
+ await handle.sync();
105
179
  await handle.close();
106
180
  handle = null;
107
181
  await fs.rename(tempPath, resolvedPath);
108
182
  await fsyncDirectory(dir, fs);
109
183
  } catch (err) {
110
- if (handle !== null) {
111
- await handle.close().catch(() => void 0);
112
- }
184
+ if (handle !== null) await handle.close().catch(() => void 0);
113
185
  await fs.unlink(tempPath).catch(() => void 0);
186
+ if (err instanceof NarsilError) throw err;
187
+ throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
188
+ key,
189
+ cause: err instanceof Error ? err.message : String(err)
190
+ });
191
+ }
192
+ },
193
+ async loadStream(key) {
194
+ const resolvedPath = await resolveAndGuard(baseDir, key);
195
+ const fs = await getFs();
196
+ let handle;
197
+ try {
198
+ handle = await fs.open(resolvedPath, "r");
199
+ } catch (err) {
200
+ if (err.code === "ENOENT") return null;
114
201
  throw err;
115
202
  }
203
+ return readChunks(handle);
116
204
  },
117
205
  async load(key) {
118
206
  const resolvedPath = await resolveAndGuard(baseDir, key);
@@ -0,0 +1,49 @@
1
+ import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
2
+
3
+ /**
4
+ * The service account key that {@link createGcsPersistence} signs its requests with.
5
+ *
6
+ * @public
7
+ */
8
+ interface GcsCredentials {
9
+ /** The service account's email address. */
10
+ client_email: string;
11
+ /** The service account's private key in PEM form. */
12
+ private_key: string;
13
+ }
14
+ /**
15
+ * Where {@link createGcsPersistence} stores its objects, and how it reaches Google Cloud Storage.
16
+ *
17
+ * @public
18
+ */
19
+ interface GcsPersistenceConfig {
20
+ /** The adapter writes every object to this bucket. */
21
+ bucket: string;
22
+ /** The project that owns the bucket, which the SDK otherwise reads from the environment. */
23
+ projectId?: string;
24
+ /** The path of a service account key file, which the SDK otherwise finds through its default credentials. */
25
+ keyFilename?: string;
26
+ /** A service account key held in memory, as an alternative to `keyFilename`. */
27
+ credentials?: GcsCredentials;
28
+ /** The address of a service that speaks the Cloud Storage JSON API, such as an emulator. */
29
+ apiEndpoint?: string;
30
+ /** The adapter stores every key under this prefix inside the bucket, so several stores can share one bucket. */
31
+ keyPrefix?: string;
32
+ }
33
+ /**
34
+ * Builds a persistence adapter that stores each key as one object in a Google Cloud Storage bucket.
35
+ *
36
+ * Cluster backups use it as the store outside the cluster. The adapter loads `@google-cloud/storage`, an optional
37
+ * peer dependency, when you call this function, checks each download against the checksum that Cloud Storage keeps,
38
+ * and refuses to load an object larger than one stored file can be.
39
+ *
40
+ * @param config - The bucket, the project, and the credentials.
41
+ * @returns An adapter you pass wherever Narsil takes a {@link PersistenceAdapter}.
42
+ * @throws A `NarsilError` with `PERSISTENCE_DEPENDENCY_MISSING` and the install command when the SDK is missing, and
43
+ * with `CONFIG_INVALID` when the bucket name is empty.
44
+ *
45
+ * @public
46
+ */
47
+ declare function createGcsPersistence(config: GcsPersistenceConfig): Promise<PersistenceAdapter>;
48
+
49
+ export { type GcsCredentials, type GcsPersistenceConfig, createGcsPersistence };
@@ -0,0 +1,213 @@
1
+ import { requireBucket, objectKeyPrefix, objectStoreFailure, requireObjectKey, requireStreamLength, countedChunks, readWhole, refuseOversizedObject, joinChunks } from '../chunk-PRID7CA6.mjs';
2
+ import { MAX_FILE_BYTES } from '../chunk-U476IXKQ.mjs';
3
+ import { OBJECT_STORE_LIST_PAGE_KEYS, OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
4
+ import { ErrorCodes, NarsilError } from '../chunk-IQHZMCLX.mjs';
5
+ import { Readable } from 'stream';
6
+ import { pipeline } from 'stream/promises';
7
+
8
+ var GCS_NOT_FOUND_STATUS = 404;
9
+ var GCS_PRECONDITION_FAILED_STATUS = 412;
10
+ var GCS_CONTENT_TYPE = "application/octet-stream";
11
+ async function loadGcsModule() {
12
+ try {
13
+ return await import('@google-cloud/storage');
14
+ } catch (err) {
15
+ throw new NarsilError(
16
+ ErrorCodes.PERSISTENCE_DEPENDENCY_MISSING,
17
+ "The `@google-cloud/storage` package is not installed. Install it with `pnpm add @google-cloud/storage` (or the npm or yarn equivalent) to use createGcsPersistence.",
18
+ { cause: err instanceof Error ? err.message : String(err) }
19
+ );
20
+ }
21
+ }
22
+ function statusOf(error) {
23
+ return typeof error === "object" && error !== null ? error.code : void 0;
24
+ }
25
+ function isNotFound(error) {
26
+ return statusOf(error) === GCS_NOT_FOUND_STATUS;
27
+ }
28
+ function isPreconditionFailed(error) {
29
+ return statusOf(error) === GCS_PRECONDITION_FAILED_STATUS;
30
+ }
31
+ function readObject(file, key) {
32
+ return new Promise((resolve, reject) => {
33
+ const chunks = [];
34
+ let total = 0;
35
+ let settled = false;
36
+ const stream = file.createReadStream();
37
+ stream.on("data", (chunk) => {
38
+ total += chunk.byteLength;
39
+ if (total > MAX_FILE_BYTES) {
40
+ stream.destroy();
41
+ if (!settled) {
42
+ settled = true;
43
+ try {
44
+ refuseOversizedObject(key, total);
45
+ } catch (error) {
46
+ reject(error);
47
+ }
48
+ }
49
+ return;
50
+ }
51
+ chunks.push(chunk);
52
+ });
53
+ stream.on("error", (error) => {
54
+ if (settled) return;
55
+ settled = true;
56
+ if (isNotFound(error)) resolve(null);
57
+ else reject(objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error));
58
+ });
59
+ stream.on("end", () => {
60
+ if (settled) return;
61
+ settled = true;
62
+ resolve(joinChunks(chunks, total));
63
+ });
64
+ });
65
+ }
66
+ async function* remainingChunks(key, first, rest, stream) {
67
+ let total = first.byteLength;
68
+ try {
69
+ refuseOversizedObject(key, total);
70
+ yield first;
71
+ for (; ; ) {
72
+ const step = await rest.next();
73
+ if (step.done === true) return;
74
+ total += step.value.byteLength;
75
+ refuseOversizedObject(key, total);
76
+ yield step.value;
77
+ }
78
+ } catch (error) {
79
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
80
+ } finally {
81
+ stream.destroy();
82
+ }
83
+ }
84
+ async function* chunksOf(chunks) {
85
+ yield* chunks;
86
+ }
87
+ async function openObjectStream(file, key) {
88
+ const stream = file.createReadStream();
89
+ const chunks = stream[Symbol.asyncIterator]();
90
+ let first;
91
+ try {
92
+ first = await chunks.next();
93
+ } catch (error) {
94
+ stream.destroy();
95
+ if (isNotFound(error)) return null;
96
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
97
+ }
98
+ if (first.done === true) {
99
+ stream.destroy();
100
+ return chunksOf([]);
101
+ }
102
+ return remainingChunks(key, first.value, chunks, stream);
103
+ }
104
+ async function createGcsPersistence(config) {
105
+ const bucketName = requireBucket(config.bucket, "Google Cloud Storage");
106
+ const root = objectKeyPrefix(config.keyPrefix, "Google Cloud Storage");
107
+ const sdk = await loadGcsModule();
108
+ const storage = new sdk.Storage({
109
+ ...config.projectId === void 0 ? {} : { projectId: config.projectId },
110
+ ...config.keyFilename === void 0 ? {} : { keyFilename: config.keyFilename },
111
+ ...config.credentials === void 0 ? {} : { credentials: config.credentials },
112
+ ...config.apiEndpoint === void 0 ? {} : { apiEndpoint: config.apiEndpoint },
113
+ retryOptions: { idempotencyStrategy: sdk.IdempotencyStrategy.RetryAlways }
114
+ });
115
+ const bucket = storage.bucket(bucketName);
116
+ return {
117
+ async save(key, data) {
118
+ requireObjectKey(key);
119
+ try {
120
+ await bucket.file(`${root}${key}`).save(data, { resumable: false, contentType: GCS_CONTENT_TYPE });
121
+ } catch (error) {
122
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
123
+ }
124
+ },
125
+ async saveIfAbsent(key, data) {
126
+ requireObjectKey(key);
127
+ try {
128
+ await bucket.file(`${root}${key}`).save(data, {
129
+ resumable: false,
130
+ contentType: GCS_CONTENT_TYPE,
131
+ preconditionOpts: { ifGenerationMatch: 0 }
132
+ });
133
+ return true;
134
+ } catch (error) {
135
+ if (isPreconditionFailed(error)) return false;
136
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
137
+ }
138
+ },
139
+ async load(key) {
140
+ requireObjectKey(key);
141
+ return readObject(bucket.file(`${root}${key}`), key);
142
+ },
143
+ async saveStream(key, data, length) {
144
+ requireObjectKey(key);
145
+ try {
146
+ requireStreamLength(key, length);
147
+ const counted = countedChunks(key, data, length);
148
+ const file = bucket.file(`${root}${key}`);
149
+ if (length <= OBJECT_STORE_PART_BYTES) {
150
+ await file.save(await readWhole(counted, length), { resumable: false, contentType: GCS_CONTENT_TYPE });
151
+ return;
152
+ }
153
+ await pipeline(
154
+ Readable.from(counted, { objectMode: false }),
155
+ file.createWriteStream({
156
+ resumable: true,
157
+ chunkSize: OBJECT_STORE_PART_BYTES,
158
+ contentType: GCS_CONTENT_TYPE
159
+ })
160
+ );
161
+ } catch (error) {
162
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
163
+ }
164
+ },
165
+ async loadStream(key) {
166
+ requireObjectKey(key);
167
+ return openObjectStream(bucket.file(`${root}${key}`), key);
168
+ },
169
+ async delete(key) {
170
+ requireObjectKey(key);
171
+ const file = bucket.file(`${root}${key}`);
172
+ try {
173
+ const [metadata] = await file.getMetadata();
174
+ const generation = metadata.generation;
175
+ if (generation === void 0) {
176
+ throw new NarsilError(
177
+ ErrorCodes.PERSISTENCE_DELETE_FAILED,
178
+ `Cloud Storage reported no generation for "${key}", so the adapter deletes nothing`,
179
+ { key }
180
+ );
181
+ }
182
+ await file.delete({ ignoreNotFound: true, ifGenerationMatch: generation });
183
+ } catch (error) {
184
+ if (isNotFound(error) || isPreconditionFailed(error)) return;
185
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key, error);
186
+ }
187
+ },
188
+ async list(prefix) {
189
+ const keys = [];
190
+ let pageToken;
191
+ try {
192
+ do {
193
+ const [files, next] = await bucket.getFiles({
194
+ prefix: `${root}${prefix}`,
195
+ autoPaginate: false,
196
+ maxResults: OBJECT_STORE_LIST_PAGE_KEYS,
197
+ ...pageToken === void 0 ? {} : { pageToken }
198
+ });
199
+ for (const file of files) {
200
+ if (file.name.startsWith(root)) keys.push(file.name.slice(root.length));
201
+ }
202
+ const token = next !== null && typeof next === "object" && "pageToken" in next ? next.pageToken : void 0;
203
+ pageToken = typeof token === "string" && token.length > 0 ? token : void 0;
204
+ } while (pageToken !== void 0);
205
+ } catch (error) {
206
+ throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "list the keys under", prefix, error);
207
+ }
208
+ return keys;
209
+ }
210
+ };
211
+ }
212
+
213
+ export { createGcsPersistence };
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
1
+ import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
2
2
 
3
3
  /**
4
4
  * Which IndexedDB database and object store
@@ -1,10 +1,5 @@
1
- import { NarsilError, ErrorCodes } from '../chunk-HNHCNTVG.mjs';
2
-
3
- // src/persistence/constants.ts
4
- var DEFAULT_INDEXEDDB_NAME = "narsil";
5
- var DEFAULT_INDEXEDDB_STORE_NAME = "partitions";
6
- var MAX_PERSISTENCE_KEY_LENGTH = 1024;
7
- var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
1
+ import { DEFAULT_INDEXEDDB_NAME, DEFAULT_INDEXEDDB_STORE_NAME, INDEXEDDB_BLOCKED_TIMEOUT_MS, MAX_PERSISTENCE_KEY_LENGTH } from '../chunk-K5HSUPYS.mjs';
2
+ import { NarsilError, ErrorCodes } from '../chunk-IQHZMCLX.mjs';
8
3
 
9
4
  // src/persistence/indexeddb.ts
10
5
  function validateKey(key) {
@@ -1,4 +1,4 @@
1
- import { P as PersistenceAdapter } from '../adapters-BQFQ4PRz.js';
1
+ import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
2
2
 
3
3
  /**
4
4
  * Builds a persistence adapter that keeps every partition in a map, which is
@@ -5,6 +5,11 @@ function createMemoryPersistence() {
5
5
  async save(key, data) {
6
6
  store.set(key, new Uint8Array(data));
7
7
  },
8
+ async saveIfAbsent(key, data) {
9
+ if (store.has(key)) return false;
10
+ store.set(key, new Uint8Array(data));
11
+ return true;
12
+ },
8
13
  async load(key) {
9
14
  const stored = store.get(key);
10
15
  if (stored === void 0) {
@@ -0,0 +1,53 @@
1
+ import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
2
+
3
+ /**
4
+ * The access keys {@link createS3Persistence} signs its requests with.
5
+ *
6
+ * @public
7
+ */
8
+ interface S3Credentials {
9
+ /** The access key ID. */
10
+ accessKeyId: string;
11
+ /** The secret access key. */
12
+ secretAccessKey: string;
13
+ /** The session token of temporary credentials, which long-lived keys leave out. */
14
+ sessionToken?: string;
15
+ }
16
+ /**
17
+ * Where {@link createS3Persistence} stores its objects, and how it reaches the store.
18
+ *
19
+ * @public
20
+ */
21
+ interface S3PersistenceConfig {
22
+ /** The adapter writes every object to this bucket. */
23
+ bucket: string;
24
+ /** The bucket's region, which the SDK otherwise reads from the environment. */
25
+ region?: string;
26
+ /** The address of an S3-compatible service, which you leave out for Amazon S3 itself. */
27
+ endpoint?: string;
28
+ /** Puts the bucket name in the path of each request, which most S3-compatible services need. */
29
+ forcePathStyle?: boolean;
30
+ /** The keys to sign with, which the SDK otherwise reads from its default credential chain. */
31
+ credentials?: S3Credentials;
32
+ /** The adapter stores every key under this prefix inside the bucket, so several stores can share one bucket. */
33
+ keyPrefix?: string;
34
+ /** The SDK tries each request this many times before it reports the failure. */
35
+ maxAttempts?: number;
36
+ }
37
+ /**
38
+ * Builds a persistence adapter that stores each key as one object in an S3 bucket, or in any service that speaks
39
+ * the S3 protocol.
40
+ *
41
+ * Cluster backups use it as the store outside the cluster. The adapter loads `@aws-sdk/client-s3`, an optional
42
+ * peer dependency, when you call this function, and it refuses to load an object larger than one stored file can be.
43
+ *
44
+ * @param config - The bucket, the region or endpoint, and the credentials.
45
+ * @returns An adapter you pass wherever Narsil takes a {@link PersistenceAdapter}.
46
+ * @throws A `NarsilError` with `PERSISTENCE_DEPENDENCY_MISSING` and the install command when the SDK is missing, and
47
+ * with `CONFIG_INVALID` when the bucket name is empty.
48
+ *
49
+ * @public
50
+ */
51
+ declare function createS3Persistence(config: S3PersistenceConfig): Promise<PersistenceAdapter>;
52
+
53
+ export { type S3Credentials, type S3PersistenceConfig, createS3Persistence };