@delali/narsil 0.3.1-next.143 → 0.3.1-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
- package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
- package/dist/browser/import-FTMLTPGQ.mjs +9 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
- package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
- package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
- package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
- package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
- package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
- package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
- package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
- package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
- package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
- package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
- package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
- package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
- package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
- package/dist/chunk-NWTEJ2T7.mjs +100 -0
- package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
- package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/chunk-PRID7CA6.mjs +104 -0
- package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
- package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
- package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
- package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
- package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
- package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
- package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
- package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
- package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
- package/dist/chunk-SVO6NV4D.mjs +248 -0
- package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
- package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
- package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
- package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
- package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
- package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
- package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +44 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
- package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
- package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16219 -14535
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
- package/dist/import-6KXFY6XI.mjs +35 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -30
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
- package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
- package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +9 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
- package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +153 -36
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
- package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +5 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
- package/openapi.json +349 -2
- package/package.json +27 -8
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-KGOXHLLZ.mjs +0 -693
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-QSK35UC2.mjs +0 -35
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
|
@@ -1,6 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { requireStreamLength, countedChunks } from '../chunk-PRID7CA6.mjs';
|
|
2
|
+
import '../chunk-U476IXKQ.mjs';
|
|
3
|
+
import { OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from '../chunk-IQHZMCLX.mjs';
|
|
2
5
|
|
|
3
6
|
// src/persistence/filesystem.ts
|
|
7
|
+
function unfinishedSaveName(basename) {
|
|
8
|
+
return `${basename}~tmp-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
|
9
|
+
}
|
|
10
|
+
async function* readChunks(handle) {
|
|
11
|
+
try {
|
|
12
|
+
let remaining = (await handle.stat()).size;
|
|
13
|
+
while (remaining > 0) {
|
|
14
|
+
const buffer = new Uint8Array(Math.min(remaining, OBJECT_STORE_PART_BYTES));
|
|
15
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.byteLength, null);
|
|
16
|
+
if (bytesRead === 0) return;
|
|
17
|
+
remaining -= bytesRead;
|
|
18
|
+
yield bytesRead === buffer.byteLength ? buffer : buffer.subarray(0, bytesRead);
|
|
19
|
+
}
|
|
20
|
+
} finally {
|
|
21
|
+
await handle.close().catch(() => void 0);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
4
24
|
var cachedFs = null;
|
|
5
25
|
var cachedPath = null;
|
|
6
26
|
async function getFs() {
|
|
@@ -49,6 +69,25 @@ async function fsyncDirectory(dirPath, fs) {
|
|
|
49
69
|
}
|
|
50
70
|
}
|
|
51
71
|
}
|
|
72
|
+
async function writeSyncedFile(fs, path, data, key) {
|
|
73
|
+
let handle = null;
|
|
74
|
+
try {
|
|
75
|
+
handle = await fs.open(path, "w");
|
|
76
|
+
await handle.write(data);
|
|
77
|
+
try {
|
|
78
|
+
await handle.sync();
|
|
79
|
+
} catch (err) {
|
|
80
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_FSYNC_FAILED, `fsync failed for "${key}"`, {
|
|
81
|
+
key,
|
|
82
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
await handle.close();
|
|
86
|
+
handle = null;
|
|
87
|
+
} finally {
|
|
88
|
+
if (handle !== null) await handle.close().catch(() => void 0);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
52
91
|
async function listDirectoryRecursive(dir, baseDir, pathMod, fs) {
|
|
53
92
|
const results = [];
|
|
54
93
|
let entries;
|
|
@@ -88,31 +127,80 @@ function createFilesystemPersistence(config) {
|
|
|
88
127
|
const pathMod = await getPath();
|
|
89
128
|
const dir = pathMod.dirname(resolvedPath);
|
|
90
129
|
await fs.mkdir(dir, { recursive: true });
|
|
91
|
-
const
|
|
92
|
-
const tempPath = pathMod.join(dir, tempName);
|
|
93
|
-
let handle = null;
|
|
130
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
94
131
|
try {
|
|
95
|
-
|
|
96
|
-
await
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
132
|
+
await writeSyncedFile(fs, tempPath, data, key);
|
|
133
|
+
await fs.rename(tempPath, resolvedPath);
|
|
134
|
+
await fsyncDirectory(dir, fs);
|
|
135
|
+
} catch (err) {
|
|
136
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
async saveIfAbsent(key, data) {
|
|
141
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
142
|
+
const fs = await getFs();
|
|
143
|
+
const pathMod = await getPath();
|
|
144
|
+
const dir = pathMod.dirname(resolvedPath);
|
|
145
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
146
|
+
let stored = false;
|
|
147
|
+
try {
|
|
148
|
+
await fs.mkdir(dir, { recursive: true });
|
|
149
|
+
await writeSyncedFile(fs, tempPath, data, key);
|
|
150
|
+
await fs.link(tempPath, resolvedPath);
|
|
151
|
+
stored = true;
|
|
152
|
+
} catch (err) {
|
|
153
|
+
if (err.code !== "EEXIST") {
|
|
154
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
155
|
+
if (err instanceof NarsilError) throw err;
|
|
156
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
|
|
101
157
|
key,
|
|
102
158
|
cause: err instanceof Error ? err.message : String(err)
|
|
103
159
|
});
|
|
104
160
|
}
|
|
161
|
+
}
|
|
162
|
+
await fs.unlink(tempPath).catch(() => void 0);
|
|
163
|
+
await fsyncDirectory(dir, fs);
|
|
164
|
+
return stored;
|
|
165
|
+
},
|
|
166
|
+
async saveStream(key, data, length) {
|
|
167
|
+
requireStreamLength(key, length);
|
|
168
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
169
|
+
const fs = await getFs();
|
|
170
|
+
const pathMod = await getPath();
|
|
171
|
+
const dir = pathMod.dirname(resolvedPath);
|
|
172
|
+
await fs.mkdir(dir, { recursive: true });
|
|
173
|
+
const tempPath = pathMod.join(dir, unfinishedSaveName(pathMod.basename(resolvedPath)));
|
|
174
|
+
let handle = null;
|
|
175
|
+
try {
|
|
176
|
+
handle = await fs.open(tempPath, "w");
|
|
177
|
+
for await (const chunk of countedChunks(key, data, length)) await handle.write(chunk);
|
|
178
|
+
await handle.sync();
|
|
105
179
|
await handle.close();
|
|
106
180
|
handle = null;
|
|
107
181
|
await fs.rename(tempPath, resolvedPath);
|
|
108
182
|
await fsyncDirectory(dir, fs);
|
|
109
183
|
} catch (err) {
|
|
110
|
-
if (handle !== null)
|
|
111
|
-
await handle.close().catch(() => void 0);
|
|
112
|
-
}
|
|
184
|
+
if (handle !== null) await handle.close().catch(() => void 0);
|
|
113
185
|
await fs.unlink(tempPath).catch(() => void 0);
|
|
186
|
+
if (err instanceof NarsilError) throw err;
|
|
187
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, `The adapter could not save "${key}"`, {
|
|
188
|
+
key,
|
|
189
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async loadStream(key) {
|
|
194
|
+
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
195
|
+
const fs = await getFs();
|
|
196
|
+
let handle;
|
|
197
|
+
try {
|
|
198
|
+
handle = await fs.open(resolvedPath, "r");
|
|
199
|
+
} catch (err) {
|
|
200
|
+
if (err.code === "ENOENT") return null;
|
|
114
201
|
throw err;
|
|
115
202
|
}
|
|
203
|
+
return readChunks(handle);
|
|
116
204
|
},
|
|
117
205
|
async load(key) {
|
|
118
206
|
const resolvedPath = await resolveAndGuard(baseDir, key);
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The service account key that {@link createGcsPersistence} signs its requests with.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface GcsCredentials {
|
|
9
|
+
/** The service account's email address. */
|
|
10
|
+
client_email: string;
|
|
11
|
+
/** The service account's private key in PEM form. */
|
|
12
|
+
private_key: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Where {@link createGcsPersistence} stores its objects, and how it reaches Google Cloud Storage.
|
|
16
|
+
*
|
|
17
|
+
* @public
|
|
18
|
+
*/
|
|
19
|
+
interface GcsPersistenceConfig {
|
|
20
|
+
/** The adapter writes every object to this bucket. */
|
|
21
|
+
bucket: string;
|
|
22
|
+
/** The project that owns the bucket, which the SDK otherwise reads from the environment. */
|
|
23
|
+
projectId?: string;
|
|
24
|
+
/** The path of a service account key file, which the SDK otherwise finds through its default credentials. */
|
|
25
|
+
keyFilename?: string;
|
|
26
|
+
/** A service account key held in memory, as an alternative to `keyFilename`. */
|
|
27
|
+
credentials?: GcsCredentials;
|
|
28
|
+
/** The address of a service that speaks the Cloud Storage JSON API, such as an emulator. */
|
|
29
|
+
apiEndpoint?: string;
|
|
30
|
+
/** The adapter stores every key under this prefix inside the bucket, so several stores can share one bucket. */
|
|
31
|
+
keyPrefix?: string;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Builds a persistence adapter that stores each key as one object in a Google Cloud Storage bucket.
|
|
35
|
+
*
|
|
36
|
+
* Cluster backups use it as the store outside the cluster. The adapter loads `@google-cloud/storage`, an optional
|
|
37
|
+
* peer dependency, when you call this function, checks each download against the checksum that Cloud Storage keeps,
|
|
38
|
+
* and refuses to load an object larger than one stored file can be.
|
|
39
|
+
*
|
|
40
|
+
* @param config - The bucket, the project, and the credentials.
|
|
41
|
+
* @returns An adapter you pass wherever Narsil takes a {@link PersistenceAdapter}.
|
|
42
|
+
* @throws A `NarsilError` with `PERSISTENCE_DEPENDENCY_MISSING` and the install command when the SDK is missing, and
|
|
43
|
+
* with `CONFIG_INVALID` when the bucket name is empty.
|
|
44
|
+
*
|
|
45
|
+
* @public
|
|
46
|
+
*/
|
|
47
|
+
declare function createGcsPersistence(config: GcsPersistenceConfig): Promise<PersistenceAdapter>;
|
|
48
|
+
|
|
49
|
+
export { type GcsCredentials, type GcsPersistenceConfig, createGcsPersistence };
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { requireBucket, objectKeyPrefix, objectStoreFailure, requireObjectKey, requireStreamLength, countedChunks, readWhole, refuseOversizedObject, joinChunks } from '../chunk-PRID7CA6.mjs';
|
|
2
|
+
import { MAX_FILE_BYTES } from '../chunk-U476IXKQ.mjs';
|
|
3
|
+
import { OBJECT_STORE_LIST_PAGE_KEYS, OBJECT_STORE_PART_BYTES } from '../chunk-K5HSUPYS.mjs';
|
|
4
|
+
import { ErrorCodes, NarsilError } from '../chunk-IQHZMCLX.mjs';
|
|
5
|
+
import { Readable } from 'stream';
|
|
6
|
+
import { pipeline } from 'stream/promises';
|
|
7
|
+
|
|
8
|
+
var GCS_NOT_FOUND_STATUS = 404;
|
|
9
|
+
var GCS_PRECONDITION_FAILED_STATUS = 412;
|
|
10
|
+
var GCS_CONTENT_TYPE = "application/octet-stream";
|
|
11
|
+
async function loadGcsModule() {
|
|
12
|
+
try {
|
|
13
|
+
return await import('@google-cloud/storage');
|
|
14
|
+
} catch (err) {
|
|
15
|
+
throw new NarsilError(
|
|
16
|
+
ErrorCodes.PERSISTENCE_DEPENDENCY_MISSING,
|
|
17
|
+
"The `@google-cloud/storage` package is not installed. Install it with `pnpm add @google-cloud/storage` (or the npm or yarn equivalent) to use createGcsPersistence.",
|
|
18
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function statusOf(error) {
|
|
23
|
+
return typeof error === "object" && error !== null ? error.code : void 0;
|
|
24
|
+
}
|
|
25
|
+
function isNotFound(error) {
|
|
26
|
+
return statusOf(error) === GCS_NOT_FOUND_STATUS;
|
|
27
|
+
}
|
|
28
|
+
function isPreconditionFailed(error) {
|
|
29
|
+
return statusOf(error) === GCS_PRECONDITION_FAILED_STATUS;
|
|
30
|
+
}
|
|
31
|
+
function readObject(file, key) {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
const chunks = [];
|
|
34
|
+
let total = 0;
|
|
35
|
+
let settled = false;
|
|
36
|
+
const stream = file.createReadStream();
|
|
37
|
+
stream.on("data", (chunk) => {
|
|
38
|
+
total += chunk.byteLength;
|
|
39
|
+
if (total > MAX_FILE_BYTES) {
|
|
40
|
+
stream.destroy();
|
|
41
|
+
if (!settled) {
|
|
42
|
+
settled = true;
|
|
43
|
+
try {
|
|
44
|
+
refuseOversizedObject(key, total);
|
|
45
|
+
} catch (error) {
|
|
46
|
+
reject(error);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
chunks.push(chunk);
|
|
52
|
+
});
|
|
53
|
+
stream.on("error", (error) => {
|
|
54
|
+
if (settled) return;
|
|
55
|
+
settled = true;
|
|
56
|
+
if (isNotFound(error)) resolve(null);
|
|
57
|
+
else reject(objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error));
|
|
58
|
+
});
|
|
59
|
+
stream.on("end", () => {
|
|
60
|
+
if (settled) return;
|
|
61
|
+
settled = true;
|
|
62
|
+
resolve(joinChunks(chunks, total));
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function* remainingChunks(key, first, rest, stream) {
|
|
67
|
+
let total = first.byteLength;
|
|
68
|
+
try {
|
|
69
|
+
refuseOversizedObject(key, total);
|
|
70
|
+
yield first;
|
|
71
|
+
for (; ; ) {
|
|
72
|
+
const step = await rest.next();
|
|
73
|
+
if (step.done === true) return;
|
|
74
|
+
total += step.value.byteLength;
|
|
75
|
+
refuseOversizedObject(key, total);
|
|
76
|
+
yield step.value;
|
|
77
|
+
}
|
|
78
|
+
} catch (error) {
|
|
79
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
80
|
+
} finally {
|
|
81
|
+
stream.destroy();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async function* chunksOf(chunks) {
|
|
85
|
+
yield* chunks;
|
|
86
|
+
}
|
|
87
|
+
async function openObjectStream(file, key) {
|
|
88
|
+
const stream = file.createReadStream();
|
|
89
|
+
const chunks = stream[Symbol.asyncIterator]();
|
|
90
|
+
let first;
|
|
91
|
+
try {
|
|
92
|
+
first = await chunks.next();
|
|
93
|
+
} catch (error) {
|
|
94
|
+
stream.destroy();
|
|
95
|
+
if (isNotFound(error)) return null;
|
|
96
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key, error);
|
|
97
|
+
}
|
|
98
|
+
if (first.done === true) {
|
|
99
|
+
stream.destroy();
|
|
100
|
+
return chunksOf([]);
|
|
101
|
+
}
|
|
102
|
+
return remainingChunks(key, first.value, chunks, stream);
|
|
103
|
+
}
|
|
104
|
+
async function createGcsPersistence(config) {
|
|
105
|
+
const bucketName = requireBucket(config.bucket, "Google Cloud Storage");
|
|
106
|
+
const root = objectKeyPrefix(config.keyPrefix, "Google Cloud Storage");
|
|
107
|
+
const sdk = await loadGcsModule();
|
|
108
|
+
const storage = new sdk.Storage({
|
|
109
|
+
...config.projectId === void 0 ? {} : { projectId: config.projectId },
|
|
110
|
+
...config.keyFilename === void 0 ? {} : { keyFilename: config.keyFilename },
|
|
111
|
+
...config.credentials === void 0 ? {} : { credentials: config.credentials },
|
|
112
|
+
...config.apiEndpoint === void 0 ? {} : { apiEndpoint: config.apiEndpoint },
|
|
113
|
+
retryOptions: { idempotencyStrategy: sdk.IdempotencyStrategy.RetryAlways }
|
|
114
|
+
});
|
|
115
|
+
const bucket = storage.bucket(bucketName);
|
|
116
|
+
return {
|
|
117
|
+
async save(key, data) {
|
|
118
|
+
requireObjectKey(key);
|
|
119
|
+
try {
|
|
120
|
+
await bucket.file(`${root}${key}`).save(data, { resumable: false, contentType: GCS_CONTENT_TYPE });
|
|
121
|
+
} catch (error) {
|
|
122
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
async saveIfAbsent(key, data) {
|
|
126
|
+
requireObjectKey(key);
|
|
127
|
+
try {
|
|
128
|
+
await bucket.file(`${root}${key}`).save(data, {
|
|
129
|
+
resumable: false,
|
|
130
|
+
contentType: GCS_CONTENT_TYPE,
|
|
131
|
+
preconditionOpts: { ifGenerationMatch: 0 }
|
|
132
|
+
});
|
|
133
|
+
return true;
|
|
134
|
+
} catch (error) {
|
|
135
|
+
if (isPreconditionFailed(error)) return false;
|
|
136
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
async load(key) {
|
|
140
|
+
requireObjectKey(key);
|
|
141
|
+
return readObject(bucket.file(`${root}${key}`), key);
|
|
142
|
+
},
|
|
143
|
+
async saveStream(key, data, length) {
|
|
144
|
+
requireObjectKey(key);
|
|
145
|
+
try {
|
|
146
|
+
requireStreamLength(key, length);
|
|
147
|
+
const counted = countedChunks(key, data, length);
|
|
148
|
+
const file = bucket.file(`${root}${key}`);
|
|
149
|
+
if (length <= OBJECT_STORE_PART_BYTES) {
|
|
150
|
+
await file.save(await readWhole(counted, length), { resumable: false, contentType: GCS_CONTENT_TYPE });
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
await pipeline(
|
|
154
|
+
Readable.from(counted, { objectMode: false }),
|
|
155
|
+
file.createWriteStream({
|
|
156
|
+
resumable: true,
|
|
157
|
+
chunkSize: OBJECT_STORE_PART_BYTES,
|
|
158
|
+
contentType: GCS_CONTENT_TYPE
|
|
159
|
+
})
|
|
160
|
+
);
|
|
161
|
+
} catch (error) {
|
|
162
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key, error);
|
|
163
|
+
}
|
|
164
|
+
},
|
|
165
|
+
async loadStream(key) {
|
|
166
|
+
requireObjectKey(key);
|
|
167
|
+
return openObjectStream(bucket.file(`${root}${key}`), key);
|
|
168
|
+
},
|
|
169
|
+
async delete(key) {
|
|
170
|
+
requireObjectKey(key);
|
|
171
|
+
const file = bucket.file(`${root}${key}`);
|
|
172
|
+
try {
|
|
173
|
+
const [metadata] = await file.getMetadata();
|
|
174
|
+
const generation = metadata.generation;
|
|
175
|
+
if (generation === void 0) {
|
|
176
|
+
throw new NarsilError(
|
|
177
|
+
ErrorCodes.PERSISTENCE_DELETE_FAILED,
|
|
178
|
+
`Cloud Storage reported no generation for "${key}", so the adapter deletes nothing`,
|
|
179
|
+
{ key }
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
await file.delete({ ignoreNotFound: true, ifGenerationMatch: generation });
|
|
183
|
+
} catch (error) {
|
|
184
|
+
if (isNotFound(error) || isPreconditionFailed(error)) return;
|
|
185
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key, error);
|
|
186
|
+
}
|
|
187
|
+
},
|
|
188
|
+
async list(prefix) {
|
|
189
|
+
const keys = [];
|
|
190
|
+
let pageToken;
|
|
191
|
+
try {
|
|
192
|
+
do {
|
|
193
|
+
const [files, next] = await bucket.getFiles({
|
|
194
|
+
prefix: `${root}${prefix}`,
|
|
195
|
+
autoPaginate: false,
|
|
196
|
+
maxResults: OBJECT_STORE_LIST_PAGE_KEYS,
|
|
197
|
+
...pageToken === void 0 ? {} : { pageToken }
|
|
198
|
+
});
|
|
199
|
+
for (const file of files) {
|
|
200
|
+
if (file.name.startsWith(root)) keys.push(file.name.slice(root.length));
|
|
201
|
+
}
|
|
202
|
+
const token = next !== null && typeof next === "object" && "pageToken" in next ? next.pageToken : void 0;
|
|
203
|
+
pageToken = typeof token === "string" && token.length > 0 ? token : void 0;
|
|
204
|
+
} while (pageToken !== void 0);
|
|
205
|
+
} catch (error) {
|
|
206
|
+
throw objectStoreFailure(ErrorCodes.PERSISTENCE_LOAD_FAILED, "list the keys under", prefix, error);
|
|
207
|
+
}
|
|
208
|
+
return keys;
|
|
209
|
+
}
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export { createGcsPersistence };
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
// src/persistence/constants.ts
|
|
4
|
-
var DEFAULT_INDEXEDDB_NAME = "narsil";
|
|
5
|
-
var DEFAULT_INDEXEDDB_STORE_NAME = "partitions";
|
|
6
|
-
var MAX_PERSISTENCE_KEY_LENGTH = 1024;
|
|
7
|
-
var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
|
|
1
|
+
import { DEFAULT_INDEXEDDB_NAME, DEFAULT_INDEXEDDB_STORE_NAME, INDEXEDDB_BLOCKED_TIMEOUT_MS, MAX_PERSISTENCE_KEY_LENGTH } from '../chunk-K5HSUPYS.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from '../chunk-IQHZMCLX.mjs';
|
|
8
3
|
|
|
9
4
|
// src/persistence/indexeddb.ts
|
|
10
5
|
function validateKey(key) {
|
package/dist/adapters/memory.mjs
CHANGED
|
@@ -5,6 +5,11 @@ function createMemoryPersistence() {
|
|
|
5
5
|
async save(key, data) {
|
|
6
6
|
store.set(key, new Uint8Array(data));
|
|
7
7
|
},
|
|
8
|
+
async saveIfAbsent(key, data) {
|
|
9
|
+
if (store.has(key)) return false;
|
|
10
|
+
store.set(key, new Uint8Array(data));
|
|
11
|
+
return true;
|
|
12
|
+
},
|
|
8
13
|
async load(key) {
|
|
9
14
|
const stored = store.get(key);
|
|
10
15
|
if (stored === void 0) {
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { P as PersistenceAdapter } from '../adapters-pljAqmrC.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The access keys {@link createS3Persistence} signs its requests with.
|
|
5
|
+
*
|
|
6
|
+
* @public
|
|
7
|
+
*/
|
|
8
|
+
interface S3Credentials {
|
|
9
|
+
/** The access key ID. */
|
|
10
|
+
accessKeyId: string;
|
|
11
|
+
/** The secret access key. */
|
|
12
|
+
secretAccessKey: string;
|
|
13
|
+
/** The session token of temporary credentials, which long-lived keys leave out. */
|
|
14
|
+
sessionToken?: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Where {@link createS3Persistence} stores its objects, and how it reaches the store.
|
|
18
|
+
*
|
|
19
|
+
* @public
|
|
20
|
+
*/
|
|
21
|
+
interface S3PersistenceConfig {
|
|
22
|
+
/** The adapter writes every object to this bucket. */
|
|
23
|
+
bucket: string;
|
|
24
|
+
/** The bucket's region, which the SDK otherwise reads from the environment. */
|
|
25
|
+
region?: string;
|
|
26
|
+
/** The address of an S3-compatible service, which you leave out for Amazon S3 itself. */
|
|
27
|
+
endpoint?: string;
|
|
28
|
+
/** Puts the bucket name in the path of each request, which most S3-compatible services need. */
|
|
29
|
+
forcePathStyle?: boolean;
|
|
30
|
+
/** The keys to sign with, which the SDK otherwise reads from its default credential chain. */
|
|
31
|
+
credentials?: S3Credentials;
|
|
32
|
+
/** The adapter stores every key under this prefix inside the bucket, so several stores can share one bucket. */
|
|
33
|
+
keyPrefix?: string;
|
|
34
|
+
/** The SDK tries each request this many times before it reports the failure. */
|
|
35
|
+
maxAttempts?: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Builds a persistence adapter that stores each key as one object in an S3 bucket, or in any service that speaks
|
|
39
|
+
* the S3 protocol.
|
|
40
|
+
*
|
|
41
|
+
* Cluster backups use it as the store outside the cluster. The adapter loads `@aws-sdk/client-s3`, an optional
|
|
42
|
+
* peer dependency, when you call this function, and it refuses to load an object larger than one stored file can be.
|
|
43
|
+
*
|
|
44
|
+
* @param config - The bucket, the region or endpoint, and the credentials.
|
|
45
|
+
* @returns An adapter you pass wherever Narsil takes a {@link PersistenceAdapter}.
|
|
46
|
+
* @throws A `NarsilError` with `PERSISTENCE_DEPENDENCY_MISSING` and the install command when the SDK is missing, and
|
|
47
|
+
* with `CONFIG_INVALID` when the bucket name is empty.
|
|
48
|
+
*
|
|
49
|
+
* @public
|
|
50
|
+
*/
|
|
51
|
+
declare function createS3Persistence(config: S3PersistenceConfig): Promise<PersistenceAdapter>;
|
|
52
|
+
|
|
53
|
+
export { type S3Credentials, type S3PersistenceConfig, createS3Persistence };
|