@fireproof/core 0.19.8-dev-global → 0.19.9-dev-frag
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/chunk-7EWIAXTM.js +7 -0
- package/chunk-7EWIAXTM.js.map +1 -0
- package/chunk-JO5AVWG7.js +67 -0
- package/chunk-JO5AVWG7.js.map +1 -0
- package/chunk-PB4BKL4O.js +7 -0
- package/chunk-PB4BKL4O.js.map +1 -0
- package/chunk-YS4GL6OK.js +266 -0
- package/chunk-YS4GL6OK.js.map +1 -0
- package/{store-indexdb-WLRSICCB.js → gateway-IZRHJWPE.js} +48 -80
- package/gateway-IZRHJWPE.js.map +1 -0
- package/gateway-YSNUK2L3.js +145 -0
- package/gateway-YSNUK2L3.js.map +1 -0
- package/index.cjs +2132 -1783
- package/index.cjs.map +1 -1
- package/index.d.cts +613 -513
- package/index.d.ts +613 -513
- package/index.global.js +19366 -20107
- package/index.global.js.map +1 -1
- package/index.js +1512 -1022
- package/index.js.map +1 -1
- package/key-bag-file-NMEBFSPM.js +54 -0
- package/key-bag-file-NMEBFSPM.js.map +1 -0
- package/key-bag-indexdb-X5V6GNBZ.js +50 -0
- package/key-bag-indexdb-X5V6GNBZ.js.map +1 -0
- package/mem-filesystem-B6C6QOIP.js +41 -0
- package/mem-filesystem-B6C6QOIP.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/node-filesystem-5JLBSHKQ.js +41 -0
- package/node-filesystem-5JLBSHKQ.js.map +1 -0
- package/package.json +8 -7
- package/tests/blockstore/fragment-gateway.test.ts +107 -0
- package/tests/blockstore/keyed-crypto.test.ts +302 -0
- package/tests/blockstore/loader.test.ts +24 -19
- package/tests/blockstore/store.test.ts +34 -28
- package/tests/blockstore/transaction.test.ts +19 -15
- package/tests/fireproof/config.test.ts +94 -78
- package/tests/fireproof/crdt.test.ts +34 -28
- package/tests/fireproof/database.test.ts +22 -14
- package/tests/fireproof/fireproof.test.fixture.ts +133 -0
- package/tests/fireproof/fireproof.test.ts +331 -219
- package/tests/fireproof/hello.test.ts +6 -4
- package/tests/fireproof/indexer.test.ts +34 -27
- package/tests/fireproof/utils.test.ts +65 -0
- package/tests/helpers.ts +25 -57
- package/utils-IZPK4QS7.js +14 -0
- package/utils-IZPK4QS7.js.map +1 -0
- package/chunk-BNL4PVBF.js +0 -314
- package/chunk-BNL4PVBF.js.map +0 -1
- package/chunk-JW2QT6BF.js +0 -184
- package/chunk-JW2QT6BF.js.map +0 -1
- package/node-sys-container-MIEX6ELJ.js +0 -29
- package/node-sys-container-MIEX6ELJ.js.map +0 -1
- package/store-file-VJ6BI4II.js +0 -191
- package/store-file-VJ6BI4II.js.map +0 -1
- package/store-indexdb-WLRSICCB.js.map +0 -1
package/index.cjs
CHANGED
@@ -30,452 +30,50 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
30
30
|
));
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
32
32
|
|
33
|
-
// src/
|
34
|
-
function
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
return value;
|
48
|
-
}
|
49
|
-
var init_types = __esm({
|
50
|
-
"src/types.ts"() {
|
51
|
-
"use strict";
|
52
|
-
}
|
53
|
-
});
|
54
|
-
|
55
|
-
// src/runtime/node-sys-container.ts
|
56
|
-
var node_sys_container_exports = {};
|
57
|
-
__export(node_sys_container_exports, {
|
58
|
-
createNodeSysContainer: () => createNodeSysContainer
|
59
|
-
});
|
60
|
-
async function createNodeSysContainer() {
|
61
|
-
return {
|
62
|
-
state: "node",
|
63
|
-
...path,
|
64
|
-
// ...(await import("node:os")),
|
65
|
-
// ...(await import("node:url")),
|
66
|
-
...os,
|
67
|
-
...url,
|
68
|
-
...fs,
|
69
|
-
join,
|
70
|
-
stat: fs.stat,
|
71
|
-
readdir: fs.readdir,
|
72
|
-
readfile: fs.readFile,
|
73
|
-
writefile: fs.writeFile
|
74
|
-
};
|
75
|
-
}
|
76
|
-
var fs, path, os, url;
|
77
|
-
var init_node_sys_container = __esm({
|
78
|
-
"src/runtime/node-sys-container.ts"() {
|
79
|
-
"use strict";
|
80
|
-
init_sys_container();
|
81
|
-
fs = __toESM(require("fs/promises"), 1);
|
82
|
-
path = __toESM(require("path"), 1);
|
83
|
-
os = __toESM(require("os"), 1);
|
84
|
-
url = __toESM(require("url"), 1);
|
85
|
-
}
|
86
|
-
});
|
87
|
-
|
88
|
-
// src/runtime/sys-container.ts
|
89
|
-
function isSet(value, ref = globalThis) {
|
90
|
-
const [head, ...tail] = value.split(".");
|
91
|
-
if (["object", "function"].includes(typeof ref) && ref && ["object", "function"].includes(typeof ref[head]) && ref[head]) {
|
92
|
-
if (tail.length <= 1) {
|
93
|
-
return true;
|
94
|
-
}
|
95
|
-
return isSet(tail.join("."), ref[head]);
|
96
|
-
}
|
97
|
-
return false;
|
98
|
-
}
|
99
|
-
function runtimeFn() {
|
100
|
-
const isNodeIsh = isSet("process.versions.node");
|
101
|
-
const isDeno = isSet("Deno");
|
102
|
-
return {
|
103
|
-
isNodeIsh,
|
104
|
-
isBrowser: !(isNodeIsh || isDeno),
|
105
|
-
isDeno,
|
106
|
-
isReactNative: false
|
107
|
-
};
|
108
|
-
}
|
109
|
-
function join(...paths) {
|
110
|
-
return paths.map((i) => i.replace(/\/+$/, "")).join("/");
|
111
|
-
}
|
112
|
-
var import_uuidv7, import_cement, onceStart, envImpl, sysContainer, SysContainer;
|
113
|
-
var init_sys_container = __esm({
|
114
|
-
"src/runtime/sys-container.ts"() {
|
115
|
-
"use strict";
|
116
|
-
import_uuidv7 = require("uuidv7");
|
117
|
-
import_cement = require("@adviser/cement");
|
118
|
-
init_types();
|
119
|
-
onceStart = new import_cement.ResolveOnce();
|
120
|
-
envImpl = new import_cement.EnvImpl({
|
121
|
-
symbol: "FP_ENV",
|
122
|
-
presetEnv: /* @__PURE__ */ new Map([
|
123
|
-
// ["FP_DEBUG", "xxx"],
|
124
|
-
// ["FP_ENV", "development"],
|
125
|
-
])
|
126
|
-
});
|
127
|
-
sysContainer = class {
|
128
|
-
constructor() {
|
129
|
-
this.freight = {
|
130
|
-
state: "seeded",
|
131
|
-
join,
|
132
|
-
dirname: (path2) => path2.split("/").slice(0, -1).join("/"),
|
133
|
-
homedir: () => {
|
134
|
-
throw new Error("SysContainer:homedir is not available in seeded state");
|
135
|
-
},
|
136
|
-
fileURLToPath: (strurl) => {
|
137
|
-
let url2;
|
138
|
-
if (typeof strurl === "string") {
|
139
|
-
url2 = new URL(strurl);
|
140
|
-
} else {
|
141
|
-
url2 = strurl;
|
142
|
-
}
|
143
|
-
return url2.pathname;
|
144
|
-
},
|
145
|
-
// assert: (condition: unknown, message?: string | Error) => {
|
146
|
-
// if (!condition) {
|
147
|
-
// if (message instanceof Error) {
|
148
|
-
// throw message;
|
149
|
-
// } else {
|
150
|
-
// throw new Error(message);
|
151
|
-
// }
|
152
|
-
// }
|
153
|
-
// },
|
154
|
-
mkdir: () => Promise.reject(new Error("SysContainer:mkdir is not available in seeded state")),
|
155
|
-
readdir: () => Promise.reject(new Error("SysContainer:readdir is not available in seeded state")),
|
156
|
-
rm: () => Promise.reject(new Error("SysContainer:rm is not available in seeded state")),
|
157
|
-
copyFile: () => Promise.reject(new Error("SysContainer:copyFile is not available in seeded state")),
|
158
|
-
readfile: () => Promise.reject(new Error("SysContainer:readfile is not available in seeded state")),
|
159
|
-
unlink: () => Promise.reject(new Error("SysContainer:unlink is not available in seeded state")),
|
160
|
-
writefile: () => Promise.reject(new Error("SysContainer:writefile is not available in seeded state")),
|
161
|
-
stat: () => Promise.reject(new Error("SysContainer:stat is not available in seeded state"))
|
162
|
-
};
|
163
|
-
this.id = (0, import_uuidv7.uuidv4)();
|
164
|
-
this.homedir = () => {
|
165
|
-
this.logSeeded("homedir");
|
166
|
-
return throwFalsy(this.freight).homedir();
|
167
|
-
};
|
168
|
-
this.runtime = runtimeFn;
|
169
|
-
this.env = envImpl;
|
170
|
-
}
|
171
|
-
async start() {
|
172
|
-
await onceStart.once(async () => {
|
173
|
-
switch (this.freight.state) {
|
174
|
-
case "seeded":
|
175
|
-
if (this.runtime().isNodeIsh) {
|
176
|
-
const { createNodeSysContainer: createNodeSysContainer2 } = await Promise.resolve().then(() => (init_node_sys_container(), node_sys_container_exports));
|
177
|
-
this.freight = await createNodeSysContainer2();
|
178
|
-
} else {
|
179
|
-
this.freight.state = "browser";
|
180
|
-
}
|
181
|
-
return;
|
182
|
-
case "browser":
|
183
|
-
case "node":
|
184
|
-
return;
|
185
|
-
}
|
186
|
-
});
|
187
|
-
}
|
188
|
-
async readdir(path2, options) {
|
189
|
-
this.logSeeded("readdir");
|
190
|
-
return throwFalsy(this.freight).readdir(path2, options) || [];
|
191
|
-
}
|
192
|
-
async readdirent(path2, options) {
|
193
|
-
this.logSeeded("readdirent");
|
194
|
-
return throwFalsy(this.freight).readdir(path2, { ...options, withFileTypes: true }) || [];
|
195
|
-
}
|
196
|
-
async readfile(path2, options) {
|
197
|
-
this.logSeeded("readfile");
|
198
|
-
return throwFalsy(this.freight).readfile(path2, options);
|
199
|
-
}
|
200
|
-
async mkdir(path2, options) {
|
201
|
-
this.logSeeded("mkdir");
|
202
|
-
return throwFalsy(this.freight).mkdir(path2, options);
|
203
|
-
}
|
204
|
-
async rm(path2, options) {
|
205
|
-
this.logSeeded("rm");
|
206
|
-
return throwFalsy(this.freight).rm(path2, options);
|
207
|
-
}
|
208
|
-
async unlink(path2) {
|
209
|
-
this.logSeeded("unlink");
|
210
|
-
return throwFalsy(this.freight).unlink(path2);
|
211
|
-
}
|
212
|
-
async writefile(path2, data) {
|
213
|
-
this.logSeeded("writefile");
|
214
|
-
return throwFalsy(this.freight).writefile(path2, data);
|
215
|
-
}
|
216
|
-
async copyFile(source, destination) {
|
217
|
-
this.logSeeded("copyFile");
|
218
|
-
return throwFalsy(this.freight).copyFile(source, destination);
|
219
|
-
}
|
220
|
-
async stat(path2) {
|
221
|
-
this.logSeeded("stat");
|
222
|
-
return throwFalsy(this.freight).stat(path2);
|
223
|
-
}
|
224
|
-
fileURLToPath(url2) {
|
225
|
-
this.logSeeded("fileURLToPath");
|
226
|
-
return throwFalsy(this.freight).fileURLToPath(url2);
|
227
|
-
}
|
228
|
-
dirname(path2) {
|
229
|
-
this.logSeeded("dirname");
|
230
|
-
return throwFalsy(this.freight).dirname(path2);
|
231
|
-
}
|
232
|
-
join(...args) {
|
233
|
-
this.logSeeded("join");
|
234
|
-
return throwFalsy(this.freight).join(...args);
|
235
|
-
}
|
236
|
-
logSeeded(method) {
|
237
|
-
if (this.freight.state === "seeded") {
|
238
|
-
const err = new Error();
|
239
|
-
console.warn(`SysContainer.${method} is not available in seeded state:`, err.stack);
|
240
|
-
}
|
241
|
-
}
|
242
|
-
};
|
243
|
-
SysContainer = new sysContainer();
|
244
|
-
}
|
245
|
-
});
|
246
|
-
|
247
|
-
// src/runtime/data-dir.ts
|
248
|
-
function dataDir(name, base) {
|
249
|
-
if (!base) {
|
250
|
-
if (SysContainer.runtime().isNodeIsh || SysContainer.runtime().isDeno) {
|
251
|
-
base = SysContainer.env.get("FP_STORAGE_URL") || `file://${SysContainer.join(SysContainer.homedir(), ".fireproof")}`;
|
252
|
-
} else {
|
253
|
-
base = `indexdb://fp`;
|
254
|
-
}
|
255
|
-
}
|
256
|
-
let url2;
|
257
|
-
if (typeof base === "string") {
|
258
|
-
try {
|
259
|
-
url2 = new URL(base.toString());
|
260
|
-
} catch (e) {
|
261
|
-
try {
|
262
|
-
base = `file://${base}`;
|
263
|
-
url2 = new URL(base);
|
264
|
-
} catch (e2) {
|
265
|
-
throw new Error(`invalid base url: ${base}`);
|
266
|
-
}
|
267
|
-
}
|
268
|
-
} else {
|
269
|
-
url2 = base;
|
270
|
-
}
|
271
|
-
url2.searchParams.set("name", name || "");
|
272
|
-
return url2.toString();
|
273
|
-
}
|
274
|
-
var init_data_dir = __esm({
|
275
|
-
"src/runtime/data-dir.ts"() {
|
276
|
-
"use strict";
|
277
|
-
init_sys_container();
|
278
|
-
}
|
279
|
-
});
|
280
|
-
|
281
|
-
// src/runtime/store-file-utils.ts
|
282
|
-
async function getPath(url2, logger) {
|
283
|
-
const basePath = url2.toString().replace(new RegExp(`^${url2.protocol}//`), "").replace(/\?.*$/, "");
|
284
|
-
const name = url2.searchParams.get("name");
|
285
|
-
if (name) {
|
286
|
-
const version = url2.searchParams.get("version");
|
287
|
-
if (!version) throw logger.Error().Str("url", url2.toString()).Msg(`version not found`).AsError();
|
288
|
-
return SysContainer.join(basePath, version, name);
|
289
|
-
}
|
290
|
-
return SysContainer.join(basePath);
|
291
|
-
}
|
292
|
-
function getFileName(url2, key, logger) {
|
293
|
-
switch (getStore(url2, logger, (...a) => a.join("/"))) {
|
294
|
-
case "data":
|
295
|
-
return key + ".car";
|
296
|
-
case "meta":
|
297
|
-
return key + ".json";
|
298
|
-
default:
|
299
|
-
throw logger.Error().Str("url", url2.toString()).Msg(`unsupported store type`).AsError();
|
300
|
-
}
|
301
|
-
}
|
302
|
-
function ensureIndexName(url2, name) {
|
303
|
-
if (url2.searchParams.has("index")) {
|
304
|
-
name = (url2.searchParams.get("index")?.replace(/[^a-zA-Z0-9]/g, "") || "idx") + "-" + name;
|
305
|
-
}
|
306
|
-
return name;
|
307
|
-
}
|
308
|
-
var init_store_file_utils = __esm({
|
309
|
-
"src/runtime/store-file-utils.ts"() {
|
310
|
-
"use strict";
|
311
|
-
init_utils();
|
312
|
-
init_sys_container();
|
313
|
-
}
|
314
|
-
});
|
315
|
-
|
316
|
-
// src/runtime/crypto.ts
|
317
|
-
var crypto_exports = {};
|
318
|
-
__export(crypto_exports, {
|
319
|
-
toCryptoOpts: () => toCryptoOpts
|
320
|
-
});
|
321
|
-
function randomBytes(size) {
|
322
|
-
const bytes = new Uint8Array(size);
|
323
|
-
if (size > 0) {
|
324
|
-
crypto.getRandomValues(bytes);
|
325
|
-
}
|
326
|
-
return bytes;
|
327
|
-
}
|
328
|
-
function digestSHA256(data) {
|
329
|
-
return Promise.resolve(crypto.subtle.digest("SHA-256", data));
|
330
|
-
}
|
331
|
-
function toCryptoOpts(cryptoOpts = {}) {
|
332
|
-
const opts = {
|
333
|
-
importKey: cryptoOpts.importKey || crypto.subtle.importKey.bind(crypto.subtle),
|
334
|
-
encrypt: cryptoOpts.encrypt || crypto.subtle.encrypt.bind(crypto.subtle),
|
335
|
-
decrypt: cryptoOpts.decrypt || crypto.subtle.decrypt.bind(crypto.subtle),
|
336
|
-
randomBytes: cryptoOpts.randomBytes || randomBytes,
|
337
|
-
digestSHA256: cryptoOpts.digestSHA256 || digestSHA256
|
338
|
-
};
|
339
|
-
return opts;
|
340
|
-
}
|
341
|
-
var init_crypto = __esm({
|
342
|
-
"src/runtime/crypto.ts"() {
|
343
|
-
"use strict";
|
344
|
-
}
|
345
|
-
});
|
346
|
-
|
347
|
-
// src/runtime/files.ts
|
348
|
-
var files_exports = {};
|
349
|
-
__export(files_exports, {
|
350
|
-
decodeFile: () => decodeFile,
|
351
|
-
encodeFile: () => encodeFile
|
352
|
-
});
|
353
|
-
async function collect(collectable) {
|
354
|
-
const chunks = [];
|
355
|
-
await collectable.pipeTo(
|
356
|
-
new WritableStream({
|
357
|
-
write(chunk) {
|
358
|
-
chunks.push(chunk);
|
359
|
-
}
|
360
|
-
})
|
361
|
-
);
|
362
|
-
return chunks;
|
363
|
-
}
|
364
|
-
async function encodeFile(blob) {
|
365
|
-
const readable = createFileEncoderStream(blob);
|
366
|
-
const blocks = await collect(readable);
|
367
|
-
return { cid: blocks.at(-1).cid, blocks };
|
33
|
+
// src/utils.ts
|
34
|
+
function presetEnv() {
|
35
|
+
const penv = new Map([
|
36
|
+
// ["FP_DEBUG", "xxx"],
|
37
|
+
// ["FP_ENV", "development"],
|
38
|
+
...Array.from(
|
39
|
+
Object.entries(
|
40
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
41
|
+
globalThis[Symbol.for("FP_PRESET_ENV")] || {}
|
42
|
+
)
|
43
|
+
)
|
44
|
+
// .map(([k, v]) => [k, v as string])
|
45
|
+
]);
|
46
|
+
return penv;
|
368
47
|
}
|
369
|
-
|
370
|
-
const
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
48
|
+
function ensureSuperThis(osthis) {
|
49
|
+
const env = (0, import_cement.envFactory)({
|
50
|
+
symbol: osthis?.env?.symbol || "FP_ENV",
|
51
|
+
presetEnv: osthis?.env?.presetEnv || presetEnv()
|
52
|
+
});
|
53
|
+
return new superThis({
|
54
|
+
logger: osthis?.logger || globalLogger,
|
55
|
+
env,
|
56
|
+
crypto: osthis?.crypto || (0, import_cement.toCryptoRuntime)(),
|
57
|
+
ctx: osthis?.ctx || {},
|
58
|
+
pathOps,
|
59
|
+
txt: osthis?.txt || txtOps
|
60
|
+
});
|
376
61
|
}
|
377
|
-
function
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
void (async () => {
|
382
|
-
await fileBuilder.finalize(unixfsWriter);
|
383
|
-
await unixfsWriter.close();
|
384
|
-
})();
|
385
|
-
return readable;
|
62
|
+
function ensureSuperLog(sthis, componentName, ctx) {
|
63
|
+
return sthis.clone({
|
64
|
+
logger: ensureLogger(sthis, componentName, ctx)
|
65
|
+
});
|
386
66
|
}
|
387
|
-
|
388
|
-
var init_files = __esm({
|
389
|
-
"src/runtime/files.ts"() {
|
390
|
-
"use strict";
|
391
|
-
UnixFS = __toESM(require("@ipld/unixfs"), 1);
|
392
|
-
raw = __toESM(require("multiformats/codecs/raw"), 1);
|
393
|
-
import_fixed = require("@ipld/unixfs/file/chunker/fixed");
|
394
|
-
import_balanced = require("@ipld/unixfs/file/layout/balanced");
|
395
|
-
import_ipfs_unixfs_exporter = require("ipfs-unixfs-exporter");
|
396
|
-
queuingStrategy = UnixFS.withCapacity();
|
397
|
-
settings = UnixFS.configure({
|
398
|
-
fileChunkEncoder: raw,
|
399
|
-
smallFileEncoder: raw,
|
400
|
-
chunker: (0, import_fixed.withMaxChunkSize)(1024 * 1024),
|
401
|
-
fileLayout: (0, import_balanced.withWidth)(1024)
|
402
|
-
});
|
403
|
-
UnixFSFileBuilder = class {
|
404
|
-
#file;
|
405
|
-
constructor(name, file) {
|
406
|
-
this.name = name;
|
407
|
-
this.#file = file;
|
408
|
-
}
|
409
|
-
async finalize(writer) {
|
410
|
-
const unixfsFileWriter = UnixFS.createFileWriter(writer);
|
411
|
-
await this.#file.stream().pipeTo(
|
412
|
-
new WritableStream({
|
413
|
-
async write(chunk) {
|
414
|
-
await unixfsFileWriter.write(chunk);
|
415
|
-
}
|
416
|
-
})
|
417
|
-
);
|
418
|
-
return await unixfsFileWriter.close();
|
419
|
-
}
|
420
|
-
};
|
421
|
-
}
|
422
|
-
});
|
423
|
-
|
424
|
-
// src/runtime/store-file-version.ts
|
425
|
-
var FILESTORE_VERSION;
|
426
|
-
var init_store_file_version = __esm({
|
427
|
-
"src/runtime/store-file-version.ts"() {
|
428
|
-
"use strict";
|
429
|
-
FILESTORE_VERSION = "v0.19-file";
|
430
|
-
}
|
431
|
-
});
|
432
|
-
|
433
|
-
// src/runtime/store-indexdb-version.ts
|
434
|
-
var INDEXDB_VERSION;
|
435
|
-
var init_store_indexdb_version = __esm({
|
436
|
-
"src/runtime/store-indexdb-version.ts"() {
|
437
|
-
"use strict";
|
438
|
-
INDEXDB_VERSION = "v0.19-indexdb";
|
439
|
-
}
|
440
|
-
});
|
441
|
-
|
442
|
-
// src/runtime/index.ts
|
443
|
-
var runtime_exports = {};
|
444
|
-
__export(runtime_exports, {
|
445
|
-
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
446
|
-
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
447
|
-
SysContainer: () => SysContainer,
|
448
|
-
crypto: () => crypto_exports,
|
449
|
-
dataDir: () => dataDir,
|
450
|
-
ensureIndexName: () => ensureIndexName,
|
451
|
-
files: () => files_exports,
|
452
|
-
getFileName: () => getFileName,
|
453
|
-
getPath: () => getPath,
|
454
|
-
join: () => join
|
455
|
-
});
|
456
|
-
var init_runtime = __esm({
|
457
|
-
"src/runtime/index.ts"() {
|
458
|
-
"use strict";
|
459
|
-
init_sys_container();
|
460
|
-
init_data_dir();
|
461
|
-
init_store_file_utils();
|
462
|
-
init_crypto();
|
463
|
-
init_files();
|
464
|
-
init_store_file_version();
|
465
|
-
init_store_indexdb_version();
|
466
|
-
}
|
467
|
-
});
|
468
|
-
|
469
|
-
// src/utils.ts
|
470
|
-
function ensureLogger(optsOrLogger, componentName, ctx) {
|
67
|
+
function ensureLogger(sthis, componentName, ctx) {
|
471
68
|
let logger = globalLogger;
|
472
|
-
if ((0,
|
473
|
-
logger =
|
474
|
-
} else if (
|
475
|
-
logger =
|
69
|
+
if ((0, import_cement.IsLogger)(sthis)) {
|
70
|
+
logger = sthis;
|
71
|
+
} else if (sthis && (0, import_cement.IsLogger)(sthis.logger)) {
|
72
|
+
logger = sthis.logger;
|
476
73
|
}
|
477
74
|
const cLogger = logger.With().Module(componentName);
|
478
75
|
const debug = [];
|
76
|
+
let exposeStack = false;
|
479
77
|
if (ctx) {
|
480
78
|
if ("debug" in ctx) {
|
481
79
|
if (typeof ctx.debug === "string" && ctx.debug.length > 0) {
|
@@ -485,8 +83,12 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
485
83
|
}
|
486
84
|
delete ctx.debug;
|
487
85
|
}
|
86
|
+
if ("exposeStack" in ctx) {
|
87
|
+
exposeStack = true;
|
88
|
+
delete ctx.exposeStack;
|
89
|
+
}
|
488
90
|
if ("this" in ctx) {
|
489
|
-
cLogger.Str("this",
|
91
|
+
cLogger.Str("this", sthis.nextId(4).str);
|
490
92
|
delete ctx.this;
|
491
93
|
}
|
492
94
|
for (const [key, value] of Object.entries(ctx)) {
|
@@ -500,7 +102,7 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
500
102
|
default:
|
501
103
|
if (value instanceof Date) {
|
502
104
|
cLogger.Str(key, value.toISOString());
|
503
|
-
} else if (
|
105
|
+
} else if ((0, import_cement.isURL)(value)) {
|
504
106
|
cLogger.Str(key, value.toString());
|
505
107
|
} else if (typeof value === "function") {
|
506
108
|
cLogger.Ref(key, value);
|
@@ -512,74 +114,152 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
512
114
|
}
|
513
115
|
}
|
514
116
|
registerFP_DEBUG.once(async () => {
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
117
|
+
sthis.env.onSet(
|
118
|
+
(key, value) => {
|
119
|
+
switch (key) {
|
120
|
+
case "FP_DEBUG":
|
121
|
+
logger.SetDebug(value || []);
|
122
|
+
break;
|
123
|
+
case "FP_STACK":
|
124
|
+
logger.SetExposeStack(!!value);
|
125
|
+
break;
|
126
|
+
}
|
127
|
+
},
|
128
|
+
"FP_DEBUG",
|
129
|
+
"FP_STACK"
|
130
|
+
);
|
520
131
|
}).finally(() => {
|
521
132
|
});
|
522
133
|
if (debug.length > 0) {
|
523
134
|
logger.SetDebug(debug);
|
524
135
|
}
|
136
|
+
if (exposeStack) {
|
137
|
+
logger.SetExposeStack(true);
|
138
|
+
}
|
525
139
|
const out = cLogger.Logger();
|
526
140
|
return out;
|
527
141
|
}
|
528
|
-
function getStore(
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
142
|
+
function getStore(url, sthis, joiner) {
|
143
|
+
const store = url.getParam("store");
|
144
|
+
switch (store) {
|
145
|
+
case "data":
|
146
|
+
case "wal":
|
147
|
+
case "meta":
|
148
|
+
break;
|
149
|
+
default:
|
150
|
+
throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
|
533
151
|
}
|
534
|
-
|
152
|
+
let name = store;
|
153
|
+
if (url.hasParam("index")) {
|
154
|
+
name = joiner(url.getParam("index") || "idx", name);
|
155
|
+
}
|
156
|
+
return { store, name };
|
535
157
|
}
|
536
|
-
function getKey(
|
537
|
-
const result =
|
538
|
-
if (!result) throw logger.Error().Str("url",
|
158
|
+
function getKey(url, logger) {
|
159
|
+
const result = url.getParam("key");
|
160
|
+
if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
|
539
161
|
return result;
|
540
162
|
}
|
541
|
-
function getName(
|
542
|
-
let result =
|
163
|
+
function getName(sthis, url) {
|
164
|
+
let result = url.getParam("name");
|
543
165
|
if (!result) {
|
544
|
-
result =
|
166
|
+
result = sthis.pathOps.dirname(url.pathname);
|
545
167
|
if (result.length === 0) {
|
546
|
-
throw logger.Error().Str("url",
|
168
|
+
throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
|
547
169
|
}
|
548
170
|
}
|
549
171
|
return result;
|
550
172
|
}
|
551
173
|
function exception2Result(fn) {
|
552
|
-
return fn().then((value) =>
|
174
|
+
return fn().then((value) => import_cement.Result.Ok(value)).catch((e) => import_cement.Result.Err(e));
|
553
175
|
}
|
554
176
|
async function exceptionWrapper(fn) {
|
555
|
-
return fn().catch((e) =>
|
177
|
+
return fn().catch((e) => import_cement.Result.Err(e));
|
556
178
|
}
|
557
|
-
var import_cement2, import_uuidv72, globalLogger, registerFP_DEBUG;
|
558
|
-
var init_utils = __esm({
|
559
|
-
"src/utils.ts"() {
|
560
|
-
"use strict";
|
561
|
-
import_cement2 = require("@adviser/cement");
|
562
|
-
init_runtime();
|
563
|
-
import_uuidv72 = require("uuidv7");
|
564
|
-
globalLogger = new import_cement2.LoggerImpl();
|
565
|
-
registerFP_DEBUG = new import_cement2.ResolveOnce();
|
566
|
-
}
|
567
|
-
});
|
568
|
-
|
569
|
-
// src/blockstore/gateway.ts
|
570
179
|
function isNotFoundError(e) {
|
571
|
-
if (
|
180
|
+
if (import_cement.Result.Is(e)) {
|
572
181
|
if (e.isOk()) return false;
|
573
182
|
e = e.Err();
|
574
183
|
}
|
575
184
|
if (e.code === "ENOENT") return true;
|
576
185
|
return false;
|
577
186
|
}
|
578
|
-
|
579
|
-
|
580
|
-
|
187
|
+
function dataDir(sthis, name, base) {
|
188
|
+
if (!base) {
|
189
|
+
if (!(0, import_cement.runtimeFn)().isBrowser) {
|
190
|
+
const home = sthis.env.get("HOME") || "./";
|
191
|
+
base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
|
192
|
+
} else {
|
193
|
+
base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
|
194
|
+
}
|
195
|
+
}
|
196
|
+
return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
|
197
|
+
}
|
198
|
+
function UInt8ArrayEqual(a, b) {
|
199
|
+
if (a.length !== b.length) {
|
200
|
+
return false;
|
201
|
+
}
|
202
|
+
for (let i = 0; i < a.length; i++) {
|
203
|
+
if (a[i] !== b[i]) {
|
204
|
+
return false;
|
205
|
+
}
|
206
|
+
}
|
207
|
+
return true;
|
208
|
+
}
|
209
|
+
var import_cement, import_base58, globalLogger, registerFP_DEBUG, superThis, pathOpsImpl, pathOps, txtOps, NotFoundError;
|
210
|
+
var init_utils = __esm({
|
211
|
+
"src/utils.ts"() {
|
581
212
|
"use strict";
|
582
|
-
|
213
|
+
import_cement = require("@adviser/cement");
|
214
|
+
import_base58 = require("multiformats/bases/base58");
|
215
|
+
globalLogger = new import_cement.LoggerImpl();
|
216
|
+
registerFP_DEBUG = new import_cement.ResolveOnce();
|
217
|
+
superThis = class _superThis {
|
218
|
+
constructor(opts) {
|
219
|
+
this.logger = opts.logger;
|
220
|
+
this.env = opts.env;
|
221
|
+
this.crypto = opts.crypto;
|
222
|
+
this.pathOps = opts.pathOps;
|
223
|
+
this.txt = opts.txt;
|
224
|
+
this.ctx = { ...opts.ctx };
|
225
|
+
}
|
226
|
+
nextId(bytes = 6) {
|
227
|
+
const bin = this.crypto.randomBytes(bytes);
|
228
|
+
return {
|
229
|
+
str: import_base58.base58btc.encode(bin),
|
230
|
+
bin
|
231
|
+
};
|
232
|
+
}
|
233
|
+
start() {
|
234
|
+
return Promise.resolve();
|
235
|
+
}
|
236
|
+
clone(override) {
|
237
|
+
return new _superThis({
|
238
|
+
logger: override.logger || this.logger,
|
239
|
+
env: (0, import_cement.envFactory)(override.env) || this.env,
|
240
|
+
crypto: override.crypto || this.crypto,
|
241
|
+
pathOps: override.pathOps || this.pathOps,
|
242
|
+
txt: override.txt || this.txt,
|
243
|
+
ctx: { ...this.ctx, ...override.ctx }
|
244
|
+
});
|
245
|
+
}
|
246
|
+
};
|
247
|
+
pathOpsImpl = class {
|
248
|
+
join(...paths) {
|
249
|
+
return paths.map((i) => i.replace(/\/+$/, "")).join("/");
|
250
|
+
}
|
251
|
+
dirname(path) {
|
252
|
+
return path.split("/").slice(0, -1).join("/");
|
253
|
+
}
|
254
|
+
// homedir() {
|
255
|
+
// throw new Error("SysContainer:homedir is not available in seeded state");
|
256
|
+
// }
|
257
|
+
};
|
258
|
+
pathOps = new pathOpsImpl();
|
259
|
+
txtOps = {
|
260
|
+
encode: (input) => new TextEncoder().encode(input),
|
261
|
+
decode: (input) => new TextDecoder().decode(input)
|
262
|
+
};
|
583
263
|
NotFoundError = class extends Error {
|
584
264
|
constructor() {
|
585
265
|
super(...arguments);
|
@@ -589,38 +269,149 @@ var init_gateway = __esm({
|
|
589
269
|
}
|
590
270
|
});
|
591
271
|
|
592
|
-
// src/runtime/
|
593
|
-
var
|
594
|
-
__export(
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
272
|
+
// src/runtime/gateways/file/mem-filesystem.ts
|
273
|
+
var mem_filesystem_exports = {};
|
274
|
+
__export(mem_filesystem_exports, {
|
275
|
+
MemFileSystem: () => MemFileSystem
|
276
|
+
});
|
277
|
+
var import_memfs, MemFileSystem;
|
278
|
+
var init_mem_filesystem = __esm({
|
279
|
+
"src/runtime/gateways/file/mem-filesystem.ts"() {
|
280
|
+
"use strict";
|
281
|
+
import_memfs = require("memfs");
|
282
|
+
init_utils2();
|
283
|
+
MemFileSystem = class {
|
284
|
+
async start() {
|
285
|
+
return this;
|
286
|
+
}
|
287
|
+
mkdir(path, options) {
|
288
|
+
return import_memfs.fs.promises.mkdir(path, options);
|
289
|
+
}
|
290
|
+
readdir(path, options) {
|
291
|
+
return import_memfs.fs.promises.readdir(path, options);
|
292
|
+
}
|
293
|
+
rm(path, options) {
|
294
|
+
return import_memfs.fs.promises.rm(path, options);
|
295
|
+
}
|
296
|
+
copyFile(source, destination) {
|
297
|
+
return import_memfs.fs.promises.copyFile(source, destination);
|
298
|
+
}
|
299
|
+
async readfile(path, options) {
|
300
|
+
const ret = await import_memfs.fs.promises.readFile(path, options);
|
301
|
+
return toArrayBuffer(ret);
|
302
|
+
}
|
303
|
+
stat(path) {
|
304
|
+
return import_memfs.fs.promises.stat(path);
|
305
|
+
}
|
306
|
+
unlink(path) {
|
307
|
+
return import_memfs.fs.promises.unlink(path);
|
308
|
+
}
|
309
|
+
writefile(path, data) {
|
310
|
+
return import_memfs.fs.promises.writeFile(path, Buffer.from(data));
|
311
|
+
}
|
312
|
+
};
|
313
|
+
}
|
314
|
+
});
|
315
|
+
|
316
|
+
// src/runtime/gateways/file/node-filesystem.ts
|
317
|
+
var node_filesystem_exports = {};
|
318
|
+
__export(node_filesystem_exports, {
|
319
|
+
NodeFileSystem: () => NodeFileSystem
|
320
|
+
});
|
321
|
+
var NodeFileSystem;
|
322
|
+
var init_node_filesystem = __esm({
|
323
|
+
"src/runtime/gateways/file/node-filesystem.ts"() {
|
324
|
+
"use strict";
|
325
|
+
init_utils2();
|
326
|
+
NodeFileSystem = class {
|
327
|
+
async start() {
|
328
|
+
this.fs = await import("fs/promises");
|
329
|
+
return this;
|
330
|
+
}
|
331
|
+
async mkdir(path, options) {
|
332
|
+
return this.fs?.mkdir(path, options);
|
333
|
+
}
|
334
|
+
async readdir(path, options) {
|
335
|
+
return this.fs?.readdir(path, options);
|
336
|
+
}
|
337
|
+
async rm(path, options) {
|
338
|
+
return this.fs?.rm(path, options);
|
339
|
+
}
|
340
|
+
async copyFile(source, destination) {
|
341
|
+
return this.fs?.copyFile(source, destination);
|
342
|
+
}
|
343
|
+
async readfile(path, options) {
|
344
|
+
const ret = await this.fs?.readFile(path, options);
|
345
|
+
return toArrayBuffer(ret);
|
346
|
+
}
|
347
|
+
stat(path) {
|
348
|
+
return this.fs?.stat(path);
|
349
|
+
}
|
350
|
+
async unlink(path) {
|
351
|
+
return this.fs?.unlink(path);
|
352
|
+
}
|
353
|
+
async writefile(path, data) {
|
354
|
+
return this.fs?.writeFile(path, Buffer.from(data));
|
355
|
+
}
|
356
|
+
};
|
357
|
+
}
|
358
|
+
});
|
359
|
+
|
360
|
+
// src/runtime/gateways/file/utils.ts
|
361
|
+
var utils_exports = {};
|
362
|
+
__export(utils_exports, {
|
363
|
+
getFileName: () => getFileName,
|
364
|
+
getFileSystem: () => getFileSystem,
|
365
|
+
getPath: () => getPath,
|
366
|
+
toArrayBuffer: () => toArrayBuffer
|
599
367
|
});
|
600
|
-
async function
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
}
|
616
|
-
const v = await SysContainer.readfile(vFile);
|
617
|
-
if (v.toString() !== FILESTORE_VERSION) {
|
618
|
-
console.warn(`version mismatch:${vFile}: ${v.toString()}!=${FILESTORE_VERSION}`);
|
368
|
+
async function getFileSystem(url) {
|
369
|
+
const name = url.getParam("fs");
|
370
|
+
let fs2;
|
371
|
+
switch (name) {
|
372
|
+
case "mem":
|
373
|
+
{
|
374
|
+
const { MemFileSystem: MemFileSystem2 } = await Promise.resolve().then(() => (init_mem_filesystem(), mem_filesystem_exports));
|
375
|
+
fs2 = new MemFileSystem2();
|
376
|
+
}
|
377
|
+
break;
|
378
|
+
case "node":
|
379
|
+
case "sys":
|
380
|
+
default: {
|
381
|
+
const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
|
382
|
+
fs2 = new NodeFileSystem2();
|
619
383
|
}
|
620
|
-
}
|
621
|
-
return
|
384
|
+
}
|
385
|
+
return fs2.start();
|
386
|
+
}
|
387
|
+
function getPath(url, sthis) {
|
388
|
+
const basePath = url.pathname;
|
389
|
+
const name = url.getParam("name");
|
390
|
+
if (name) {
|
391
|
+
const version = url.getParam("version");
|
392
|
+
if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
|
393
|
+
return sthis.pathOps.join(basePath, version, name);
|
394
|
+
}
|
395
|
+
return sthis.pathOps.join(basePath);
|
396
|
+
}
|
397
|
+
function getFileName(url, sthis) {
|
398
|
+
const key = url.getParam("key");
|
399
|
+
if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
|
400
|
+
const res = getStore(url, sthis, (...a) => a.join("-"));
|
401
|
+
switch (res.store) {
|
402
|
+
case "data":
|
403
|
+
return sthis.pathOps.join(res.name, key + ".car");
|
404
|
+
case "wal":
|
405
|
+
case "meta":
|
406
|
+
return sthis.pathOps.join(res.name, key + ".json");
|
407
|
+
default:
|
408
|
+
throw sthis.logger.Error().Url(url).Msg(`unsupported store type`).AsError();
|
409
|
+
}
|
622
410
|
}
|
623
411
|
function toArrayBuffer(buffer) {
|
412
|
+
if (typeof buffer === "string") {
|
413
|
+
buffer = Buffer.from(buffer);
|
414
|
+
}
|
624
415
|
const ab = new ArrayBuffer(buffer.length);
|
625
416
|
const view = new Uint8Array(ab);
|
626
417
|
for (let i = 0; i < buffer.length; ++i) {
|
@@ -628,167 +419,290 @@ function toArrayBuffer(buffer) {
|
|
628
419
|
}
|
629
420
|
return view;
|
630
421
|
}
|
631
|
-
var
|
632
|
-
|
633
|
-
|
422
|
+
var init_utils2 = __esm({
|
423
|
+
"src/runtime/gateways/file/utils.ts"() {
|
424
|
+
"use strict";
|
425
|
+
init_utils();
|
426
|
+
}
|
427
|
+
});
|
428
|
+
|
429
|
+
// src/runtime/key-bag-file.ts
|
430
|
+
var key_bag_file_exports = {};
|
431
|
+
__export(key_bag_file_exports, {
|
432
|
+
KeyBagProviderFile: () => KeyBagProviderFile
|
433
|
+
});
|
434
|
+
var KeyBagProviderFile;
|
435
|
+
var init_key_bag_file = __esm({
|
436
|
+
"src/runtime/key-bag-file.ts"() {
|
437
|
+
"use strict";
|
438
|
+
init_utils();
|
439
|
+
KeyBagProviderFile = class {
|
440
|
+
async _prepare(id) {
|
441
|
+
await this.sthis.start();
|
442
|
+
let sysFS;
|
443
|
+
switch (this.url.protocol) {
|
444
|
+
case "file:": {
|
445
|
+
const { getFileSystem: getFileSystem2 } = await Promise.resolve().then(() => (init_utils2(), utils_exports));
|
446
|
+
sysFS = await getFileSystem2(this.url);
|
447
|
+
break;
|
448
|
+
}
|
449
|
+
default:
|
450
|
+
throw this.logger.Error().Url(this.url).Msg("unsupported protocol").AsError();
|
451
|
+
}
|
452
|
+
const dirName = this.url.pathname;
|
453
|
+
await sysFS.mkdir(dirName, { recursive: true });
|
454
|
+
return {
|
455
|
+
dirName,
|
456
|
+
sysFS,
|
457
|
+
fName: this.sthis.pathOps.join(dirName, `${id.replace(/[^a-zA-Z0-9]/g, "_")}.json`)
|
458
|
+
};
|
459
|
+
}
|
460
|
+
constructor(url, sthis) {
|
461
|
+
this.url = url;
|
462
|
+
this.sthis = sthis;
|
463
|
+
this.logger = sthis.logger;
|
464
|
+
}
|
465
|
+
async get(id) {
|
466
|
+
const ctx = await this._prepare(id);
|
467
|
+
try {
|
468
|
+
const p = await ctx.sysFS.readfile(ctx.fName);
|
469
|
+
const ki = JSON.parse(this.sthis.txt.decode(p));
|
470
|
+
return ki;
|
471
|
+
} catch (e) {
|
472
|
+
if (isNotFoundError(e)) {
|
473
|
+
return void 0;
|
474
|
+
}
|
475
|
+
throw this.logger.Error().Err(e).Str("file", ctx.dirName).Msg("read bag failed").AsError();
|
476
|
+
}
|
477
|
+
}
|
478
|
+
async set(id, item) {
|
479
|
+
const ctx = await this._prepare(id);
|
480
|
+
const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
|
481
|
+
await ctx.sysFS.writefile(ctx.fName, p);
|
482
|
+
}
|
483
|
+
};
|
484
|
+
}
|
485
|
+
});
|
486
|
+
|
487
|
+
// src/runtime/key-bag-indexdb.ts
|
488
|
+
var key_bag_indexdb_exports = {};
|
489
|
+
__export(key_bag_indexdb_exports, {
|
490
|
+
KeyBagProviderIndexDB: () => KeyBagProviderIndexDB
|
491
|
+
});
|
492
|
+
var import_idb, import_cement4, KeyBagProviderIndexDB;
|
493
|
+
var init_key_bag_indexdb = __esm({
|
494
|
+
"src/runtime/key-bag-indexdb.ts"() {
|
495
|
+
"use strict";
|
496
|
+
import_idb = require("idb");
|
497
|
+
init_utils2();
|
498
|
+
import_cement4 = require("@adviser/cement");
|
499
|
+
KeyBagProviderIndexDB = class {
|
500
|
+
constructor(url, sthis) {
|
501
|
+
this._db = new import_cement4.ResolveOnce();
|
502
|
+
this.sthis = sthis;
|
503
|
+
this.logger = sthis.logger;
|
504
|
+
this.url = url;
|
505
|
+
this.dbName = getPath(this.url, this.sthis);
|
506
|
+
}
|
507
|
+
async _prepare() {
|
508
|
+
return this._db.once(async () => {
|
509
|
+
return await (0, import_idb.openDB)(this.dbName, 1, {
|
510
|
+
upgrade(db) {
|
511
|
+
["bag"].map((store) => {
|
512
|
+
db.createObjectStore(store, {
|
513
|
+
autoIncrement: false
|
514
|
+
});
|
515
|
+
});
|
516
|
+
}
|
517
|
+
});
|
518
|
+
});
|
519
|
+
}
|
520
|
+
async get(id) {
|
521
|
+
const db = await this._prepare();
|
522
|
+
const tx = db.transaction(["bag"], "readonly");
|
523
|
+
const keyItem = await tx.objectStore("bag").get(id);
|
524
|
+
await tx.done;
|
525
|
+
if (!keyItem) {
|
526
|
+
return void 0;
|
527
|
+
}
|
528
|
+
return keyItem;
|
529
|
+
}
|
530
|
+
async set(id, item) {
|
531
|
+
const db = await this._prepare();
|
532
|
+
const tx = db.transaction(["bag"], "readwrite");
|
533
|
+
await tx.objectStore("bag").put(item, id);
|
534
|
+
await tx.done;
|
535
|
+
}
|
536
|
+
};
|
537
|
+
}
|
538
|
+
});
|
539
|
+
|
540
|
+
// src/runtime/gateways/file/version.ts
|
541
|
+
var FILESTORE_VERSION;
|
542
|
+
var init_version = __esm({
|
543
|
+
"src/runtime/gateways/file/version.ts"() {
|
634
544
|
"use strict";
|
635
|
-
|
636
|
-
|
637
|
-
|
545
|
+
FILESTORE_VERSION = "v0.19-file";
|
546
|
+
}
|
547
|
+
});
|
548
|
+
|
549
|
+
// src/runtime/gateways/file/gateway.ts
|
550
|
+
var gateway_exports = {};
|
551
|
+
__export(gateway_exports, {
|
552
|
+
FileGateway: () => FileGateway,
|
553
|
+
FileTestStore: () => FileTestStore
|
554
|
+
});
|
555
|
+
var import_cement8, versionFiles, FileGateway, FileTestStore;
|
556
|
+
var init_gateway = __esm({
|
557
|
+
"src/runtime/gateways/file/gateway.ts"() {
|
558
|
+
"use strict";
|
559
|
+
init_version();
|
560
|
+
import_cement8 = require("@adviser/cement");
|
638
561
|
init_utils();
|
639
|
-
|
640
|
-
|
641
|
-
versionFiles = /* @__PURE__ */ new Map();
|
562
|
+
init_utils2();
|
563
|
+
versionFiles = new import_cement8.KeyedResolvOnce();
|
642
564
|
FileGateway = class {
|
643
|
-
|
644
|
-
this.logger
|
565
|
+
get fs() {
|
566
|
+
if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
|
567
|
+
return this._fs;
|
568
|
+
}
|
569
|
+
constructor(sthis) {
|
570
|
+
this.sthis = sthis;
|
571
|
+
this.logger = sthis.logger;
|
572
|
+
}
|
573
|
+
async getVersionFromFile(path, logger) {
|
574
|
+
return versionFiles.get(path).once(async () => {
|
575
|
+
await this.fs.mkdir(path, { recursive: true });
|
576
|
+
const vFile = this.sthis.pathOps.join(path, "version");
|
577
|
+
const vFileStat = await this.fs.stat(vFile).catch(() => void 0);
|
578
|
+
if (!vFileStat) {
|
579
|
+
await this.fs.writefile(this.sthis.pathOps.join(path, "version"), FILESTORE_VERSION);
|
580
|
+
return FILESTORE_VERSION;
|
581
|
+
} else if (!vFileStat.isFile()) {
|
582
|
+
throw logger.Error().Str("file", vFile).Msg(`version file is a directory`).AsError();
|
583
|
+
}
|
584
|
+
const v = await this.fs.readfile(vFile);
|
585
|
+
const vStr = new TextDecoder().decode(v);
|
586
|
+
if (vStr !== FILESTORE_VERSION) {
|
587
|
+
logger.Warn().Str("file", vFile).Str("from", vStr).Str("expected", FILESTORE_VERSION).Msg(`version mismatch`);
|
588
|
+
}
|
589
|
+
return vStr;
|
590
|
+
});
|
645
591
|
}
|
646
592
|
start(baseURL) {
|
647
593
|
return exception2Result(async () => {
|
648
|
-
await
|
649
|
-
|
650
|
-
const
|
651
|
-
|
652
|
-
const
|
653
|
-
|
654
|
-
|
655
|
-
this.
|
656
|
-
|
594
|
+
this._fs = await getFileSystem(baseURL);
|
595
|
+
await this.fs.start();
|
596
|
+
const url = baseURL.build();
|
597
|
+
url.defParam("version", FILESTORE_VERSION);
|
598
|
+
const dbUrl = await this.buildUrl(url.URI(), "dummy");
|
599
|
+
const dbdirFile = this.getFilePath(dbUrl.Ok());
|
600
|
+
await this.fs.mkdir(this.sthis.pathOps.dirname(dbdirFile), { recursive: true });
|
601
|
+
const dbroot = this.sthis.pathOps.dirname(dbdirFile);
|
602
|
+
this.logger.Debug().Url(url.URI()).Str("dbroot", dbroot).Msg("start");
|
603
|
+
url.setParam("version", await this.getVersionFromFile(dbroot, this.logger));
|
604
|
+
return url.URI();
|
657
605
|
});
|
658
606
|
}
|
607
|
+
async buildUrl(baseUrl, key) {
|
608
|
+
return import_cement8.Result.Ok(baseUrl.build().setParam("key", key).URI());
|
609
|
+
}
|
659
610
|
async close() {
|
660
|
-
return
|
611
|
+
return import_cement8.Result.Ok(void 0);
|
661
612
|
}
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
613
|
+
// abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
|
614
|
+
getFilePath(url) {
|
615
|
+
const key = url.getParam("key");
|
616
|
+
if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
|
617
|
+
return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
666
618
|
}
|
667
|
-
async put(
|
619
|
+
async put(url, body) {
|
668
620
|
return exception2Result(async () => {
|
669
|
-
const file = this.getFilePath(
|
670
|
-
this.logger.Debug().Str("url",
|
671
|
-
await
|
621
|
+
const file = await this.getFilePath(url);
|
622
|
+
this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
|
623
|
+
await this.fs.writefile(file, body);
|
672
624
|
});
|
673
625
|
}
|
674
|
-
async get(
|
626
|
+
async get(url) {
|
675
627
|
return exceptionWrapper(async () => {
|
676
|
-
const file = this.getFilePath(
|
628
|
+
const file = this.getFilePath(url);
|
677
629
|
try {
|
678
|
-
const res = await
|
679
|
-
this.logger.Debug().Url(
|
680
|
-
return
|
630
|
+
const res = await this.fs.readfile(file);
|
631
|
+
this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
|
632
|
+
return import_cement8.Result.Ok(new Uint8Array(res));
|
681
633
|
} catch (e) {
|
682
634
|
if (isNotFoundError(e)) {
|
683
|
-
return
|
635
|
+
return import_cement8.Result.Err(new NotFoundError(`file not found: ${file}`));
|
684
636
|
}
|
685
|
-
return
|
637
|
+
return import_cement8.Result.Err(e);
|
686
638
|
}
|
687
639
|
});
|
688
640
|
}
|
689
|
-
async delete(
|
641
|
+
async delete(url) {
|
690
642
|
return exception2Result(async () => {
|
691
|
-
await
|
643
|
+
await this.fs.unlink(this.getFilePath(url));
|
692
644
|
});
|
693
645
|
}
|
694
|
-
async
|
695
|
-
const
|
696
|
-
if (
|
697
|
-
const filepath =
|
698
|
-
let
|
646
|
+
async destroy(baseURL) {
|
647
|
+
const url = await this.buildUrl(baseURL, "x");
|
648
|
+
if (url.isErr()) return url;
|
649
|
+
const filepath = this.sthis.pathOps.dirname(this.getFilePath(url.Ok()));
|
650
|
+
let files = [];
|
699
651
|
try {
|
700
|
-
|
652
|
+
files = await this.fs.readdir(filepath);
|
701
653
|
} catch (e) {
|
702
654
|
if (!isNotFoundError(e)) {
|
703
655
|
throw this.logger.Error().Err(e).Str("dir", filepath).Msg("destroy:readdir").AsError();
|
704
656
|
}
|
705
657
|
}
|
706
|
-
for (const file of
|
707
|
-
const pathed =
|
658
|
+
for (const file of files) {
|
659
|
+
const pathed = this.sthis.pathOps.join(filepath, file);
|
708
660
|
try {
|
709
|
-
await
|
661
|
+
await this.fs.unlink(pathed);
|
710
662
|
} catch (e) {
|
711
663
|
if (!isNotFoundError(e)) {
|
712
664
|
throw this.logger.Error().Err(e).Str("file", pathed).Msg("destroy:unlink").AsError();
|
713
665
|
}
|
714
666
|
}
|
715
667
|
}
|
716
|
-
return
|
717
|
-
}
|
718
|
-
};
|
719
|
-
FileWALGateway = class extends FileGateway {
|
720
|
-
constructor(logger) {
|
721
|
-
super(ensureLogger(logger, "FileWALGateway"));
|
722
|
-
}
|
723
|
-
async destroy(baseURL) {
|
724
|
-
return this.destroyDir(baseURL);
|
725
|
-
}
|
726
|
-
async buildUrl(baseUrl, key) {
|
727
|
-
const url2 = new URL(baseUrl.toString());
|
728
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "wal"), key + ".json");
|
729
|
-
return import_cement6.Result.Ok(url2);
|
730
|
-
}
|
731
|
-
};
|
732
|
-
FileMetaGateway = class extends FileGateway {
|
733
|
-
constructor(logger) {
|
734
|
-
super(ensureLogger(logger, "FileMetaGateway"));
|
735
|
-
}
|
736
|
-
async destroy(baseURL) {
|
737
|
-
return this.destroyDir(baseURL);
|
738
|
-
}
|
739
|
-
async buildUrl(baseUrl, key) {
|
740
|
-
const url2 = new URL(baseUrl.toString());
|
741
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "meta"), key + ".json");
|
742
|
-
return import_cement6.Result.Ok(url2);
|
743
|
-
}
|
744
|
-
};
|
745
|
-
FileDataGateway = class extends FileGateway {
|
746
|
-
constructor(logger) {
|
747
|
-
super(ensureLogger(logger, "FileDataGateway"));
|
748
|
-
this.branches = /* @__PURE__ */ new Set();
|
749
|
-
}
|
750
|
-
async destroy(baseURL) {
|
751
|
-
return this.destroyDir(baseURL);
|
752
|
-
}
|
753
|
-
async buildUrl(baseUrl, key) {
|
754
|
-
const url2 = new URL(baseUrl.toString());
|
755
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "data"), key + ".car");
|
756
|
-
return import_cement6.Result.Ok(url2);
|
668
|
+
return import_cement8.Result.Ok(void 0);
|
757
669
|
}
|
758
670
|
};
|
759
671
|
FileTestStore = class {
|
760
|
-
constructor(
|
761
|
-
this.logger = ensureLogger(
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
const
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
);
|
770
|
-
|
771
|
-
const buffer = await SysContainer.readfile(dbFile);
|
772
|
-
logger.Debug().Str("dbFile", dbFile).Len(buffer).Msg("got");
|
773
|
-
return toArrayBuffer(buffer);
|
672
|
+
constructor(sthis) {
|
673
|
+
this.logger = ensureLogger(sthis, "FileTestStore");
|
674
|
+
this.sthis = sthis;
|
675
|
+
}
|
676
|
+
async get(iurl, key) {
|
677
|
+
const url = iurl.build().setParam("key", key).URI();
|
678
|
+
const dbFile = this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
679
|
+
this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
|
680
|
+
const buffer = await (await getFileSystem(url)).readfile(dbFile);
|
681
|
+
this.logger.Debug().Url(url).Str("dbFile", dbFile).Len(buffer).Msg("got");
|
682
|
+
return buffer;
|
774
683
|
}
|
775
684
|
};
|
776
685
|
}
|
777
686
|
});
|
778
687
|
|
779
|
-
// src/runtime/
|
780
|
-
var
|
781
|
-
|
782
|
-
|
783
|
-
|
688
|
+
// src/runtime/gateways/indexdb/version.ts
|
689
|
+
var INDEXDB_VERSION;
|
690
|
+
var init_version2 = __esm({
|
691
|
+
"src/runtime/gateways/indexdb/version.ts"() {
|
692
|
+
"use strict";
|
693
|
+
INDEXDB_VERSION = "v0.19-indexdb";
|
694
|
+
}
|
695
|
+
});
|
696
|
+
|
697
|
+
// src/runtime/gateways/indexdb/gateway.ts
|
698
|
+
var gateway_exports2 = {};
|
699
|
+
__export(gateway_exports2, {
|
700
|
+
IndexDBGateway: () => IndexDBGateway,
|
784
701
|
IndexDBTestStore: () => IndexDBTestStore,
|
785
|
-
IndexDBWalGateway: () => IndexDBWalGateway,
|
786
702
|
getIndexDBName: () => getIndexDBName
|
787
703
|
});
|
788
|
-
function ensureVersion(
|
789
|
-
|
790
|
-
ret.searchParams.set("version", url2.searchParams.get("version") || INDEXDB_VERSION);
|
791
|
-
return ret;
|
704
|
+
function ensureVersion(url) {
|
705
|
+
return url.build().defParam("version", INDEXDB_VERSION).URI();
|
792
706
|
}
|
793
707
|
function sanitzeKey(key) {
|
794
708
|
if (key.length === 1) {
|
@@ -796,10 +710,10 @@ function sanitzeKey(key) {
|
|
796
710
|
}
|
797
711
|
return key;
|
798
712
|
}
|
799
|
-
async function connectIdb(
|
800
|
-
const dbName = getIndexDBName(
|
713
|
+
async function connectIdb(url, sthis) {
|
714
|
+
const dbName = getIndexDBName(url, sthis);
|
801
715
|
const once = await onceIndexDB.get(dbName.fullDb).once(async () => {
|
802
|
-
const db = await (0,
|
716
|
+
const db = await (0, import_idb2.openDB)(dbName.fullDb, 1, {
|
803
717
|
upgrade(db2) {
|
804
718
|
["version", "data", "wal", "meta", "idx.data", "idx.wal", "idx.meta"].map((store) => {
|
805
719
|
db2.createObjectStore(store, {
|
@@ -809,27 +723,29 @@ async function connectIdb(url2, logger) {
|
|
809
723
|
}
|
810
724
|
});
|
811
725
|
const found = await db.get("version", "version");
|
812
|
-
const version =
|
726
|
+
const version = ensureVersion(url).getParam("version");
|
813
727
|
if (!found) {
|
814
728
|
await db.put("version", { version }, "version");
|
815
729
|
} else if (found.version !== version) {
|
816
|
-
logger.Warn().Str("url",
|
730
|
+
sthis.logger.Warn().Str("url", url.toString()).Str("version", version).Str("found", found.version).Msg("version mismatch");
|
817
731
|
}
|
818
|
-
return { db, dbName, version };
|
732
|
+
return { db, dbName, version, url };
|
819
733
|
});
|
820
|
-
|
821
|
-
|
734
|
+
return {
|
735
|
+
...once,
|
736
|
+
url: url.build().setParam("version", once.version).URI()
|
737
|
+
};
|
822
738
|
}
|
823
739
|
function joinDBName(...names) {
|
824
740
|
return names.map((i) => i.replace(/^[^a-zA-Z0-9]+/g, "").replace(/[^a-zA-Z0-9]+/g, "_")).filter((i) => i.length).join(".");
|
825
741
|
}
|
826
|
-
function getIndexDBName(iurl,
|
827
|
-
const
|
828
|
-
const fullDb =
|
829
|
-
const dbName =
|
830
|
-
if (!dbName) throw logger.Error().Str("url",
|
742
|
+
function getIndexDBName(iurl, sthis) {
|
743
|
+
const url = ensureVersion(iurl);
|
744
|
+
const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
|
745
|
+
const dbName = url.getParam("name");
|
746
|
+
if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
|
831
747
|
const result = joinDBName(fullDb, dbName);
|
832
|
-
const objStore = getStore(
|
748
|
+
const objStore = getStore(url, sthis, joinDBName).name;
|
833
749
|
const connectionKey = [result, objStore].join(":");
|
834
750
|
return {
|
835
751
|
fullDb: result,
|
@@ -838,40 +754,38 @@ function getIndexDBName(iurl, logger) {
|
|
838
754
|
dbName
|
839
755
|
};
|
840
756
|
}
|
841
|
-
var
|
842
|
-
var
|
843
|
-
"src/runtime/
|
757
|
+
var import_idb2, import_cement9, onceIndexDB, IndexDBGateway, IndexDBTestStore;
|
758
|
+
var init_gateway2 = __esm({
|
759
|
+
"src/runtime/gateways/indexdb/gateway.ts"() {
|
844
760
|
"use strict";
|
845
|
-
|
846
|
-
|
847
|
-
|
761
|
+
import_idb2 = require("idb");
|
762
|
+
import_cement9 = require("@adviser/cement");
|
763
|
+
init_version2();
|
848
764
|
init_utils();
|
849
|
-
|
850
|
-
init_sys_container();
|
851
|
-
onceIndexDB = new import_cement7.KeyedResolvOnce();
|
765
|
+
onceIndexDB = new import_cement9.KeyedResolvOnce();
|
852
766
|
IndexDBGateway = class {
|
853
|
-
constructor(
|
854
|
-
this.
|
855
|
-
this.logger =
|
856
|
-
|
857
|
-
idb() {
|
858
|
-
this.db;
|
767
|
+
constructor(sthis) {
|
768
|
+
this._db = {};
|
769
|
+
this.logger = ensureLogger(sthis, "IndexDBGateway");
|
770
|
+
this.sthis = sthis;
|
859
771
|
}
|
860
772
|
async start(baseURL) {
|
861
773
|
return exception2Result(async () => {
|
862
774
|
this.logger.Debug().Url(baseURL).Msg("starting");
|
863
|
-
await
|
864
|
-
|
865
|
-
this.
|
775
|
+
await this.sthis.start();
|
776
|
+
const ic = await connectIdb(baseURL, this.sthis);
|
777
|
+
this._db = ic.db;
|
778
|
+
this.logger.Debug().Url(ic.url).Msg("started");
|
779
|
+
return ic.url;
|
866
780
|
});
|
867
781
|
}
|
868
782
|
async close() {
|
869
|
-
return
|
783
|
+
return import_cement9.Result.Ok(void 0);
|
870
784
|
}
|
871
785
|
async destroy(baseUrl) {
|
872
786
|
return exception2Result(async () => {
|
873
|
-
const type = getStore(baseUrl, this.
|
874
|
-
const idb = this.
|
787
|
+
const type = getStore(baseUrl, this.sthis, joinDBName).name;
|
788
|
+
const idb = this._db;
|
875
789
|
const trans = idb.transaction(type, "readwrite");
|
876
790
|
const object_store = trans.objectStore(type);
|
877
791
|
const toDelete = [];
|
@@ -884,87 +798,58 @@ var init_store_indexdb = __esm({
|
|
884
798
|
await trans.done;
|
885
799
|
});
|
886
800
|
}
|
887
|
-
|
801
|
+
buildUrl(baseUrl, key) {
|
802
|
+
return Promise.resolve(import_cement9.Result.Ok(baseUrl.build().setParam("key", key).URI()));
|
803
|
+
}
|
804
|
+
async get(url) {
|
888
805
|
return exceptionWrapper(async () => {
|
889
|
-
const key = getKey(
|
890
|
-
const store = getStore(
|
891
|
-
this.logger.Debug().Url(
|
892
|
-
const tx = this.
|
806
|
+
const key = getKey(url, this.logger);
|
807
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
808
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("getting");
|
809
|
+
const tx = this._db.transaction([store], "readonly");
|
893
810
|
const bytes = await tx.objectStore(store).get(sanitzeKey(key));
|
894
811
|
await tx.done;
|
895
812
|
if (!bytes) {
|
896
|
-
return
|
813
|
+
return import_cement9.Result.Err(new NotFoundError(`missing ${key}`));
|
897
814
|
}
|
898
|
-
return
|
815
|
+
return import_cement9.Result.Ok(bytes);
|
899
816
|
});
|
900
817
|
}
|
901
|
-
async put(
|
818
|
+
async put(url, value) {
|
902
819
|
return exception2Result(async () => {
|
903
|
-
const key = getKey(
|
904
|
-
const store = getStore(
|
905
|
-
this.logger.Debug().Url(
|
906
|
-
const tx = this.
|
820
|
+
const key = getKey(url, this.logger);
|
821
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
822
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
|
823
|
+
const tx = this._db.transaction([store], "readwrite");
|
907
824
|
await tx.objectStore(store).put(value, sanitzeKey(key));
|
908
825
|
await tx.done;
|
909
826
|
});
|
910
827
|
}
|
911
|
-
async delete(
|
828
|
+
async delete(url) {
|
912
829
|
return exception2Result(async () => {
|
913
|
-
const key = getKey(
|
914
|
-
const store = getStore(
|
915
|
-
this.logger.Debug().Url(
|
916
|
-
const tx = this.
|
830
|
+
const key = getKey(url, this.logger);
|
831
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
832
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
|
833
|
+
const tx = this._db.transaction([store], "readwrite");
|
917
834
|
await tx.objectStore(store).delete(sanitzeKey(key));
|
918
835
|
await tx.done;
|
919
|
-
return
|
836
|
+
return import_cement9.Result.Ok(void 0);
|
920
837
|
});
|
921
838
|
}
|
922
839
|
};
|
923
|
-
IndexDBDataGateway = class extends IndexDBGateway {
|
924
|
-
constructor(logger) {
|
925
|
-
super(ensureLogger(logger, "IndexDBDataGateway", {}));
|
926
|
-
}
|
927
|
-
buildUrl(baseUrl, key) {
|
928
|
-
const url2 = new URL(baseUrl.toString());
|
929
|
-
url2.searchParams.set("key", key);
|
930
|
-
return Promise.resolve(import_cement7.Result.Ok(url2));
|
931
|
-
}
|
932
|
-
};
|
933
|
-
IndexDBWalGateway = class extends IndexDBGateway {
|
934
|
-
constructor(logger) {
|
935
|
-
super(ensureLogger(logger, "IndexDBWalGateway", {}));
|
936
|
-
}
|
937
|
-
buildUrl(baseUrl, key) {
|
938
|
-
const url2 = new URL(baseUrl.toString());
|
939
|
-
url2.searchParams.set("key", key);
|
940
|
-
return Promise.resolve(import_cement7.Result.Ok(url2));
|
941
|
-
}
|
942
|
-
};
|
943
|
-
IndexDBMetaGateway = class extends IndexDBGateway {
|
944
|
-
constructor(logger) {
|
945
|
-
super(ensureLogger(logger, "IndexDBDataGateway", {}));
|
946
|
-
this.branches = /* @__PURE__ */ new Set();
|
947
|
-
}
|
948
|
-
async buildUrl(baseUrl, key) {
|
949
|
-
const url2 = new URL(baseUrl.toString());
|
950
|
-
this.branches.add(key);
|
951
|
-
url2.searchParams.set("key", key);
|
952
|
-
return import_cement7.Result.Ok(url2);
|
953
|
-
}
|
954
|
-
};
|
955
|
-
txtEncoder = new TextEncoder();
|
956
840
|
IndexDBTestStore = class {
|
957
|
-
constructor(
|
958
|
-
this.
|
841
|
+
constructor(sthis) {
|
842
|
+
this.sthis = sthis;
|
843
|
+
this.logger = ensureLogger(sthis, "IndexDBTestStore", {});
|
959
844
|
}
|
960
|
-
async get(
|
961
|
-
const
|
962
|
-
const store = getStore(
|
845
|
+
async get(url, key) {
|
846
|
+
const ic = await connectIdb(url, this.sthis);
|
847
|
+
const store = getStore(ic.url, this.sthis, joinDBName).name;
|
963
848
|
this.logger.Debug().Str("key", key).Str("store", store).Msg("getting");
|
964
|
-
let bytes = await db.get(store, sanitzeKey(key));
|
849
|
+
let bytes = await ic.db.get(store, sanitzeKey(key));
|
965
850
|
this.logger.Debug().Str("key", key).Str("store", store).Int("len", bytes.length).Msg("got");
|
966
851
|
if (typeof bytes === "string") {
|
967
|
-
bytes =
|
852
|
+
bytes = this.sthis.txt.encode(bytes);
|
968
853
|
}
|
969
854
|
return bytes;
|
970
855
|
}
|
@@ -978,11 +863,16 @@ __export(src_exports, {
|
|
978
863
|
CRDT: () => CRDT,
|
979
864
|
Database: () => Database,
|
980
865
|
Index: () => Index,
|
866
|
+
NotFoundError: () => NotFoundError,
|
981
867
|
PACKAGE_VERSION: () => PACKAGE_VERSION,
|
982
|
-
Result: () =>
|
868
|
+
Result: () => import_cement.Result,
|
869
|
+
UInt8ArrayEqual: () => UInt8ArrayEqual,
|
983
870
|
blockstore: () => blockstore_exports,
|
984
871
|
bs: () => blockstore_exports,
|
872
|
+
dataDir: () => dataDir,
|
985
873
|
ensureLogger: () => ensureLogger,
|
874
|
+
ensureSuperLog: () => ensureSuperLog,
|
875
|
+
ensureSuperThis: () => ensureSuperThis,
|
986
876
|
exception2Result: () => exception2Result,
|
987
877
|
exceptionWrapper: () => exceptionWrapper,
|
988
878
|
falsyToUndef: () => falsyToUndef,
|
@@ -992,6 +882,7 @@ __export(src_exports, {
|
|
992
882
|
getStore: () => getStore,
|
993
883
|
index: () => index,
|
994
884
|
isFalsy: () => isFalsy,
|
885
|
+
isNotFoundError: () => isNotFoundError,
|
995
886
|
rt: () => runtime_exports,
|
996
887
|
runtime: () => runtime_exports,
|
997
888
|
throwFalsy: () => throwFalsy
|
@@ -999,8 +890,7 @@ __export(src_exports, {
|
|
999
890
|
module.exports = __toCommonJS(src_exports);
|
1000
891
|
|
1001
892
|
// src/database.ts
|
1002
|
-
var
|
1003
|
-
var import_cement11 = require("@adviser/cement");
|
893
|
+
var import_cement13 = require("@adviser/cement");
|
1004
894
|
|
1005
895
|
// src/write-queue.ts
|
1006
896
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -1043,13 +933,83 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
1043
933
|
}
|
1044
934
|
|
1045
935
|
// src/crdt.ts
|
1046
|
-
var
|
936
|
+
var import_cement12 = require("@adviser/cement");
|
937
|
+
|
938
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
939
|
+
var block_exports = {};
|
940
|
+
__export(block_exports, {
|
941
|
+
Block: () => Block,
|
942
|
+
create: () => create,
|
943
|
+
createUnsafe: () => createUnsafe,
|
944
|
+
decode: () => decode,
|
945
|
+
encode: () => encode
|
946
|
+
});
|
947
|
+
var import_multiformats = require("multiformats");
|
948
|
+
var import_block = require("multiformats/block");
|
949
|
+
var Block = import_block.Block;
|
950
|
+
async function decode({
|
951
|
+
bytes,
|
952
|
+
codec: codec3,
|
953
|
+
hasher: hasher7
|
954
|
+
}) {
|
955
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
956
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
957
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
958
|
+
const hash = await hasher7.digest(bytes);
|
959
|
+
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
960
|
+
return new import_block.Block({ value, bytes, cid });
|
961
|
+
}
|
962
|
+
async function encode({
|
963
|
+
value,
|
964
|
+
codec: codec3,
|
965
|
+
hasher: hasher7
|
966
|
+
}) {
|
967
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
968
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
969
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
970
|
+
const hash = await hasher7.digest(bytes);
|
971
|
+
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
972
|
+
return new import_block.Block({ value, bytes, cid });
|
973
|
+
}
|
974
|
+
async function create({
|
975
|
+
bytes,
|
976
|
+
cid,
|
977
|
+
hasher: hasher7,
|
978
|
+
codec: codec3
|
979
|
+
}) {
|
980
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
981
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
982
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
983
|
+
const hash = await hasher7.digest(bytes);
|
984
|
+
if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
|
985
|
+
throw new Error("CID hash does not match bytes");
|
986
|
+
}
|
987
|
+
return createUnsafe({
|
988
|
+
bytes,
|
989
|
+
cid,
|
990
|
+
value,
|
991
|
+
codec: codec3
|
992
|
+
});
|
993
|
+
}
|
994
|
+
async function createUnsafe({
|
995
|
+
bytes,
|
996
|
+
cid,
|
997
|
+
value: maybeValue,
|
998
|
+
codec: codec3
|
999
|
+
}) {
|
1000
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
1001
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
1002
|
+
return new Block({
|
1003
|
+
cid,
|
1004
|
+
bytes,
|
1005
|
+
value
|
1006
|
+
});
|
1007
|
+
}
|
1047
1008
|
|
1048
1009
|
// src/crdt-helpers.ts
|
1049
|
-
var import_block6 = require("multiformats/block");
|
1050
1010
|
var import_link = require("multiformats/link");
|
1051
|
-
var
|
1052
|
-
var
|
1011
|
+
var import_sha25 = require("multiformats/hashes/sha2");
|
1012
|
+
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
1053
1013
|
var import_crdt = require("@web3-storage/pail/crdt");
|
1054
1014
|
var import_clock2 = require("@web3-storage/pail/clock");
|
1055
1015
|
var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
|
@@ -1060,525 +1020,149 @@ __export(blockstore_exports, {
|
|
1060
1020
|
BaseBlockstore: () => BaseBlockstore,
|
1061
1021
|
CarTransaction: () => CarTransaction,
|
1062
1022
|
CompactionFetcher: () => CompactionFetcher,
|
1063
|
-
ConnectREST: () => ConnectREST,
|
1064
1023
|
ConnectionBase: () => ConnectionBase,
|
1065
|
-
DataStore: () => DataStore,
|
1066
1024
|
EncryptedBlockstore: () => EncryptedBlockstore,
|
1067
|
-
|
1025
|
+
FragmentGateway: () => FragmentGateway,
|
1068
1026
|
Loader: () => Loader,
|
1069
|
-
|
1070
|
-
|
1071
|
-
RemoteWAL: () => RemoteWAL,
|
1072
|
-
isNotFoundError: () => isNotFoundError,
|
1027
|
+
ensureStart: () => ensureStart,
|
1028
|
+
getGatewayFromURL: () => getGatewayFromURL,
|
1073
1029
|
parseCarFile: () => parseCarFile,
|
1074
1030
|
registerStoreProtocol: () => registerStoreProtocol,
|
1075
1031
|
testStoreFactory: () => testStoreFactory,
|
1076
|
-
|
1077
|
-
|
1032
|
+
toCIDBlock: () => toCIDBlock,
|
1033
|
+
toStoreRuntime: () => toStoreRuntime
|
1078
1034
|
});
|
1079
1035
|
|
1080
|
-
// src/blockstore/
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1036
|
+
// src/blockstore/types.ts
|
1037
|
+
function toCIDBlock(block) {
|
1038
|
+
return block;
|
1039
|
+
}
|
1084
1040
|
|
1085
|
-
// src/blockstore/
|
1086
|
-
|
1087
|
-
var TaskManager = class {
|
1088
|
-
constructor(loader) {
|
1089
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
1090
|
-
this.queue = [];
|
1091
|
-
this.isProcessing = false;
|
1092
|
-
this.loader = loader;
|
1093
|
-
this.logger = ensureLogger(loader.logger, "TaskManager");
|
1094
|
-
}
|
1095
|
-
async handleEvent(eventBlock) {
|
1096
|
-
const cid = eventBlock.cid.toString();
|
1097
|
-
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
1098
|
-
for (const parent of parents) {
|
1099
|
-
this.eventsWeHandled.add(parent);
|
1100
|
-
}
|
1101
|
-
this.queue.push({ cid, eventBlock, retries: 0 });
|
1102
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
1103
|
-
void this.processQueue();
|
1104
|
-
}
|
1105
|
-
async processQueue() {
|
1106
|
-
if (this.isProcessing) return;
|
1107
|
-
this.isProcessing = true;
|
1108
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1109
|
-
const first = filteredQueue[0];
|
1110
|
-
if (!first) {
|
1111
|
-
return;
|
1112
|
-
}
|
1113
|
-
try {
|
1114
|
-
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
1115
|
-
this.eventsWeHandled.add(first.cid);
|
1116
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1117
|
-
} catch (err) {
|
1118
|
-
if (first.retries++ > 3) {
|
1119
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
1120
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
1121
|
-
}
|
1122
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
1123
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
1124
|
-
} finally {
|
1125
|
-
this.isProcessing = false;
|
1126
|
-
if (this.queue.length > 0) {
|
1127
|
-
void this.processQueue();
|
1128
|
-
}
|
1129
|
-
}
|
1130
|
-
}
|
1131
|
-
};
|
1041
|
+
// src/blockstore/store-factory.ts
|
1042
|
+
var import_cement10 = require("@adviser/cement");
|
1132
1043
|
|
1133
|
-
// src/
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
|
1166
|
-
|
1044
|
+
// src/runtime/files.ts
|
1045
|
+
var files_exports = {};
|
1046
|
+
__export(files_exports, {
|
1047
|
+
decodeFile: () => decodeFile,
|
1048
|
+
encodeFile: () => encodeFile
|
1049
|
+
});
|
1050
|
+
var UnixFS = __toESM(require("@ipld/unixfs"), 1);
|
1051
|
+
var raw = __toESM(require("multiformats/codecs/raw"), 1);
|
1052
|
+
var import_fixed = require("@ipld/unixfs/file/chunker/fixed");
|
1053
|
+
var import_balanced = require("@ipld/unixfs/file/layout/balanced");
|
1054
|
+
var import_ipfs_unixfs_exporter = require("ipfs-unixfs-exporter");
|
1055
|
+
var queuingStrategy = UnixFS.withCapacity();
|
1056
|
+
var settings = UnixFS.configure({
|
1057
|
+
fileChunkEncoder: raw,
|
1058
|
+
smallFileEncoder: raw,
|
1059
|
+
chunker: (0, import_fixed.withMaxChunkSize)(1024 * 1024),
|
1060
|
+
fileLayout: (0, import_balanced.withWidth)(1024)
|
1061
|
+
});
|
1062
|
+
async function collect(collectable) {
|
1063
|
+
const chunks = [];
|
1064
|
+
await collectable.pipeTo(
|
1065
|
+
new WritableStream({
|
1066
|
+
write(chunk) {
|
1067
|
+
chunks.push(chunk);
|
1068
|
+
}
|
1069
|
+
})
|
1070
|
+
);
|
1071
|
+
return chunks;
|
1072
|
+
}
|
1073
|
+
async function encodeFile(blob) {
|
1074
|
+
const readable = createFileEncoderStream(blob);
|
1075
|
+
const blocks = await collect(readable);
|
1076
|
+
return { cid: blocks.at(-1).cid, blocks };
|
1077
|
+
}
|
1078
|
+
async function decodeFile(blocks, cid, meta) {
|
1079
|
+
const entry = await (0, import_ipfs_unixfs_exporter.exporter)(cid.toString(), blocks, { length: meta.size });
|
1080
|
+
const chunks = [];
|
1081
|
+
for await (const chunk of entry.content()) {
|
1082
|
+
chunks.push(chunk);
|
1167
1083
|
}
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1084
|
+
return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
|
1085
|
+
}
|
1086
|
+
function createFileEncoderStream(blob) {
|
1087
|
+
const { readable, writable } = new TransformStream({}, queuingStrategy);
|
1088
|
+
const unixfsWriter = UnixFS.createWriter({ writable, settings });
|
1089
|
+
const fileBuilder = new UnixFSFileBuilder("", blob);
|
1090
|
+
void (async () => {
|
1091
|
+
await fileBuilder.finalize(unixfsWriter);
|
1092
|
+
await unixfsWriter.close();
|
1093
|
+
})();
|
1094
|
+
return readable;
|
1095
|
+
}
|
1096
|
+
var UnixFSFileBuilder = class {
|
1097
|
+
#file;
|
1098
|
+
constructor(name, file) {
|
1099
|
+
this.name = name;
|
1100
|
+
this.#file = file;
|
1101
|
+
}
|
1102
|
+
async finalize(writer) {
|
1103
|
+
const unixfsFileWriter = UnixFS.createFileWriter(writer);
|
1104
|
+
await this.#file.stream().pipeTo(
|
1105
|
+
new WritableStream({
|
1106
|
+
async write(chunk) {
|
1107
|
+
await unixfsFileWriter.write(chunk);
|
1108
|
+
}
|
1109
|
+
})
|
1175
1110
|
);
|
1176
|
-
await
|
1177
|
-
return event;
|
1178
|
-
}
|
1179
|
-
async decodeEventBlock(bytes) {
|
1180
|
-
const event = await (0, import_clock.decodeEventBlock)(bytes);
|
1181
|
-
return event;
|
1111
|
+
return await unixfsFileWriter.close();
|
1182
1112
|
}
|
1183
|
-
// move this stuff to connect
|
1184
|
-
// async getDashboardURL(compact = true) {
|
1185
|
-
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
1186
|
-
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
1187
|
-
// // if (compact) {
|
1188
|
-
// // await this.compact()
|
1189
|
-
// // }
|
1190
|
-
// const currents = await this.loader?.metaStore?.load()
|
1191
|
-
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
1192
|
-
// if (currents.length > 1)
|
1193
|
-
// throw new Error("Can't sync database with split heads: make an update first")
|
1194
|
-
// const current = currents[0]
|
1195
|
-
// const params = {
|
1196
|
-
// car: current.car.toString()
|
1197
|
-
// }
|
1198
|
-
// if (current.key) {
|
1199
|
-
// // @ts-ignore
|
1200
|
-
// params.key = current.key.toString()
|
1201
|
-
// }
|
1202
|
-
// // @ts-ignore
|
1203
|
-
// if (this.name) {
|
1204
|
-
// // @ts-ignore
|
1205
|
-
// params.name = this.name
|
1206
|
-
// }
|
1207
|
-
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
1208
|
-
// console.log('Import to dashboard: ' + url.toString())
|
1209
|
-
// return url
|
1210
|
-
// }
|
1211
|
-
// openDashboard() {
|
1212
|
-
// void this.getDashboardURL().then(url => {
|
1213
|
-
// if (url) window.open(url.toString(), '_blank')
|
1214
|
-
// })
|
1215
|
-
// }
|
1216
1113
|
};
|
1217
1114
|
|
1218
|
-
// src/blockstore/
|
1219
|
-
|
1220
|
-
var
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
throw this.logger.Error().Msg("failed to upload data " + done.statusText);
|
1231
|
-
}
|
1232
|
-
}
|
1233
|
-
async dataDownload(params) {
|
1234
|
-
const { car } = params;
|
1235
|
-
const fetchFromUrl = new URL(`/cars/${car.toString()}.car`, this.baseUrl);
|
1236
|
-
const response = await fetch(fetchFromUrl);
|
1237
|
-
if (!response.ok) {
|
1238
|
-
return void 0;
|
1239
|
-
}
|
1240
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
1241
|
-
return bytes;
|
1115
|
+
// src/blockstore/store.ts
|
1116
|
+
var import_p_limit2 = __toESM(require("p-limit"), 1);
|
1117
|
+
var import_dag_json = require("@ipld/dag-json");
|
1118
|
+
var import_cement7 = require("@adviser/cement");
|
1119
|
+
|
1120
|
+
// src/types.ts
|
1121
|
+
function isFalsy(value) {
|
1122
|
+
return value === false && value === null && value === void 0;
|
1123
|
+
}
|
1124
|
+
function throwFalsy(value) {
|
1125
|
+
if (isFalsy(value)) {
|
1126
|
+
throw new Error("value is Falsy");
|
1242
1127
|
}
|
1243
|
-
|
1244
|
-
|
1128
|
+
return value;
|
1129
|
+
}
|
1130
|
+
function falsyToUndef(value) {
|
1131
|
+
if (isFalsy(value)) {
|
1245
1132
|
return void 0;
|
1246
1133
|
}
|
1247
|
-
|
1248
|
-
|
1249
|
-
return [];
|
1250
|
-
}
|
1251
|
-
};
|
1252
|
-
|
1253
|
-
// src/blockstore/store-factory.ts
|
1254
|
-
var import_cement8 = require("@adviser/cement");
|
1255
|
-
init_data_dir();
|
1256
|
-
init_files();
|
1134
|
+
return value;
|
1135
|
+
}
|
1257
1136
|
|
1258
1137
|
// src/blockstore/store.ts
|
1259
|
-
var import_p_limit2 = __toESM(require("p-limit"), 1);
|
1260
|
-
var import_dag_json = require("@ipld/dag-json");
|
1261
|
-
var import_cement5 = require("@adviser/cement");
|
1262
|
-
init_types();
|
1263
|
-
init_gateway();
|
1264
1138
|
init_utils();
|
1265
1139
|
|
1266
1140
|
// src/blockstore/loader.ts
|
1267
1141
|
var import_p_limit = __toESM(require("p-limit"), 1);
|
1268
1142
|
var import_car = require("@ipld/car");
|
1269
|
-
var
|
1270
|
-
|
1271
|
-
// src/blockstore/types.ts
|
1272
|
-
function toCIDBlock(block) {
|
1273
|
-
return block;
|
1274
|
-
}
|
1143
|
+
var import_cement6 = require("@adviser/cement");
|
1275
1144
|
|
1276
1145
|
// src/blockstore/loader-helpers.ts
|
1277
|
-
var import_block2 = require("multiformats/block");
|
1278
1146
|
var import_sha2 = require("multiformats/hashes/sha2");
|
1279
|
-
var
|
1280
|
-
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
1281
|
-
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
1282
|
-
async function encodeCarFile(roots, t) {
|
1283
|
-
let size = 0;
|
1284
|
-
const headerSize = CBW.headerLength({ roots });
|
1285
|
-
size += headerSize;
|
1286
|
-
for (const { cid, bytes } of t.entries()) {
|
1287
|
-
size += CBW.blockLength({ cid, bytes });
|
1288
|
-
}
|
1289
|
-
const buffer = new Uint8Array(size);
|
1290
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
1291
|
-
for (const r of roots) {
|
1292
|
-
writer.addRoot(r);
|
1293
|
-
}
|
1294
|
-
for (const { cid, bytes } of t.entries()) {
|
1295
|
-
writer.write({ cid, bytes });
|
1296
|
-
}
|
1297
|
-
writer.close();
|
1298
|
-
return await (0, import_block2.encode)({ value: writer.bytes, hasher: import_sha2.sha256, codec: raw2 });
|
1299
|
-
}
|
1300
|
-
async function encodeCarHeader(fp) {
|
1301
|
-
return await (0, import_block2.encode)({
|
1302
|
-
value: { fp },
|
1303
|
-
hasher: import_sha2.sha256,
|
1304
|
-
codec
|
1305
|
-
});
|
1306
|
-
}
|
1147
|
+
var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
|
1307
1148
|
async function parseCarFile(reader, logger) {
|
1308
1149
|
const roots = await reader.getRoots();
|
1309
1150
|
const header = await reader.get(roots[0]);
|
1310
1151
|
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
1311
|
-
const
|
1312
|
-
const fpvalue = value;
|
1152
|
+
const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
|
1153
|
+
const fpvalue = dec.value;
|
1313
1154
|
if (fpvalue && !fpvalue.fp) {
|
1314
1155
|
throw logger.Error().Msg("missing fp").AsError();
|
1315
1156
|
}
|
1316
1157
|
return fpvalue.fp;
|
1317
1158
|
}
|
1318
1159
|
|
1319
|
-
// src/blockstore/encrypt-helpers.ts
|
1320
|
-
var import_sha22 = require("multiformats/hashes/sha2");
|
1321
|
-
var import_multiformats2 = require("multiformats");
|
1322
|
-
var import_block3 = require("multiformats/block");
|
1323
|
-
var dagcbor = __toESM(require("@ipld/dag-cbor"), 1);
|
1324
|
-
var import_block4 = require("@web3-storage/pail/block");
|
1325
|
-
var import_utils6 = require("prolly-trees/utils");
|
1326
|
-
var import_cache = require("prolly-trees/cache");
|
1327
|
-
var import_cid_set = require("prolly-trees/cid-set");
|
1328
|
-
|
1329
|
-
// src/blockstore/encrypt-codec.ts
|
1330
|
-
var import_multiformats = require("multiformats");
|
1331
|
-
init_utils();
|
1332
|
-
function makeCodec(ilogger, crypto2, randomBytes2) {
|
1333
|
-
const logger = ensureLogger(ilogger, "makeCodec");
|
1334
|
-
const enc32 = (value) => {
|
1335
|
-
value = +value;
|
1336
|
-
const buff = new Uint8Array(4);
|
1337
|
-
buff[3] = value >>> 24;
|
1338
|
-
buff[2] = value >>> 16;
|
1339
|
-
buff[1] = value >>> 8;
|
1340
|
-
buff[0] = value & 255;
|
1341
|
-
return buff;
|
1342
|
-
};
|
1343
|
-
const readUInt32LE = (buffer) => {
|
1344
|
-
const offset = buffer.byteLength - 4;
|
1345
|
-
return (buffer[offset] | buffer[offset + 1] << 8 | buffer[offset + 2] << 16) + buffer[offset + 3] * 16777216;
|
1346
|
-
};
|
1347
|
-
const concat = (buffers) => {
|
1348
|
-
const uint8Arrays = buffers.map((b) => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
|
1349
|
-
const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
|
1350
|
-
const result = new Uint8Array(totalLength);
|
1351
|
-
let offset = 0;
|
1352
|
-
for (const arr of uint8Arrays) {
|
1353
|
-
result.set(arr, offset);
|
1354
|
-
offset += arr.length;
|
1355
|
-
}
|
1356
|
-
return result;
|
1357
|
-
};
|
1358
|
-
const encode4 = ({ iv, bytes }) => concat([iv, bytes]);
|
1359
|
-
const decode4 = (bytes) => {
|
1360
|
-
const iv = bytes.subarray(0, 12);
|
1361
|
-
bytes = bytes.slice(12);
|
1362
|
-
return { iv, bytes };
|
1363
|
-
};
|
1364
|
-
const code = 3145728 + 1337;
|
1365
|
-
async function subtleKey(key) {
|
1366
|
-
return await crypto2.importKey(
|
1367
|
-
"raw",
|
1368
|
-
// raw or jwk
|
1369
|
-
key,
|
1370
|
-
// raw data
|
1371
|
-
"AES-GCM",
|
1372
|
-
false,
|
1373
|
-
// extractable
|
1374
|
-
["encrypt", "decrypt"]
|
1375
|
-
);
|
1376
|
-
}
|
1377
|
-
const decrypt = async ({ key, value }) => {
|
1378
|
-
const { bytes: inBytes, iv } = value;
|
1379
|
-
const cryKey = await subtleKey(key);
|
1380
|
-
const deBytes = await crypto2.decrypt(
|
1381
|
-
{
|
1382
|
-
name: "AES-GCM",
|
1383
|
-
iv,
|
1384
|
-
tagLength: 128
|
1385
|
-
},
|
1386
|
-
cryKey,
|
1387
|
-
inBytes
|
1388
|
-
);
|
1389
|
-
const bytes = new Uint8Array(deBytes);
|
1390
|
-
const len = readUInt32LE(bytes.subarray(0, 4));
|
1391
|
-
const cid = import_multiformats.CID.decode(bytes.subarray(4, 4 + len));
|
1392
|
-
return { cid, bytes: bytes.subarray(4 + len) };
|
1393
|
-
};
|
1394
|
-
const encrypt = async ({ key, cid, bytes }) => {
|
1395
|
-
const len = enc32(cid.bytes.byteLength);
|
1396
|
-
const iv = randomBytes2(12);
|
1397
|
-
const msg = concat([len, cid.bytes, bytes]);
|
1398
|
-
try {
|
1399
|
-
const cryKey = await subtleKey(key);
|
1400
|
-
const deBytes = await crypto2.encrypt(
|
1401
|
-
{
|
1402
|
-
name: "AES-GCM",
|
1403
|
-
iv,
|
1404
|
-
tagLength: 128
|
1405
|
-
},
|
1406
|
-
cryKey,
|
1407
|
-
msg
|
1408
|
-
);
|
1409
|
-
bytes = new Uint8Array(deBytes);
|
1410
|
-
} catch (e) {
|
1411
|
-
throw logger.Error().Err(e).Msg("encrypt failed").AsError();
|
1412
|
-
}
|
1413
|
-
return { value: { bytes, iv } };
|
1414
|
-
};
|
1415
|
-
const cryptoFn = (key) => {
|
1416
|
-
return { encrypt: (opts) => encrypt({ ...opts, key }), decrypt: (opts) => decrypt({ ...opts, key }) };
|
1417
|
-
};
|
1418
|
-
const name = "jchris@encrypted-block:aes-gcm";
|
1419
|
-
return { encode: encode4, decode: decode4, code, name, encrypt, decrypt, crypto: cryptoFn };
|
1420
|
-
}
|
1421
|
-
|
1422
|
-
// src/blockstore/encrypt-helpers.ts
|
1423
|
-
function carLogIncludesGroup(list, cidMatch) {
|
1424
|
-
return list.some((cid) => {
|
1425
|
-
return cid.toString() === cidMatch.toString();
|
1426
|
-
});
|
1427
|
-
}
|
1428
|
-
function makeEncDec(logger, crypto2, randomBytes2) {
|
1429
|
-
const codec4 = makeCodec(logger, crypto2, randomBytes2);
|
1430
|
-
const encrypt = async function* ({
|
1431
|
-
get: get2,
|
1432
|
-
cids,
|
1433
|
-
hasher: hasher4,
|
1434
|
-
key,
|
1435
|
-
cache: cache3,
|
1436
|
-
chunker: chunker2,
|
1437
|
-
root: root3
|
1438
|
-
}) {
|
1439
|
-
const set = /* @__PURE__ */ new Set();
|
1440
|
-
let eroot;
|
1441
|
-
if (!carLogIncludesGroup(cids, root3)) cids.push(root3);
|
1442
|
-
for (const cid of cids) {
|
1443
|
-
const unencrypted = await get2(cid);
|
1444
|
-
if (!unencrypted) throw logger.Error().Ref("cid", cid).Msg("missing cid block").AsError();
|
1445
|
-
const encrypted = await codec4.encrypt({ ...unencrypted, key });
|
1446
|
-
const block2 = await (0, import_block3.encode)({ ...encrypted, codec: codec4, hasher: hasher4 });
|
1447
|
-
yield block2;
|
1448
|
-
set.add(block2.cid.toString());
|
1449
|
-
if (unencrypted.cid.equals(root3)) eroot = block2.cid;
|
1450
|
-
}
|
1451
|
-
if (!eroot) throw logger.Error().Msg("cids does not include root").AsError();
|
1452
|
-
const list = [...set].map((s) => import_multiformats2.CID.parse(s));
|
1453
|
-
let last;
|
1454
|
-
for await (const node of (0, import_cid_set.create)({ list, get: get2, cache: cache3, chunker: chunker2, hasher: hasher4, codec: dagcbor })) {
|
1455
|
-
const block2 = await node.block;
|
1456
|
-
yield block2;
|
1457
|
-
last = block2;
|
1458
|
-
}
|
1459
|
-
if (!last) throw logger.Error().Msg("missing last block").AsError();
|
1460
|
-
const head = [eroot, last.cid];
|
1461
|
-
const block = await (0, import_block3.encode)({ value: head, codec: dagcbor, hasher: hasher4 });
|
1462
|
-
yield block;
|
1463
|
-
};
|
1464
|
-
const decrypt = async function* ({
|
1465
|
-
root: root3,
|
1466
|
-
get: get2,
|
1467
|
-
key,
|
1468
|
-
cache: cache3,
|
1469
|
-
chunker: chunker2,
|
1470
|
-
hasher: hasher4
|
1471
|
-
}) {
|
1472
|
-
const getWithDecode = async (cid) => get2(cid).then(async (block) => {
|
1473
|
-
if (!block) return;
|
1474
|
-
const decoded = await (0, import_block3.decode)({ ...block, codec: dagcbor, hasher: hasher4 });
|
1475
|
-
return decoded;
|
1476
|
-
});
|
1477
|
-
const getWithDecrypt = async (cid) => get2(cid).then(async (block) => {
|
1478
|
-
if (!block) return;
|
1479
|
-
const decoded = await (0, import_block3.decode)({ ...block, codec: codec4, hasher: hasher4 });
|
1480
|
-
return decoded;
|
1481
|
-
});
|
1482
|
-
const decodedRoot = await getWithDecode(root3);
|
1483
|
-
if (!decodedRoot) throw logger.Error().Msg("missing root").AsError();
|
1484
|
-
if (!decodedRoot.bytes) throw logger.Error().Msg("missing bytes").AsError();
|
1485
|
-
const {
|
1486
|
-
value: [eroot, tree]
|
1487
|
-
} = decodedRoot;
|
1488
|
-
const rootBlock = await get2(eroot);
|
1489
|
-
if (!rootBlock) throw logger.Error().Msg("missing root block").AsError();
|
1490
|
-
const cidset = await (0, import_cid_set.load)({ cid: tree, get: getWithDecode, cache: cache3, chunker: chunker2, codec: codec4, hasher: hasher4 });
|
1491
|
-
const { result: nodes } = await cidset.getAllEntries();
|
1492
|
-
const unwrap = async (eblock) => {
|
1493
|
-
if (!eblock) throw logger.Error().Msg("missing block").AsError();
|
1494
|
-
if (!eblock.value) {
|
1495
|
-
eblock = await (0, import_block3.decode)({ ...eblock, codec: codec4, hasher: hasher4 });
|
1496
|
-
if (!eblock.value) throw logger.Error().Msg("missing value").AsError();
|
1497
|
-
}
|
1498
|
-
const { bytes, cid } = await codec4.decrypt({ ...eblock, key }).catch((e) => {
|
1499
|
-
throw e;
|
1500
|
-
});
|
1501
|
-
const block = await (0, import_block3.create)({ cid, bytes, hasher: hasher4, codec: codec4 });
|
1502
|
-
return block;
|
1503
|
-
};
|
1504
|
-
const promises = [];
|
1505
|
-
for (const { cid } of nodes) {
|
1506
|
-
if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap));
|
1507
|
-
}
|
1508
|
-
yield* promises;
|
1509
|
-
yield unwrap(rootBlock);
|
1510
|
-
};
|
1511
|
-
return { encrypt, decrypt };
|
1512
|
-
}
|
1513
|
-
var chunker = (0, import_utils6.bf)(30);
|
1514
|
-
function hexStringToUint8Array(hexString) {
|
1515
|
-
const length = hexString.length;
|
1516
|
-
const uint8Array = new Uint8Array(length / 2);
|
1517
|
-
for (let i = 0; i < length; i += 2) {
|
1518
|
-
uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
|
1519
|
-
}
|
1520
|
-
return uint8Array;
|
1521
|
-
}
|
1522
|
-
async function encryptedEncodeCarFile(logger, crypto2, key, rootCid, t) {
|
1523
|
-
const encryptionKey = hexStringToUint8Array(key);
|
1524
|
-
const encryptedBlocks = new import_block4.MemoryBlockstore();
|
1525
|
-
const cidsToEncrypt = [];
|
1526
|
-
for (const { cid, bytes } of t.entries()) {
|
1527
|
-
cidsToEncrypt.push(cid);
|
1528
|
-
const g = await t.get(cid);
|
1529
|
-
if (!g) throw logger.Error().Ref("cid", cid).Int("bytes", bytes.length).Msg("missing cid block").AsError();
|
1530
|
-
}
|
1531
|
-
let last = null;
|
1532
|
-
const { encrypt } = makeEncDec(logger, crypto2, crypto2.randomBytes);
|
1533
|
-
for await (const block of encrypt({
|
1534
|
-
cids: cidsToEncrypt,
|
1535
|
-
get: t.get.bind(t),
|
1536
|
-
key: encryptionKey,
|
1537
|
-
hasher: import_sha22.sha256,
|
1538
|
-
chunker,
|
1539
|
-
cache: import_cache.nocache,
|
1540
|
-
root: rootCid
|
1541
|
-
})) {
|
1542
|
-
await encryptedBlocks.put(block.cid, block.bytes);
|
1543
|
-
last = block;
|
1544
|
-
}
|
1545
|
-
if (!last) throw logger.Error().Msg("no blocks encrypted").AsError();
|
1546
|
-
const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
|
1547
|
-
return encryptedCar;
|
1548
|
-
}
|
1549
|
-
async function decodeEncryptedCar(logger, crypto2, key, reader) {
|
1550
|
-
const roots = await reader.getRoots();
|
1551
|
-
const root3 = roots[0];
|
1552
|
-
return await decodeCarBlocks(logger, crypto2, root3, reader.get.bind(reader), key);
|
1553
|
-
}
|
1554
|
-
async function decodeCarBlocks(logger, crypto2, root3, get2, keyMaterial) {
|
1555
|
-
const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
|
1556
|
-
const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength);
|
1557
|
-
const decryptedBlocks = new import_block4.MemoryBlockstore();
|
1558
|
-
let last = null;
|
1559
|
-
const { decrypt } = makeEncDec(logger, crypto2, crypto2.randomBytes);
|
1560
|
-
for await (const block of decrypt({
|
1561
|
-
root: root3,
|
1562
|
-
get: get2,
|
1563
|
-
key: decryptionKey,
|
1564
|
-
hasher: import_sha22.sha256,
|
1565
|
-
chunker,
|
1566
|
-
cache: import_cache.nocache
|
1567
|
-
})) {
|
1568
|
-
await decryptedBlocks.put(block.cid, block.bytes);
|
1569
|
-
last = block;
|
1570
|
-
}
|
1571
|
-
if (!last) throw logger.Error().Msg("no blocks decrypted").AsError();
|
1572
|
-
return { blocks: decryptedBlocks, root: last.cid };
|
1573
|
-
}
|
1574
|
-
|
1575
1160
|
// src/blockstore/transaction.ts
|
1576
|
-
var
|
1577
|
-
|
1578
|
-
init_crypto();
|
1161
|
+
var import_block3 = require("@web3-storage/pail/block");
|
1162
|
+
var import_cement2 = require("@adviser/cement");
|
1579
1163
|
init_utils();
|
1580
|
-
var CarTransaction = class extends
|
1581
|
-
constructor(parent, opts = { add: true }) {
|
1164
|
+
var CarTransaction = class extends import_block3.MemoryBlockstore {
|
1165
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
1582
1166
|
super();
|
1583
1167
|
if (opts.add) {
|
1584
1168
|
parent.transactions.add(this);
|
@@ -1592,8 +1176,8 @@ var CarTransaction = class extends import_block5.MemoryBlockstore {
|
|
1592
1176
|
return super.get(cid);
|
1593
1177
|
}
|
1594
1178
|
};
|
1595
|
-
function defaultedBlockstoreRuntime(opts, component, ctx) {
|
1596
|
-
const logger = ensureLogger(
|
1179
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
1180
|
+
const logger = ensureLogger(sthis, component, ctx);
|
1597
1181
|
const store = opts.store || {};
|
1598
1182
|
return {
|
1599
1183
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
@@ -1610,22 +1194,24 @@ function defaultedBlockstoreRuntime(opts, component, ctx) {
|
|
1610
1194
|
threshold: 1e3 * 1e3,
|
1611
1195
|
...opts,
|
1612
1196
|
logger,
|
1613
|
-
|
1197
|
+
keyBag: opts.keyBag || {},
|
1198
|
+
crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
|
1614
1199
|
store,
|
1615
|
-
storeRuntime: toStoreRuntime(store,
|
1200
|
+
storeRuntime: toStoreRuntime(store, sthis)
|
1616
1201
|
};
|
1617
1202
|
}
|
1618
|
-
|
1203
|
+
function blockstoreFactory(sthis, opts) {
|
1619
1204
|
if (opts.name) {
|
1620
|
-
return new EncryptedBlockstore(opts);
|
1205
|
+
return new EncryptedBlockstore(sthis, opts);
|
1621
1206
|
} else {
|
1622
1207
|
return new BaseBlockstore(opts);
|
1623
1208
|
}
|
1624
|
-
}
|
1209
|
+
}
|
1625
1210
|
var BaseBlockstore = class {
|
1626
1211
|
constructor(ebOpts = {}) {
|
1627
1212
|
this.transactions = /* @__PURE__ */ new Set();
|
1628
|
-
this.
|
1213
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1214
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1629
1215
|
this.logger = this.ebOpts.logger;
|
1630
1216
|
}
|
1631
1217
|
// ready: Promise<void>;
|
@@ -1648,8 +1234,8 @@ var BaseBlockstore = class {
|
|
1648
1234
|
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
1649
1235
|
}
|
1650
1236
|
// TransactionMeta
|
1651
|
-
async transaction(fn, _opts
|
1652
|
-
const t = new CarTransaction(this);
|
1237
|
+
async transaction(fn, _opts) {
|
1238
|
+
const t = new CarTransaction(this, _opts);
|
1653
1239
|
const done = await fn(t);
|
1654
1240
|
this.lastTxMeta = done;
|
1655
1241
|
return { t, meta: done };
|
@@ -1666,16 +1252,16 @@ var BaseBlockstore = class {
|
|
1666
1252
|
}
|
1667
1253
|
};
|
1668
1254
|
var EncryptedBlockstore = class extends BaseBlockstore {
|
1669
|
-
constructor(ebOpts) {
|
1255
|
+
constructor(sthis, ebOpts) {
|
1670
1256
|
super(ebOpts);
|
1671
1257
|
this.compacting = false;
|
1672
|
-
this.logger = ensureLogger(
|
1258
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
1673
1259
|
const { name } = ebOpts;
|
1674
1260
|
if (!name) {
|
1675
1261
|
throw this.logger.Error().Msg("name required").AsError();
|
1676
1262
|
}
|
1677
1263
|
this.name = name;
|
1678
|
-
this.loader = new Loader(this.name, ebOpts);
|
1264
|
+
this.loader = new Loader(this.name, ebOpts, sthis);
|
1679
1265
|
}
|
1680
1266
|
ready() {
|
1681
1267
|
return this.loader.ready();
|
@@ -1706,10 +1292,13 @@ var EncryptedBlockstore = class extends BaseBlockstore {
|
|
1706
1292
|
}
|
1707
1293
|
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1708
1294
|
}
|
1709
|
-
async getFile(car, cid
|
1295
|
+
async getFile(car, cid) {
|
1710
1296
|
await this.ready();
|
1711
1297
|
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
1712
|
-
const reader = await this.loader.loadFileCar(
|
1298
|
+
const reader = await this.loader.loadFileCar(
|
1299
|
+
car
|
1300
|
+
/*, isPublic */
|
1301
|
+
);
|
1713
1302
|
const block = await reader.get(cid);
|
1714
1303
|
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1715
1304
|
return block.bytes;
|
@@ -1765,43 +1354,345 @@ var CompactionFetcher = class {
|
|
1765
1354
|
};
|
1766
1355
|
|
1767
1356
|
// src/blockstore/commit-queue.ts
|
1357
|
+
var import_cement3 = require("@adviser/cement");
|
1768
1358
|
var CommitQueue = class {
|
1769
1359
|
constructor() {
|
1770
1360
|
this.queue = [];
|
1771
1361
|
this.processing = false;
|
1362
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
1363
|
+
}
|
1364
|
+
waitIdle() {
|
1365
|
+
if (this.queue.length === 0 && !this.processing) {
|
1366
|
+
return Promise.resolve();
|
1367
|
+
}
|
1368
|
+
const fn = new import_cement3.Future();
|
1369
|
+
this._waitIdleItems.add(fn);
|
1370
|
+
return fn.asPromise();
|
1371
|
+
}
|
1372
|
+
async enqueue(fn) {
|
1373
|
+
return new Promise((resolve, reject) => {
|
1374
|
+
const queueFn = async () => {
|
1375
|
+
try {
|
1376
|
+
resolve(await fn());
|
1377
|
+
} catch (e) {
|
1378
|
+
reject(e);
|
1379
|
+
} finally {
|
1380
|
+
this.processing = false;
|
1381
|
+
this.processNext();
|
1382
|
+
}
|
1383
|
+
};
|
1384
|
+
this.queue.push(queueFn);
|
1385
|
+
if (!this.processing) {
|
1386
|
+
this.processNext();
|
1387
|
+
}
|
1388
|
+
});
|
1389
|
+
}
|
1390
|
+
processNext() {
|
1391
|
+
if (this.queue.length > 0 && !this.processing) {
|
1392
|
+
this.processing = true;
|
1393
|
+
const queueFn = this.queue.shift();
|
1394
|
+
if (queueFn) {
|
1395
|
+
queueFn().finally(() => {
|
1396
|
+
});
|
1397
|
+
}
|
1398
|
+
}
|
1399
|
+
if (this.queue.length === 0 && !this.processing) {
|
1400
|
+
const toResolve = Array.from(this._waitIdleItems);
|
1401
|
+
this._waitIdleItems.clear();
|
1402
|
+
toResolve.map((fn) => fn.resolve());
|
1403
|
+
}
|
1404
|
+
}
|
1405
|
+
};
|
1406
|
+
|
1407
|
+
// src/runtime/key-bag.ts
|
1408
|
+
var key_bag_exports = {};
|
1409
|
+
__export(key_bag_exports, {
|
1410
|
+
KeyBag: () => KeyBag,
|
1411
|
+
getKeyBag: () => getKeyBag,
|
1412
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
1413
|
+
});
|
1414
|
+
var import_cement5 = require("@adviser/cement");
|
1415
|
+
init_utils();
|
1416
|
+
var import_base582 = require("multiformats/bases/base58");
|
1417
|
+
var KeyBag = class {
|
1418
|
+
constructor(rt) {
|
1419
|
+
this.rt = rt;
|
1420
|
+
this._seq = new import_cement5.ResolveSeq();
|
1421
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag", {
|
1422
|
+
id: rt.id()
|
1423
|
+
});
|
1424
|
+
this.logger.Debug().Msg("KeyBag created");
|
1425
|
+
}
|
1426
|
+
async subtleKey(key) {
|
1427
|
+
return await this.rt.crypto.importKey(
|
1428
|
+
"raw",
|
1429
|
+
// raw or jwk
|
1430
|
+
import_base582.base58btc.decode(key),
|
1431
|
+
// hexStringToUint8Array(key), // raw data
|
1432
|
+
"AES-GCM",
|
1433
|
+
false,
|
1434
|
+
// extractable
|
1435
|
+
["encrypt", "decrypt"]
|
1436
|
+
);
|
1437
|
+
}
|
1438
|
+
async ensureKeyFromUrl(url, keyFactory) {
|
1439
|
+
const storeKey = url.getParam("storekey");
|
1440
|
+
if (storeKey === "insecure") {
|
1441
|
+
return import_cement5.Result.Ok(url);
|
1442
|
+
}
|
1443
|
+
if (!storeKey) {
|
1444
|
+
const keyName = `@${keyFactory()}@`;
|
1445
|
+
const ret = await this.getNamedKey(keyName);
|
1446
|
+
if (ret.isErr()) {
|
1447
|
+
return ret;
|
1448
|
+
}
|
1449
|
+
const urb = url.build().setParam("storekey", keyName);
|
1450
|
+
return import_cement5.Result.Ok(urb.URI());
|
1451
|
+
}
|
1452
|
+
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
1453
|
+
const ret = await this.getNamedKey(storeKey);
|
1454
|
+
if (ret.isErr()) {
|
1455
|
+
return ret;
|
1456
|
+
}
|
1457
|
+
}
|
1458
|
+
return import_cement5.Result.Ok(url);
|
1459
|
+
}
|
1460
|
+
async toKeyWithFingerPrint(keyStr) {
|
1461
|
+
const material = import_base582.base58btc.decode(keyStr);
|
1462
|
+
const key = await this.subtleKey(keyStr);
|
1463
|
+
const fpr = await this.rt.crypto.digestSHA256(material);
|
1464
|
+
return import_cement5.Result.Ok({
|
1465
|
+
key,
|
1466
|
+
fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
|
1467
|
+
});
|
1468
|
+
}
|
1469
|
+
async setNamedKey(name, key) {
|
1470
|
+
return this._seq.add(() => this._setNamedKey(name, key));
|
1471
|
+
}
|
1472
|
+
// avoid deadlock
|
1473
|
+
async _setNamedKey(name, key) {
|
1474
|
+
const item = {
|
1475
|
+
name,
|
1476
|
+
key
|
1477
|
+
};
|
1478
|
+
const bag = await this.rt.getBag();
|
1479
|
+
this.logger.Debug().Str("name", name).Msg("setNamedKey");
|
1480
|
+
await bag.set(name, item);
|
1481
|
+
return await this.toKeyWithFingerPrint(item.key);
|
1482
|
+
}
|
1483
|
+
async getNamedKey(name, failIfNotFound = false) {
|
1484
|
+
const id = this.rt.sthis.nextId(4).str;
|
1485
|
+
return this._seq.add(async () => {
|
1486
|
+
const bag = await this.rt.getBag();
|
1487
|
+
const named = await bag.get(name);
|
1488
|
+
if (named) {
|
1489
|
+
const fpr = await this.toKeyWithFingerPrint(named.key);
|
1490
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
|
1491
|
+
return fpr;
|
1492
|
+
}
|
1493
|
+
if (failIfNotFound) {
|
1494
|
+
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
1495
|
+
return import_cement5.Result.Err(new Error(`Key not found: ${name}`));
|
1496
|
+
}
|
1497
|
+
const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
1498
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
1499
|
+
return ret;
|
1500
|
+
});
|
1501
|
+
}
|
1502
|
+
};
|
1503
|
+
var keyBagProviderFactories = new Map(
|
1504
|
+
[
|
1505
|
+
{
|
1506
|
+
protocol: "file:",
|
1507
|
+
factory: async (url, sthis) => {
|
1508
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1509
|
+
return new KeyBagProviderFile2(url, sthis);
|
1510
|
+
}
|
1511
|
+
},
|
1512
|
+
{
|
1513
|
+
protocol: "indexdb:",
|
1514
|
+
factory: async (url, sthis) => {
|
1515
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1516
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
1517
|
+
}
|
1518
|
+
}
|
1519
|
+
].map((i) => [i.protocol, i])
|
1520
|
+
);
|
1521
|
+
function registerKeyBagProviderFactory(item) {
|
1522
|
+
const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
|
1523
|
+
keyBagProviderFactories.set(protocol, {
|
1524
|
+
...item,
|
1525
|
+
protocol
|
1526
|
+
});
|
1527
|
+
}
|
1528
|
+
function defaultKeyBagOpts(sthis, kbo) {
|
1529
|
+
if (kbo.keyRuntime) {
|
1530
|
+
return kbo.keyRuntime;
|
1531
|
+
}
|
1532
|
+
const logger = ensureLogger(sthis, "KeyBag");
|
1533
|
+
let url;
|
1534
|
+
if (kbo.url) {
|
1535
|
+
url = import_cement5.URI.from(kbo.url);
|
1536
|
+
logger.Debug().Url(url).Msg("from opts");
|
1537
|
+
} else {
|
1538
|
+
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
1539
|
+
if ((0, import_cement5.runtimeFn)().isBrowser) {
|
1540
|
+
url = import_cement5.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
1541
|
+
} else {
|
1542
|
+
if (!bagFnameOrUrl) {
|
1543
|
+
const home = sthis.env.get("HOME");
|
1544
|
+
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
1545
|
+
url = import_cement5.URI.from(`file://${bagFnameOrUrl}`);
|
1546
|
+
} else {
|
1547
|
+
url = import_cement5.URI.from(bagFnameOrUrl);
|
1548
|
+
}
|
1549
|
+
}
|
1550
|
+
logger.Debug().Url(url).Msg("from env");
|
1551
|
+
}
|
1552
|
+
let keyProviderFactory;
|
1553
|
+
switch (url.protocol) {
|
1554
|
+
case "file:":
|
1555
|
+
keyProviderFactory = async () => {
|
1556
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1557
|
+
return new KeyBagProviderFile2(url, sthis);
|
1558
|
+
};
|
1559
|
+
break;
|
1560
|
+
case "indexdb:":
|
1561
|
+
keyProviderFactory = async () => {
|
1562
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1563
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
1564
|
+
};
|
1565
|
+
break;
|
1566
|
+
default:
|
1567
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
1568
|
+
}
|
1569
|
+
if (url.hasParam("masterkey")) {
|
1570
|
+
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
1571
|
+
}
|
1572
|
+
return {
|
1573
|
+
url,
|
1574
|
+
crypto: kbo.crypto || (0, import_cement5.toCryptoRuntime)({}),
|
1575
|
+
sthis,
|
1576
|
+
logger,
|
1577
|
+
keyLength: kbo.keyLength || 16,
|
1578
|
+
getBag: keyProviderFactory,
|
1579
|
+
id: () => {
|
1580
|
+
return url.toString();
|
1581
|
+
}
|
1582
|
+
};
|
1583
|
+
}
|
1584
|
+
var _keyBags = new import_cement5.KeyedResolvOnce();
|
1585
|
+
async function getKeyBag(sthis, kbo = {}) {
|
1586
|
+
await sthis.start();
|
1587
|
+
const rt = defaultKeyBagOpts(sthis, kbo);
|
1588
|
+
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
1589
|
+
}
|
1590
|
+
|
1591
|
+
// src/blockstore/commitor.ts
|
1592
|
+
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
1593
|
+
var import_sha22 = require("multiformats/hashes/sha2");
|
1594
|
+
var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
1595
|
+
async function encodeCarFile(roots, t, codec3) {
|
1596
|
+
let size = 0;
|
1597
|
+
const headerSize = CBW.headerLength({ roots });
|
1598
|
+
size += headerSize;
|
1599
|
+
for (const { cid, bytes } of t.entries()) {
|
1600
|
+
size += CBW.blockLength({ cid, bytes });
|
1601
|
+
}
|
1602
|
+
const buffer = new Uint8Array(size);
|
1603
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
1604
|
+
for (const r of roots) {
|
1605
|
+
writer.addRoot(r);
|
1772
1606
|
}
|
1773
|
-
|
1774
|
-
|
1775
|
-
const queueFn = async () => {
|
1776
|
-
try {
|
1777
|
-
resolve(await fn());
|
1778
|
-
} catch (e) {
|
1779
|
-
reject(e);
|
1780
|
-
} finally {
|
1781
|
-
this.processing = false;
|
1782
|
-
this.processNext();
|
1783
|
-
}
|
1784
|
-
};
|
1785
|
-
this.queue.push(queueFn);
|
1786
|
-
if (!this.processing) {
|
1787
|
-
this.processNext();
|
1788
|
-
}
|
1789
|
-
});
|
1607
|
+
for (const { cid, bytes } of t.entries()) {
|
1608
|
+
writer.write({ cid, bytes });
|
1790
1609
|
}
|
1791
|
-
|
1792
|
-
|
1793
|
-
|
1794
|
-
|
1795
|
-
|
1796
|
-
|
1797
|
-
|
1610
|
+
writer.close();
|
1611
|
+
return await encode({ value: writer.bytes, hasher: import_sha22.sha256, codec: codec3 });
|
1612
|
+
}
|
1613
|
+
async function createCarFile(encoder, cid, t) {
|
1614
|
+
return encodeCarFile([cid], t, encoder);
|
1615
|
+
}
|
1616
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
1617
|
+
const { files: roots } = makeFileCarHeader(done);
|
1618
|
+
const cids = [];
|
1619
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1620
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1621
|
+
for (const car of cars) {
|
1622
|
+
const { cid, bytes } = car;
|
1623
|
+
await fileStore.save({ cid, bytes });
|
1624
|
+
await walStore.enqueueFile(
|
1625
|
+
cid
|
1626
|
+
/*, !!opts.public*/
|
1627
|
+
);
|
1628
|
+
cids.push(cid);
|
1629
|
+
}
|
1630
|
+
return cids;
|
1631
|
+
}
|
1632
|
+
function makeFileCarHeader(result) {
|
1633
|
+
const files = [];
|
1634
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
1635
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1636
|
+
files.push(meta.cid);
|
1798
1637
|
}
|
1799
1638
|
}
|
1800
|
-
};
|
1639
|
+
return { ...result, files };
|
1640
|
+
}
|
1641
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
1642
|
+
return [await encodeCarFile(roots, t, encoder)];
|
1643
|
+
}
|
1644
|
+
function makeCarHeader(meta, cars, compact = false) {
|
1645
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1646
|
+
return { ...coreHeader, meta };
|
1647
|
+
}
|
1648
|
+
async function encodeCarHeader(fp) {
|
1649
|
+
return await encode({
|
1650
|
+
value: { fp },
|
1651
|
+
hasher: import_sha22.sha256,
|
1652
|
+
codec: dagCodec2
|
1653
|
+
});
|
1654
|
+
}
|
1655
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
1656
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
1657
|
+
const rootBlock = await encodeCarHeader(fp);
|
1658
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
1659
|
+
const cids = [];
|
1660
|
+
for (const car of cars) {
|
1661
|
+
const { cid, bytes } = car;
|
1662
|
+
await params.carStore.save({ cid, bytes });
|
1663
|
+
cids.push(cid);
|
1664
|
+
}
|
1665
|
+
const newDbMeta = { cars: cids };
|
1666
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
1667
|
+
await params.metaStore.save(newDbMeta);
|
1668
|
+
return { cgrp: cids, header: fp };
|
1669
|
+
}
|
1670
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
1671
|
+
const carFiles = [];
|
1672
|
+
threshold = threshold || 1e3 * 1e3;
|
1673
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1674
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1675
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
1676
|
+
let cidRootBlock = rootBlock;
|
1677
|
+
for (const { cid, bytes } of t.entries()) {
|
1678
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1679
|
+
if (newsize >= threshold) {
|
1680
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1681
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1682
|
+
clonedt.putSync(cid, bytes);
|
1683
|
+
cidRootBlock = { cid, bytes };
|
1684
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1685
|
+
} else {
|
1686
|
+
clonedt.putSync(cid, bytes);
|
1687
|
+
}
|
1688
|
+
}
|
1689
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1690
|
+
return carFiles;
|
1691
|
+
}
|
1801
1692
|
|
1802
1693
|
// src/blockstore/loader.ts
|
1803
|
-
var
|
1804
|
-
function
|
1694
|
+
var import_sha23 = require("multiformats/hashes/sha2");
|
1695
|
+
function carLogIncludesGroup(list, cids) {
|
1805
1696
|
return list.some((arr) => {
|
1806
1697
|
return arr.toString() === cids.toString();
|
1807
1698
|
});
|
@@ -1814,17 +1705,8 @@ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
|
1814
1705
|
}
|
1815
1706
|
return [...byString.values()];
|
1816
1707
|
}
|
1817
|
-
function toHexString(byteArray) {
|
1818
|
-
return Array.from(byteArray).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
1819
|
-
}
|
1820
|
-
var Loadable = class {
|
1821
|
-
constructor() {
|
1822
|
-
this.name = "";
|
1823
|
-
this.carLog = new Array();
|
1824
|
-
}
|
1825
|
-
};
|
1826
1708
|
var Loader = class {
|
1827
|
-
constructor(name, ebOpts) {
|
1709
|
+
constructor(name, ebOpts, sthis) {
|
1828
1710
|
this.commitQueue = new CommitQueue();
|
1829
1711
|
this.isCompacting = false;
|
1830
1712
|
this.carReaders = /* @__PURE__ */ new Map();
|
@@ -1834,9 +1716,11 @@ var Loader = class {
|
|
1834
1716
|
this.getBlockCache = /* @__PURE__ */ new Map();
|
1835
1717
|
this.seenMeta = /* @__PURE__ */ new Set();
|
1836
1718
|
this.writeLimit = (0, import_p_limit.default)(1);
|
1837
|
-
this.onceReady = new
|
1719
|
+
this.onceReady = new import_cement6.ResolveOnce();
|
1838
1720
|
this.name = name;
|
1721
|
+
this.sthis = sthis;
|
1839
1722
|
this.ebOpts = defaultedBlockstoreRuntime(
|
1723
|
+
sthis,
|
1840
1724
|
{
|
1841
1725
|
...ebOpts,
|
1842
1726
|
name
|
@@ -1846,14 +1730,17 @@ var Loader = class {
|
|
1846
1730
|
this.logger = this.ebOpts.logger;
|
1847
1731
|
}
|
1848
1732
|
// readonly id = uuidv4();
|
1733
|
+
async keyBag() {
|
1734
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1735
|
+
}
|
1849
1736
|
async carStore() {
|
1850
1737
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1851
1738
|
}
|
1852
1739
|
async fileStore() {
|
1853
1740
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1854
1741
|
}
|
1855
|
-
async
|
1856
|
-
return this.ebOpts.storeRuntime.
|
1742
|
+
async WALStore() {
|
1743
|
+
return this.ebOpts.storeRuntime.makeWALStore(this);
|
1857
1744
|
}
|
1858
1745
|
async metaStore() {
|
1859
1746
|
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
@@ -1867,11 +1754,11 @@ var Loader = class {
|
|
1867
1754
|
});
|
1868
1755
|
}
|
1869
1756
|
async close() {
|
1870
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
1757
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1871
1758
|
await Promise.all(toClose.map((store) => store.close()));
|
1872
1759
|
}
|
1873
1760
|
async destroy() {
|
1874
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
1761
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1875
1762
|
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1876
1763
|
}
|
1877
1764
|
// async snapToCar(carCid: AnyLink | string) {
|
@@ -1885,6 +1772,7 @@ var Loader = class {
|
|
1885
1772
|
// await this._applyCarHeader(carHeader, true)
|
1886
1773
|
// }
|
1887
1774
|
async handleDbMetasFromStore(metas) {
|
1775
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1888
1776
|
for (const meta of metas) {
|
1889
1777
|
await this.writeLimit(async () => {
|
1890
1778
|
await this.mergeDbMetaIntoClock(meta);
|
@@ -1897,10 +1785,7 @@ var Loader = class {
|
|
1897
1785
|
}
|
1898
1786
|
if (this.seenMeta.has(meta.cars.toString())) return;
|
1899
1787
|
this.seenMeta.add(meta.cars.toString());
|
1900
|
-
if (meta.
|
1901
|
-
await this.setKey(meta.key);
|
1902
|
-
}
|
1903
|
-
if (carLogIncludesGroup2(this.carLog, meta.cars)) {
|
1788
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1904
1789
|
return;
|
1905
1790
|
}
|
1906
1791
|
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
@@ -1909,45 +1794,59 @@ var Loader = class {
|
|
1909
1794
|
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1910
1795
|
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1911
1796
|
}
|
1912
|
-
async ingestKeyFromMeta(meta) {
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
}
|
1797
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1798
|
+
// const { key } = meta;
|
1799
|
+
// if (key) {
|
1800
|
+
// await this.setKey(key);
|
1801
|
+
// }
|
1802
|
+
// }
|
1918
1803
|
async loadCarHeaderFromMeta({ cars: cids }) {
|
1919
1804
|
const reader = await this.loadCar(cids[0]);
|
1920
1805
|
return await parseCarFile(reader, this.logger);
|
1921
1806
|
}
|
1922
|
-
async _getKey() {
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
}
|
1932
|
-
// can these skip the queue? or have a file queue?
|
1933
|
-
async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
|
1807
|
+
// async _getKey(): Promise<string | undefined> {
|
1808
|
+
// if (this.key) return this.key;
|
1809
|
+
// // generate a random key
|
1810
|
+
// if (!this.ebOpts.public) {
|
1811
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1812
|
+
// }
|
1813
|
+
// return this.key || undefined;
|
1814
|
+
// }
|
1815
|
+
async commitFiles(t, done) {
|
1934
1816
|
await this.ready();
|
1935
|
-
const
|
1936
|
-
const
|
1937
|
-
|
1938
|
-
for (const car of cars) {
|
1939
|
-
const { cid, bytes } = car;
|
1940
|
-
await (await this.fileStore()).save({ cid, bytes });
|
1941
|
-
await (await this.remoteWAL()).enqueueFile(cid, !!opts.public);
|
1942
|
-
cids.push(cid);
|
1943
|
-
}
|
1944
|
-
return cids;
|
1817
|
+
const fstore = await this.fileStore();
|
1818
|
+
const wstore = await this.WALStore();
|
1819
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1945
1820
|
}
|
1946
|
-
async loadFileCar(cid
|
1947
|
-
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore
|
1821
|
+
async loadFileCar(cid) {
|
1822
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1948
1823
|
}
|
1949
1824
|
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1950
|
-
|
1825
|
+
await this.ready();
|
1826
|
+
const fstore = await this.fileStore();
|
1827
|
+
const params = {
|
1828
|
+
encoder: (await fstore.keyedCrypto()).codec(),
|
1829
|
+
carLog: this.carLog,
|
1830
|
+
carStore: fstore,
|
1831
|
+
WALStore: await this.WALStore(),
|
1832
|
+
metaStore: await this.metaStore()
|
1833
|
+
};
|
1834
|
+
return this.commitQueue.enqueue(async () => {
|
1835
|
+
await this.cacheTransaction(t);
|
1836
|
+
const ret = await commit(params, t, done, opts);
|
1837
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1838
|
+
return ret.cgrp;
|
1839
|
+
});
|
1840
|
+
}
|
1841
|
+
async updateCarLog(cids, fp, compact) {
|
1842
|
+
if (compact) {
|
1843
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1844
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1845
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1846
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1847
|
+
} else {
|
1848
|
+
this.carLog.unshift(cids);
|
1849
|
+
}
|
1951
1850
|
}
|
1952
1851
|
async cacheTransaction(t) {
|
1953
1852
|
for await (const block of t.entries()) {
|
@@ -1967,79 +1866,6 @@ var Loader = class {
|
|
1967
1866
|
}
|
1968
1867
|
}
|
1969
1868
|
}
|
1970
|
-
async _commitInternal(t, done, opts = { noLoader: false, compact: false }) {
|
1971
|
-
await this.ready();
|
1972
|
-
const fp = this.makeCarHeader(done, this.carLog, !!opts.compact);
|
1973
|
-
const rootBlock = await encodeCarHeader(fp);
|
1974
|
-
const cars = await this.prepareCarFiles(rootBlock, t, !!opts.public);
|
1975
|
-
const cids = [];
|
1976
|
-
for (const car of cars) {
|
1977
|
-
const { cid, bytes } = car;
|
1978
|
-
await (await this.carStore()).save({ cid, bytes });
|
1979
|
-
cids.push(cid);
|
1980
|
-
}
|
1981
|
-
await this.cacheTransaction(t);
|
1982
|
-
const newDbMeta = { cars: cids, key: this.key || null };
|
1983
|
-
await (await this.remoteWAL()).enqueue(newDbMeta, opts);
|
1984
|
-
await (await this.metaStore()).save(newDbMeta);
|
1985
|
-
await this.updateCarLog(cids, fp, !!opts.compact);
|
1986
|
-
return cids;
|
1987
|
-
}
|
1988
|
-
async prepareCarFilesFiles(roots, t, isPublic) {
|
1989
|
-
const theKey = isPublic ? null : await this._getKey();
|
1990
|
-
const car = theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, roots[0], t) : await encodeCarFile(roots, t);
|
1991
|
-
return [car];
|
1992
|
-
}
|
1993
|
-
async prepareCarFiles(rootBlock, t, isPublic) {
|
1994
|
-
const theKey = isPublic ? void 0 : await this._getKey();
|
1995
|
-
const carFiles = [];
|
1996
|
-
const threshold = this.ebOpts.threshold || 1e3 * 1e3;
|
1997
|
-
let clonedt = new CarTransaction(t.parent, { add: false });
|
1998
|
-
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1999
|
-
let newsize = CBW2.blockLength(toCIDBlock(rootBlock));
|
2000
|
-
let cidRootBlock = rootBlock;
|
2001
|
-
for (const { cid, bytes } of t.entries()) {
|
2002
|
-
newsize += CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
2003
|
-
if (newsize >= threshold) {
|
2004
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
2005
|
-
clonedt = new CarTransaction(t.parent, { add: false });
|
2006
|
-
clonedt.putSync(cid, bytes);
|
2007
|
-
cidRootBlock = { cid, bytes };
|
2008
|
-
newsize = CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
2009
|
-
} else {
|
2010
|
-
clonedt.putSync(cid, bytes);
|
2011
|
-
}
|
2012
|
-
}
|
2013
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
2014
|
-
return carFiles;
|
2015
|
-
}
|
2016
|
-
async createCarFile(theKey, cid, t) {
|
2017
|
-
try {
|
2018
|
-
return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t) : await encodeCarFile([cid], t);
|
2019
|
-
} catch (e) {
|
2020
|
-
console.error("error creating car file", e);
|
2021
|
-
throw e;
|
2022
|
-
}
|
2023
|
-
}
|
2024
|
-
makeFileCarHeader(result) {
|
2025
|
-
const files = [];
|
2026
|
-
for (const [, meta] of Object.entries(result.files || {})) {
|
2027
|
-
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
2028
|
-
files.push(meta.cid);
|
2029
|
-
}
|
2030
|
-
}
|
2031
|
-
return { ...result, files };
|
2032
|
-
}
|
2033
|
-
async updateCarLog(cids, fp, compact) {
|
2034
|
-
if (compact) {
|
2035
|
-
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
2036
|
-
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
2037
|
-
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
2038
|
-
await this.removeCidsForCompact(previousCompactCid[0]);
|
2039
|
-
} else {
|
2040
|
-
this.carLog.unshift(cids);
|
2041
|
-
}
|
2042
|
-
}
|
2043
1869
|
async removeCidsForCompact(cid) {
|
2044
1870
|
const carHeader = await this.loadCarHeaderFromMeta({
|
2045
1871
|
cars: [cid]
|
@@ -2058,9 +1884,9 @@ var Loader = class {
|
|
2058
1884
|
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
2059
1885
|
// }
|
2060
1886
|
// }
|
2061
|
-
async *entries(
|
1887
|
+
async *entries(cache2 = true) {
|
2062
1888
|
await this.ready();
|
2063
|
-
if (
|
1889
|
+
if (cache2) {
|
2064
1890
|
for (const [, block] of this.getBlockCache) {
|
2065
1891
|
yield block;
|
2066
1892
|
}
|
@@ -2142,10 +1968,6 @@ var Loader = class {
|
|
2142
1968
|
}
|
2143
1969
|
return got;
|
2144
1970
|
}
|
2145
|
-
makeCarHeader(meta, cars, compact = false) {
|
2146
|
-
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
2147
|
-
return { ...coreHeader, meta };
|
2148
|
-
}
|
2149
1971
|
async loadCar(cid) {
|
2150
1972
|
if (!this.carStore) {
|
2151
1973
|
throw this.logger.Error().Msg("car store not initialized").AsError();
|
@@ -2153,72 +1975,52 @@ var Loader = class {
|
|
2153
1975
|
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
2154
1976
|
return loaded;
|
2155
1977
|
}
|
2156
|
-
|
2157
|
-
async storesLoadCar(cid, local, remote, publicFiles) {
|
1978
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
2158
1979
|
const cidsString = cid.toString();
|
2159
|
-
|
2160
|
-
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2164
|
-
|
2165
|
-
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2175
|
-
|
2176
|
-
|
2177
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
2178
|
-
}
|
2179
|
-
}
|
2180
|
-
if (!loadedCar) {
|
2181
|
-
throw this.logger.Error().Url(local.url).Str("cid", cidsString).Msg("missing car files").AsError();
|
2182
|
-
}
|
2183
|
-
const rawReader = await import_car.CarReader.fromBytes(loadedCar.bytes);
|
2184
|
-
const readerP = publicFiles ? Promise.resolve(rawReader) : this.ensureDecryptedReader(rawReader);
|
2185
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
2186
|
-
await this.cacheCarReader(cidsString, reader).catch(() => {
|
2187
|
-
return;
|
2188
|
-
});
|
2189
|
-
return reader;
|
2190
|
-
});
|
2191
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
2192
|
-
return readerP;
|
2193
|
-
})().catch((e) => {
|
2194
|
-
this.carReaders.delete(cidsString);
|
2195
|
-
throw e;
|
2196
|
-
})
|
2197
|
-
);
|
1980
|
+
let loadedCar = void 0;
|
1981
|
+
let activeStore = local;
|
1982
|
+
try {
|
1983
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1984
|
+
loadedCar = await local.load(cid);
|
1985
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1986
|
+
} catch (e) {
|
1987
|
+
if (remote) {
|
1988
|
+
const remoteCar = await remote.load(cid);
|
1989
|
+
if (remoteCar) {
|
1990
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1991
|
+
await local.save(remoteCar);
|
1992
|
+
loadedCar = remoteCar;
|
1993
|
+
activeStore = remote;
|
1994
|
+
}
|
1995
|
+
} else {
|
1996
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1997
|
+
}
|
2198
1998
|
}
|
2199
|
-
|
2200
|
-
|
2201
|
-
async ensureDecryptedReader(reader) {
|
2202
|
-
const theKey = await this._getKey();
|
2203
|
-
if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) {
|
2204
|
-
return reader;
|
1999
|
+
if (!loadedCar) {
|
2000
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
2205
2001
|
}
|
2206
|
-
const
|
2207
|
-
|
2208
|
-
|
2209
|
-
|
2210
|
-
|
2211
|
-
|
2002
|
+
const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha23.sha256, codec: (await activeStore.keyedCrypto()).codec() });
|
2003
|
+
const rawReader = await import_car.CarReader.fromBytes(bytes.value);
|
2004
|
+
const readerP = Promise.resolve(rawReader);
|
2005
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
2006
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
2007
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
2008
|
+
return;
|
2009
|
+
});
|
2010
|
+
return reader;
|
2011
|
+
});
|
2012
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
2013
|
+
return readerP;
|
2212
2014
|
}
|
2213
|
-
|
2214
|
-
|
2215
|
-
|
2216
|
-
|
2217
|
-
|
2218
|
-
|
2219
|
-
|
2220
|
-
|
2221
|
-
|
2015
|
+
//What if instead it returns an Array of CarHeader
|
2016
|
+
async storesLoadCar(cid, local, remote) {
|
2017
|
+
const cidsString = cid.toString();
|
2018
|
+
let dacr = this.carReaders.get(cidsString);
|
2019
|
+
if (!dacr) {
|
2020
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
2021
|
+
this.carReaders.set(cidsString, dacr);
|
2022
|
+
}
|
2023
|
+
return dacr;
|
2222
2024
|
}
|
2223
2025
|
async getMoreReaders(cids) {
|
2224
2026
|
const limit = (0, import_p_limit.default)(5);
|
@@ -2227,20 +2029,345 @@ var Loader = class {
|
|
2227
2029
|
}
|
2228
2030
|
};
|
2229
2031
|
|
2032
|
+
// src/runtime/keyed-crypto.ts
|
2033
|
+
var keyed_crypto_exports = {};
|
2034
|
+
__export(keyed_crypto_exports, {
|
2035
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
2036
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
2037
|
+
});
|
2038
|
+
init_utils();
|
2039
|
+
var import_base583 = require("multiformats/bases/base58");
|
2040
|
+
var import_sha24 = require("multiformats/hashes/sha2");
|
2041
|
+
var CBOR = __toESM(require("cborg"), 1);
|
2042
|
+
var generateIV = {
|
2043
|
+
random: {
|
2044
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2045
|
+
calc: async (ko, crypto, data) => {
|
2046
|
+
return crypto.randomBytes(ko.ivLength);
|
2047
|
+
},
|
2048
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2049
|
+
verify: async (ko, crypto, iv, data) => {
|
2050
|
+
return true;
|
2051
|
+
}
|
2052
|
+
},
|
2053
|
+
hash: {
|
2054
|
+
calc: async (ko, crypto, data) => {
|
2055
|
+
const hash = await import_sha24.sha256.digest(data);
|
2056
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
2057
|
+
const hashArray = new Uint8Array(ko.ivLength * 8);
|
2058
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
2059
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
2060
|
+
}
|
2061
|
+
return hashArray;
|
2062
|
+
},
|
2063
|
+
verify: async function(ko, crypto, iv, data) {
|
2064
|
+
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
2065
|
+
}
|
2066
|
+
}
|
2067
|
+
};
|
2068
|
+
function getGenerateIVFn(url, opts) {
|
2069
|
+
const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
|
2070
|
+
return generateIV[ivhash] || generateIV["hash"];
|
2071
|
+
}
|
2072
|
+
var BlockIvKeyIdCodec = class {
|
2073
|
+
constructor(ko, iv, opts) {
|
2074
|
+
this.code = 3147065;
|
2075
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
2076
|
+
this.ko = ko;
|
2077
|
+
this.iv = iv;
|
2078
|
+
this.opts = opts || {};
|
2079
|
+
}
|
2080
|
+
async encode(data) {
|
2081
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
2082
|
+
const { iv } = this.ko.algo(calcIv);
|
2083
|
+
const fprt = await this.ko.fingerPrint();
|
2084
|
+
const keyId = import_base583.base58btc.decode(fprt);
|
2085
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
2086
|
+
return CBOR.encode({
|
2087
|
+
iv,
|
2088
|
+
keyId,
|
2089
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
2090
|
+
});
|
2091
|
+
}
|
2092
|
+
async decode(abytes) {
|
2093
|
+
let bytes;
|
2094
|
+
if (abytes instanceof Uint8Array) {
|
2095
|
+
bytes = abytes;
|
2096
|
+
} else {
|
2097
|
+
bytes = new Uint8Array(abytes);
|
2098
|
+
}
|
2099
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
2100
|
+
const fprt = await this.ko.fingerPrint();
|
2101
|
+
this.ko.logger.Debug().Str("fp", import_base583.base58btc.encode(keyId)).Msg("decode");
|
2102
|
+
if (import_base583.base58btc.encode(keyId) !== fprt) {
|
2103
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base583.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
2104
|
+
}
|
2105
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
2106
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
2107
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
2108
|
+
}
|
2109
|
+
return result;
|
2110
|
+
}
|
2111
|
+
};
|
2112
|
+
var keyedCrypto = class {
|
2113
|
+
constructor(url, key, cyopt, sthis) {
|
2114
|
+
this.ivLength = 12;
|
2115
|
+
this.isEncrypting = true;
|
2116
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
2117
|
+
this.crypto = cyopt;
|
2118
|
+
this.key = key;
|
2119
|
+
this.url = url;
|
2120
|
+
}
|
2121
|
+
fingerPrint() {
|
2122
|
+
return Promise.resolve(this.key.fingerPrint);
|
2123
|
+
}
|
2124
|
+
codec(iv, opts) {
|
2125
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
2126
|
+
}
|
2127
|
+
algo(iv) {
|
2128
|
+
return {
|
2129
|
+
name: "AES-GCM",
|
2130
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
2131
|
+
tagLength: 128
|
2132
|
+
};
|
2133
|
+
}
|
2134
|
+
async _decrypt(data) {
|
2135
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("decrypting");
|
2136
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
2137
|
+
}
|
2138
|
+
async _encrypt(data) {
|
2139
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
2140
|
+
const a = this.algo(data.iv);
|
2141
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
2142
|
+
}
|
2143
|
+
};
|
2144
|
+
var nullCodec = class {
|
2145
|
+
constructor() {
|
2146
|
+
this.code = 0;
|
2147
|
+
this.name = "Fireproof@unencrypted-block";
|
2148
|
+
}
|
2149
|
+
encode(data) {
|
2150
|
+
return data;
|
2151
|
+
}
|
2152
|
+
decode(data) {
|
2153
|
+
return data;
|
2154
|
+
}
|
2155
|
+
};
|
2156
|
+
var noCrypto = class {
|
2157
|
+
constructor(url, cyrt, sthis) {
|
2158
|
+
this.ivLength = 0;
|
2159
|
+
this.code = 0;
|
2160
|
+
this.name = "Fireproof@unencrypted-block";
|
2161
|
+
this.isEncrypting = false;
|
2162
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
2163
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
2164
|
+
this.crypto = cyrt;
|
2165
|
+
this.url = url;
|
2166
|
+
}
|
2167
|
+
fingerPrint() {
|
2168
|
+
return Promise.resolve(this._fingerPrint);
|
2169
|
+
}
|
2170
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2171
|
+
codec(iv) {
|
2172
|
+
return new nullCodec();
|
2173
|
+
}
|
2174
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2175
|
+
algo(iv) {
|
2176
|
+
return {
|
2177
|
+
name: "noCrypto",
|
2178
|
+
iv: new Uint8Array(),
|
2179
|
+
tagLength: 0
|
2180
|
+
};
|
2181
|
+
}
|
2182
|
+
_decrypt() {
|
2183
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
2184
|
+
}
|
2185
|
+
_encrypt() {
|
2186
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
2187
|
+
}
|
2188
|
+
};
|
2189
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
2190
|
+
const storekey = url.getParam("storekey");
|
2191
|
+
if (storekey && storekey !== "insecure") {
|
2192
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
2193
|
+
if (rkey.isErr()) {
|
2194
|
+
try {
|
2195
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
2196
|
+
} catch (e) {
|
2197
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
2198
|
+
}
|
2199
|
+
}
|
2200
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
2201
|
+
}
|
2202
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
2203
|
+
}
|
2204
|
+
|
2205
|
+
// src/blockstore/fragment-gateway.ts
|
2206
|
+
var import_base584 = require("multiformats/bases/base58");
|
2207
|
+
var import_cborg = require("cborg");
|
2208
|
+
function getFragSize(url) {
|
2209
|
+
const fragSize = url.getParam("fragSize");
|
2210
|
+
let ret = 0;
|
2211
|
+
if (fragSize) {
|
2212
|
+
ret = parseInt(fragSize);
|
2213
|
+
}
|
2214
|
+
if (isNaN(ret) || ret <= 0) {
|
2215
|
+
ret = 0;
|
2216
|
+
}
|
2217
|
+
return ret;
|
2218
|
+
}
|
2219
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
2220
|
+
const fragSize = getFragSize(url);
|
2221
|
+
if (!fragSize) {
|
2222
|
+
const res = await innerGW.get(url);
|
2223
|
+
if (res.isErr()) {
|
2224
|
+
return [res];
|
2225
|
+
}
|
2226
|
+
const data = res.unwrap();
|
2227
|
+
return [
|
2228
|
+
import_cement.Result.Ok({
|
2229
|
+
fid: new Uint8Array(0),
|
2230
|
+
ofs: 0,
|
2231
|
+
len: data.length,
|
2232
|
+
data
|
2233
|
+
})
|
2234
|
+
];
|
2235
|
+
}
|
2236
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
2237
|
+
if (firstRaw.isErr()) {
|
2238
|
+
return [firstRaw];
|
2239
|
+
}
|
2240
|
+
const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
|
2241
|
+
const blockSize = firstFragment.data.length;
|
2242
|
+
const ops = [Promise.resolve(import_cement.Result.Ok(firstFragment))];
|
2243
|
+
const fidStr = import_base584.base58btc.encode(firstFragment.fid);
|
2244
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
2245
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
2246
|
+
ops.push(
|
2247
|
+
(async (furl, ofs2) => {
|
2248
|
+
const raw2 = await innerGW.get(furl);
|
2249
|
+
if (raw2.isErr()) {
|
2250
|
+
return raw2;
|
2251
|
+
}
|
2252
|
+
const fragment = (0, import_cborg.decode)(raw2.unwrap());
|
2253
|
+
if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
|
2254
|
+
return import_cement.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
2255
|
+
}
|
2256
|
+
if (fragment.ofs !== ofs2) {
|
2257
|
+
return import_cement.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
2258
|
+
}
|
2259
|
+
return import_cement.Result.Ok(fragment);
|
2260
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
2261
|
+
);
|
2262
|
+
}
|
2263
|
+
return Promise.all(ops);
|
2264
|
+
}
|
2265
|
+
var FragmentGateway = class {
|
2266
|
+
constructor(sthis, innerGW) {
|
2267
|
+
this.fidLength = 4;
|
2268
|
+
this.headerSize = 32;
|
2269
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
2270
|
+
this.logger = this.sthis.logger;
|
2271
|
+
this.innerGW = innerGW;
|
2272
|
+
}
|
2273
|
+
slicer(url, body) {
|
2274
|
+
const fragSize = getFragSize(url);
|
2275
|
+
if (!fragSize) {
|
2276
|
+
return [this.innerGW.put(url, body)];
|
2277
|
+
}
|
2278
|
+
const blocksize = fragSize - this.headerSize;
|
2279
|
+
if (blocksize <= 0) {
|
2280
|
+
throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
|
2281
|
+
}
|
2282
|
+
const ops = [];
|
2283
|
+
const fid = this.sthis.nextId(this.fidLength);
|
2284
|
+
const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
|
2285
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
2286
|
+
const block = (0, import_cborg.encode)({
|
2287
|
+
fid: fid.bin,
|
2288
|
+
ofs,
|
2289
|
+
len: body.length,
|
2290
|
+
data: body.slice(ofs, ofs + blocksize)
|
2291
|
+
});
|
2292
|
+
if (block.length > fragSize) {
|
2293
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
2294
|
+
}
|
2295
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
2296
|
+
}
|
2297
|
+
return ops;
|
2298
|
+
}
|
2299
|
+
buildUrl(baseUrl, key) {
|
2300
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
2301
|
+
}
|
2302
|
+
async destroy(iurl) {
|
2303
|
+
return this.innerGW.destroy(iurl);
|
2304
|
+
}
|
2305
|
+
async start(url) {
|
2306
|
+
this.headerSize = (0, import_cborg.encode)({
|
2307
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
2308
|
+
ofs: 1024 * 1024,
|
2309
|
+
// 32bit
|
2310
|
+
len: 16 * 1024 * 1024,
|
2311
|
+
// 32bit
|
2312
|
+
data: new Uint8Array(1024)
|
2313
|
+
}).length - 1024;
|
2314
|
+
return this.innerGW.start(url);
|
2315
|
+
}
|
2316
|
+
async close(url) {
|
2317
|
+
return this.innerGW.close(url);
|
2318
|
+
}
|
2319
|
+
async put(url, body) {
|
2320
|
+
await Promise.all(this.slicer(url, body));
|
2321
|
+
return import_cement.Result.Ok(void 0);
|
2322
|
+
}
|
2323
|
+
async get(url) {
|
2324
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
2325
|
+
let buffer = void 0;
|
2326
|
+
for (const rfrag of rfrags) {
|
2327
|
+
if (rfrag.isErr()) {
|
2328
|
+
return import_cement.Result.Err(rfrag.Err());
|
2329
|
+
}
|
2330
|
+
const frag = rfrag.Ok();
|
2331
|
+
buffer = buffer || new Uint8Array(frag.len);
|
2332
|
+
buffer.set(frag.data, frag.ofs);
|
2333
|
+
}
|
2334
|
+
return import_cement.Result.Ok(buffer || new Uint8Array(0));
|
2335
|
+
}
|
2336
|
+
async delete(url) {
|
2337
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
2338
|
+
for (const rfrag of rfrags) {
|
2339
|
+
if (rfrag.isErr()) {
|
2340
|
+
return import_cement.Result.Err(rfrag.Err());
|
2341
|
+
}
|
2342
|
+
const frag = rfrag.Ok();
|
2343
|
+
const fidStr = import_base584.base58btc.encode(frag.fid);
|
2344
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
2345
|
+
await this.innerGW.delete(fragUrl);
|
2346
|
+
}
|
2347
|
+
return import_cement.Result.Ok(void 0);
|
2348
|
+
}
|
2349
|
+
};
|
2350
|
+
|
2230
2351
|
// src/blockstore/store.ts
|
2231
|
-
function guardVersion(
|
2232
|
-
if (!
|
2233
|
-
return
|
2352
|
+
function guardVersion(url) {
|
2353
|
+
if (!url.hasParam("version")) {
|
2354
|
+
return import_cement7.Result.Err(`missing version: ${url.toString()}`);
|
2234
2355
|
}
|
2235
|
-
return
|
2356
|
+
return import_cement7.Result.Ok(url);
|
2236
2357
|
}
|
2237
|
-
var
|
2238
|
-
constructor(name,
|
2358
|
+
var BaseStoreImpl = class {
|
2359
|
+
constructor(name, url, opts, sthis, logger) {
|
2239
2360
|
this._onStarted = [];
|
2240
2361
|
this._onClosed = [];
|
2241
2362
|
this.name = name;
|
2242
|
-
this.
|
2243
|
-
this.
|
2363
|
+
this._url = url;
|
2364
|
+
this.keybag = opts.keybag;
|
2365
|
+
this.sthis = sthis;
|
2366
|
+
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
2367
|
+
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
2368
|
+
}
|
2369
|
+
url() {
|
2370
|
+
return this._url;
|
2244
2371
|
}
|
2245
2372
|
onStarted(fn) {
|
2246
2373
|
this._onStarted.push(fn);
|
@@ -2248,110 +2375,168 @@ var VersionedStore = class {
|
|
2248
2375
|
onClosed(fn) {
|
2249
2376
|
this._onClosed.push(fn);
|
2250
2377
|
}
|
2378
|
+
async keyedCrypto() {
|
2379
|
+
return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
|
2380
|
+
}
|
2381
|
+
async start() {
|
2382
|
+
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
2383
|
+
this._url = this._url.build().setParam("store", this.storeType).URI();
|
2384
|
+
const res = await this.gateway.start(this._url);
|
2385
|
+
if (res.isErr()) {
|
2386
|
+
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
2387
|
+
return res;
|
2388
|
+
}
|
2389
|
+
this._url = res.Ok();
|
2390
|
+
const kb = await this.keybag();
|
2391
|
+
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
2392
|
+
const idx = this._url.getParam("index");
|
2393
|
+
const storeKeyName = [this.name];
|
2394
|
+
if (idx) {
|
2395
|
+
storeKeyName.push(idx);
|
2396
|
+
}
|
2397
|
+
storeKeyName.push(this.storeType);
|
2398
|
+
return storeKeyName.join(":");
|
2399
|
+
});
|
2400
|
+
if (skRes.isErr()) {
|
2401
|
+
return skRes;
|
2402
|
+
}
|
2403
|
+
this._url = skRes.Ok();
|
2404
|
+
const version = guardVersion(this._url);
|
2405
|
+
if (version.isErr()) {
|
2406
|
+
this.logger.Error().Result("version", version).Msg("guardVersion");
|
2407
|
+
await this.close();
|
2408
|
+
return version;
|
2409
|
+
}
|
2410
|
+
if (this.ready) {
|
2411
|
+
const fn = this.ready.bind(this);
|
2412
|
+
const ready = await exception2Result(fn);
|
2413
|
+
if (ready.isErr()) {
|
2414
|
+
await this.close();
|
2415
|
+
return ready;
|
2416
|
+
}
|
2417
|
+
}
|
2418
|
+
this._onStarted.forEach((fn) => fn());
|
2419
|
+
this.logger.Debug().Msg("started");
|
2420
|
+
return version;
|
2421
|
+
}
|
2251
2422
|
};
|
2252
|
-
var
|
2253
|
-
|
2254
|
-
|
2255
|
-
|
2256
|
-
|
2257
|
-
|
2258
|
-
|
2259
|
-
|
2260
|
-
|
2423
|
+
var MetaStoreImpl = class extends BaseStoreImpl {
|
2424
|
+
constructor(sthis, name, url, opts) {
|
2425
|
+
super(
|
2426
|
+
name,
|
2427
|
+
url,
|
2428
|
+
{
|
2429
|
+
...opts
|
2430
|
+
},
|
2431
|
+
sthis,
|
2432
|
+
ensureLogger(sthis, "MetaStoreImpl")
|
2433
|
+
);
|
2434
|
+
this.storeType = "meta";
|
2435
|
+
this.subscribers = /* @__PURE__ */ new Map();
|
2436
|
+
}
|
2437
|
+
onLoad(branch, loadHandler) {
|
2438
|
+
const subscribers = this.subscribers.get(branch) || [];
|
2439
|
+
subscribers.push(loadHandler);
|
2440
|
+
this.subscribers.set(branch, subscribers);
|
2441
|
+
return () => {
|
2442
|
+
const subscribers2 = this.subscribers.get(branch) || [];
|
2443
|
+
const idx = subscribers2.indexOf(loadHandler);
|
2444
|
+
if (idx > -1) subscribers2.splice(idx, 1);
|
2445
|
+
};
|
2446
|
+
}
|
2447
|
+
makeHeader({ cars }) {
|
2261
2448
|
const toEncode = { cars };
|
2262
|
-
if (key) toEncode.key = key;
|
2263
2449
|
return (0, import_dag_json.format)(toEncode);
|
2264
2450
|
}
|
2265
2451
|
parseHeader(headerData) {
|
2266
2452
|
const got = (0, import_dag_json.parse)(headerData);
|
2267
2453
|
return got;
|
2268
2454
|
}
|
2269
|
-
async
|
2270
|
-
|
2271
|
-
|
2272
|
-
|
2273
|
-
|
2455
|
+
async handleSubscribers(dbMetas, branch) {
|
2456
|
+
try {
|
2457
|
+
const subscribers = this.subscribers.get(branch) || [];
|
2458
|
+
await Promise.all(subscribers.map((subscriber) => subscriber(dbMetas)));
|
2459
|
+
} catch (e) {
|
2460
|
+
this.logger.Error().Err(e).Msg("handleSubscribers").AsError();
|
2274
2461
|
}
|
2275
|
-
|
2276
|
-
|
2462
|
+
}
|
2463
|
+
async handleByteHeads(byteHeads, branch = "main") {
|
2464
|
+
let dbMetas;
|
2465
|
+
try {
|
2466
|
+
dbMetas = this.dbMetasForByteHeads(byteHeads);
|
2467
|
+
} catch (e) {
|
2468
|
+
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
2469
|
+
}
|
2470
|
+
await this.handleSubscribers(dbMetas, branch);
|
2471
|
+
return dbMetas;
|
2472
|
+
}
|
2473
|
+
dbMetasForByteHeads(byteHeads) {
|
2474
|
+
return byteHeads.map((bytes) => {
|
2475
|
+
const txt = this.sthis.txt.decode(bytes);
|
2476
|
+
return this.parseHeader(txt);
|
2477
|
+
});
|
2277
2478
|
}
|
2278
2479
|
async load(branch) {
|
2279
|
-
|
2280
|
-
|
2281
|
-
|
2282
|
-
|
2480
|
+
branch = branch || "main";
|
2481
|
+
this.logger.Debug().Str("branch", branch).Msg("loading");
|
2482
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2483
|
+
if (url.isErr()) {
|
2484
|
+
throw this.logger.Error().Result("buidUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2283
2485
|
}
|
2284
|
-
const bytes = await this.gateway.get(
|
2486
|
+
const bytes = await this.gateway.get(url.Ok());
|
2285
2487
|
if (bytes.isErr()) {
|
2286
2488
|
if (isNotFoundError(bytes)) {
|
2287
2489
|
return void 0;
|
2288
2490
|
}
|
2289
|
-
throw this.logger.Error().Url(
|
2290
|
-
}
|
2291
|
-
try {
|
2292
|
-
return [this.parseHeader(textDecoder.decode(bytes.Ok()))];
|
2293
|
-
} catch (e) {
|
2294
|
-
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
2491
|
+
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
2295
2492
|
}
|
2493
|
+
return this.handleByteHeads([bytes.Ok()], branch);
|
2296
2494
|
}
|
2297
|
-
async save(meta, branch
|
2495
|
+
async save(meta, branch) {
|
2496
|
+
branch = branch || "main";
|
2298
2497
|
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
2299
2498
|
const bytes = this.makeHeader(meta);
|
2300
|
-
const
|
2301
|
-
if (
|
2302
|
-
throw this.logger.Error().Err(
|
2499
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2500
|
+
if (url.isErr()) {
|
2501
|
+
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2303
2502
|
}
|
2304
|
-
const res = await this.gateway.put(
|
2503
|
+
const res = await this.gateway.put(url.Ok(), this.sthis.txt.encode(bytes));
|
2305
2504
|
if (res.isErr()) {
|
2306
2505
|
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2307
2506
|
}
|
2308
|
-
|
2507
|
+
await this.handleSubscribers([meta], branch);
|
2508
|
+
return res;
|
2309
2509
|
}
|
2310
2510
|
async close() {
|
2311
|
-
await this.gateway.close(this.url);
|
2511
|
+
await this.gateway.close(this.url());
|
2312
2512
|
this._onClosed.forEach((fn) => fn());
|
2313
|
-
return
|
2513
|
+
return import_cement7.Result.Ok(void 0);
|
2314
2514
|
}
|
2315
2515
|
async destroy() {
|
2316
|
-
return this.gateway.destroy(this.url);
|
2516
|
+
return this.gateway.destroy(this.url());
|
2317
2517
|
}
|
2318
2518
|
};
|
2319
|
-
var
|
2320
|
-
|
2519
|
+
var DataStoreImpl = class extends BaseStoreImpl {
|
2520
|
+
// readonly tag: string = "car-base";
|
2521
|
+
constructor(sthis, name, url, opts) {
|
2321
2522
|
super(
|
2322
2523
|
name,
|
2323
|
-
|
2324
|
-
|
2325
|
-
|
2326
|
-
}
|
2524
|
+
url,
|
2525
|
+
{
|
2526
|
+
...opts
|
2527
|
+
},
|
2528
|
+
sthis,
|
2529
|
+
ensureLogger(sthis, "DataStoreImpl")
|
2327
2530
|
);
|
2328
|
-
this.
|
2329
|
-
this.gateway = gateway;
|
2330
|
-
}
|
2331
|
-
async start() {
|
2332
|
-
this.logger.Debug().Msg("starting-gateway");
|
2333
|
-
const res = await this.gateway.start(this.url);
|
2334
|
-
if (res.isErr()) {
|
2335
|
-
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
2336
|
-
return res;
|
2337
|
-
}
|
2338
|
-
this._onStarted.forEach((fn) => fn());
|
2339
|
-
const version = guardVersion(this.url);
|
2340
|
-
if (version.isErr()) {
|
2341
|
-
this.logger.Error().Result("version", version).Msg("guardVersion");
|
2342
|
-
await this.close();
|
2343
|
-
return version;
|
2344
|
-
}
|
2345
|
-
this.logger.Debug().Msg("started");
|
2346
|
-
return version;
|
2531
|
+
this.storeType = "data";
|
2347
2532
|
}
|
2348
2533
|
async load(cid) {
|
2349
2534
|
this.logger.Debug().Any("cid", cid).Msg("loading");
|
2350
|
-
const
|
2351
|
-
if (
|
2352
|
-
throw this.logger.Error().Err(
|
2535
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
2536
|
+
if (url.isErr()) {
|
2537
|
+
throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
|
2353
2538
|
}
|
2354
|
-
const res = await this.gateway.get(
|
2539
|
+
const res = await this.gateway.get(url.Ok());
|
2355
2540
|
if (res.isErr()) {
|
2356
2541
|
throw res.Err();
|
2357
2542
|
}
|
@@ -2360,57 +2545,64 @@ var DataStore = class extends VersionedStore {
|
|
2360
2545
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2361
2546
|
async save(car, opts) {
|
2362
2547
|
this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
|
2363
|
-
const
|
2364
|
-
if (
|
2365
|
-
throw this.logger.Error().Err(
|
2548
|
+
const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
|
2549
|
+
if (url.isErr()) {
|
2550
|
+
throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
|
2366
2551
|
}
|
2367
|
-
const res = await this.gateway.put(
|
2552
|
+
const res = await this.gateway.put(url.Ok(), car.bytes);
|
2368
2553
|
if (res.isErr()) {
|
2369
2554
|
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2370
2555
|
}
|
2371
2556
|
return res.Ok();
|
2372
2557
|
}
|
2373
2558
|
async remove(cid) {
|
2374
|
-
const
|
2375
|
-
if (
|
2376
|
-
return
|
2559
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
2560
|
+
if (url.isErr()) {
|
2561
|
+
return url;
|
2377
2562
|
}
|
2378
|
-
return this.gateway.delete(
|
2563
|
+
return this.gateway.delete(url.Ok());
|
2379
2564
|
}
|
2380
2565
|
async close() {
|
2381
|
-
await this.gateway.close(this.url);
|
2566
|
+
await this.gateway.close(this.url());
|
2382
2567
|
this._onClosed.forEach((fn) => fn());
|
2383
|
-
return
|
2568
|
+
return import_cement7.Result.Ok(void 0);
|
2384
2569
|
}
|
2385
2570
|
destroy() {
|
2386
|
-
return this.gateway.destroy(this.url);
|
2571
|
+
return this.gateway.destroy(this.url());
|
2387
2572
|
}
|
2388
2573
|
};
|
2389
|
-
var
|
2390
|
-
constructor(loader,
|
2391
|
-
super(
|
2392
|
-
|
2393
|
-
|
2574
|
+
var WALStoreImpl = class extends BaseStoreImpl {
|
2575
|
+
constructor(loader, url, opts) {
|
2576
|
+
super(
|
2577
|
+
loader.name,
|
2578
|
+
url,
|
2579
|
+
{
|
2580
|
+
...opts
|
2581
|
+
},
|
2582
|
+
loader.sthis,
|
2583
|
+
ensureLogger(loader.sthis, "WALStoreImpl")
|
2584
|
+
);
|
2585
|
+
this.storeType = "wal";
|
2586
|
+
this._ready = new import_cement7.ResolveOnce();
|
2394
2587
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
2395
2588
|
this.processing = void 0;
|
2396
2589
|
this.processQueue = new CommitQueue();
|
2397
|
-
this.
|
2398
|
-
|
2399
|
-
|
2400
|
-
|
2401
|
-
|
2402
|
-
|
2403
|
-
|
2404
|
-
|
2590
|
+
this.ready = async () => {
|
2591
|
+
return this._ready.once(async () => {
|
2592
|
+
const walState = await this.load().catch((e) => {
|
2593
|
+
this.logger.Error().Any("error", e).Msg("error loading wal");
|
2594
|
+
return void 0;
|
2595
|
+
});
|
2596
|
+
if (!walState) {
|
2597
|
+
this.walState.operations = [];
|
2598
|
+
this.walState.fileOperations = [];
|
2599
|
+
} else {
|
2600
|
+
this.walState.operations = walState.operations || [];
|
2601
|
+
this.walState.fileOperations = walState.fileOperations || [];
|
2602
|
+
}
|
2405
2603
|
});
|
2406
|
-
|
2407
|
-
|
2408
|
-
this.walState.fileOperations = [];
|
2409
|
-
} else {
|
2410
|
-
this.walState.operations = walState.operations || [];
|
2411
|
-
this.walState.fileOperations = walState.fileOperations || [];
|
2412
|
-
}
|
2413
|
-
});
|
2604
|
+
};
|
2605
|
+
this.loader = loader;
|
2414
2606
|
}
|
2415
2607
|
async enqueue(dbMeta, opts) {
|
2416
2608
|
await this.ready();
|
@@ -2420,19 +2612,23 @@ var RemoteWAL = class extends VersionedStore {
|
|
2420
2612
|
this.walState.operations.push(dbMeta);
|
2421
2613
|
}
|
2422
2614
|
await this.save(this.walState);
|
2423
|
-
void this.
|
2615
|
+
void this.process();
|
2424
2616
|
}
|
2425
2617
|
async enqueueFile(fileCid, publicFile = false) {
|
2426
2618
|
await this.ready();
|
2427
2619
|
this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
|
2428
2620
|
}
|
2429
|
-
async
|
2621
|
+
async process() {
|
2430
2622
|
await this.ready();
|
2431
2623
|
if (!this.loader.remoteCarStore) return;
|
2432
2624
|
await this.processQueue.enqueue(async () => {
|
2433
|
-
|
2625
|
+
try {
|
2626
|
+
await this._doProcess();
|
2627
|
+
} catch (e) {
|
2628
|
+
this.logger.Error().Any("error", e).Msg("error processing wal");
|
2629
|
+
}
|
2434
2630
|
if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
|
2435
|
-
setTimeout(() => void this.
|
2631
|
+
setTimeout(() => void this.process(), 0);
|
2436
2632
|
}
|
2437
2633
|
});
|
2438
2634
|
}
|
@@ -2450,7 +2646,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2450
2646
|
for (const cid of dbMeta.cars) {
|
2451
2647
|
const car = await (await this.loader.carStore()).load(cid);
|
2452
2648
|
if (!car) {
|
2453
|
-
if (
|
2649
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
2454
2650
|
throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
|
2455
2651
|
} else {
|
2456
2652
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -2465,7 +2661,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2465
2661
|
for (const cid of dbMeta.cars) {
|
2466
2662
|
const car = await (await this.loader.carStore()).load(cid).catch(() => null);
|
2467
2663
|
if (!car) {
|
2468
|
-
if (
|
2664
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
2469
2665
|
throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
|
2470
2666
|
} else {
|
2471
2667
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -2490,11 +2686,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2490
2686
|
const res = await Promise.allSettled(uploads);
|
2491
2687
|
const errors = res.filter((r) => r.status === "rejected");
|
2492
2688
|
if (errors.length) {
|
2493
|
-
throw this.logger.Error().Any(
|
2494
|
-
"errors",
|
2495
|
-
errors.map((e) => e.reason)
|
2496
|
-
).Msg("error uploading").AsError();
|
2497
|
-
errors[0].reason;
|
2689
|
+
throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
|
2498
2690
|
}
|
2499
2691
|
if (operations.length) {
|
2500
2692
|
const lastOp = operations[operations.length - 1];
|
@@ -2509,29 +2701,11 @@ var RemoteWAL = class extends VersionedStore {
|
|
2509
2701
|
})();
|
2510
2702
|
await rmlp;
|
2511
2703
|
}
|
2512
|
-
async start() {
|
2513
|
-
const res = await this.gateway.start(this.url);
|
2514
|
-
if (res.isErr()) {
|
2515
|
-
return res;
|
2516
|
-
}
|
2517
|
-
const ver = guardVersion(this.url);
|
2518
|
-
if (ver.isErr()) {
|
2519
|
-
await this.close();
|
2520
|
-
return ver;
|
2521
|
-
}
|
2522
|
-
const ready = await exception2Result(() => this.ready());
|
2523
|
-
this._onStarted.forEach((fn) => fn());
|
2524
|
-
if (ready.isErr()) {
|
2525
|
-
await this.close();
|
2526
|
-
return ready;
|
2527
|
-
}
|
2528
|
-
return ready;
|
2529
|
-
}
|
2530
2704
|
async load() {
|
2531
2705
|
this.logger.Debug().Msg("loading");
|
2532
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
2706
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
2533
2707
|
if (filepath.isErr()) {
|
2534
|
-
throw this.logger.Error().Err(filepath.Err()).
|
2708
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
2535
2709
|
}
|
2536
2710
|
const bytes = await this.gateway.get(filepath.Ok());
|
2537
2711
|
if (bytes.isErr()) {
|
@@ -2541,15 +2715,15 @@ var RemoteWAL = class extends VersionedStore {
|
|
2541
2715
|
throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
|
2542
2716
|
}
|
2543
2717
|
try {
|
2544
|
-
return bytes && (0, import_dag_json.parse)(
|
2718
|
+
return bytes && (0, import_dag_json.parse)(this.sthis.txt.decode(bytes.Ok()));
|
2545
2719
|
} catch (e) {
|
2546
2720
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2547
2721
|
}
|
2548
2722
|
}
|
2549
2723
|
async save(state) {
|
2550
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
2724
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
2551
2725
|
if (filepath.isErr()) {
|
2552
|
-
throw this.logger.Error().Err(filepath.Err()).
|
2726
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
2553
2727
|
}
|
2554
2728
|
let encoded;
|
2555
2729
|
try {
|
@@ -2557,56 +2731,68 @@ var RemoteWAL = class extends VersionedStore {
|
|
2557
2731
|
} catch (e) {
|
2558
2732
|
throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
|
2559
2733
|
}
|
2560
|
-
const res = await this.gateway.put(filepath.Ok(),
|
2734
|
+
const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
|
2561
2735
|
if (res.isErr()) {
|
2562
2736
|
throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
|
2563
2737
|
}
|
2564
2738
|
}
|
2565
2739
|
async close() {
|
2566
|
-
await this.gateway.close(this.url);
|
2740
|
+
await this.gateway.close(this.url());
|
2567
2741
|
this._onClosed.forEach((fn) => fn());
|
2568
|
-
return
|
2742
|
+
return import_cement7.Result.Ok(void 0);
|
2569
2743
|
}
|
2570
2744
|
destroy() {
|
2571
|
-
return this.gateway.destroy(this.url);
|
2745
|
+
return this.gateway.destroy(this.url());
|
2572
2746
|
}
|
2573
2747
|
};
|
2574
2748
|
|
2575
2749
|
// src/blockstore/store-factory.ts
|
2576
2750
|
init_utils();
|
2577
|
-
function ensureIsIndex(
|
2751
|
+
function ensureIsIndex(url, isIndex) {
|
2578
2752
|
if (isIndex) {
|
2579
|
-
|
2580
|
-
return url2;
|
2581
|
-
} else {
|
2582
|
-
url2.searchParams.delete("index");
|
2583
|
-
return url2;
|
2753
|
+
return url.build().setParam("index", isIndex).URI();
|
2584
2754
|
}
|
2755
|
+
return url.build().delParam("index").URI();
|
2585
2756
|
}
|
2586
|
-
function
|
2587
|
-
if (
|
2588
|
-
|
2589
|
-
const url2 = new URL(pathOrUrl);
|
2590
|
-
return ensureIsIndex(url2, isIndex);
|
2591
|
-
} catch (e) {
|
2592
|
-
const url2 = new URL(`file://${pathOrUrl}`);
|
2593
|
-
return ensureIsIndex(url2, isIndex);
|
2757
|
+
function ensureName(name, url) {
|
2758
|
+
if (!url.hasParam("name")) {
|
2759
|
+
return url.build().setParam("name", name).URI();
|
2594
2760
|
}
|
2761
|
+
return url;
|
2595
2762
|
}
|
2596
2763
|
var storeFactory = /* @__PURE__ */ new Map();
|
2597
|
-
function ensureName(name, url2) {
|
2598
|
-
if (!url2.searchParams.has("name")) {
|
2599
|
-
url2.searchParams.set("name", name);
|
2600
|
-
}
|
2601
|
-
}
|
2602
2764
|
function buildURL(optURL, loader) {
|
2603
2765
|
const storeOpts = loader.ebOpts.store;
|
2604
2766
|
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
2605
2767
|
let obuUrl;
|
2606
2768
|
if (obuItem && obuItem.overrideBaseURL) {
|
2607
|
-
obuUrl =
|
2769
|
+
obuUrl = import_cement10.URI.from(obuItem.overrideBaseURL);
|
2608
2770
|
}
|
2609
|
-
|
2771
|
+
const ret = ensureIsIndex(
|
2772
|
+
import_cement10.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2773
|
+
storeOpts.isIndex
|
2774
|
+
);
|
2775
|
+
return ret;
|
2776
|
+
}
|
2777
|
+
var onceGateway = new import_cement10.KeyedResolvOnce();
|
2778
|
+
async function getGatewayFromURL(url, sthis) {
|
2779
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2780
|
+
const item = storeFactory.get(url.protocol);
|
2781
|
+
if (item) {
|
2782
|
+
const ret = {
|
2783
|
+
gateway: await item.gateway(sthis),
|
2784
|
+
test: await item.test(sthis)
|
2785
|
+
};
|
2786
|
+
const res = await ret.gateway.start(url);
|
2787
|
+
if (res.isErr()) {
|
2788
|
+
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2789
|
+
return void 0;
|
2790
|
+
}
|
2791
|
+
return ret;
|
2792
|
+
}
|
2793
|
+
sthis.logger.Warn().Url(url).Msg("unsupported protocol");
|
2794
|
+
return void 0;
|
2795
|
+
});
|
2610
2796
|
}
|
2611
2797
|
function registerStoreProtocol(item) {
|
2612
2798
|
let protocol = item.protocol;
|
@@ -2615,8 +2801,7 @@ function registerStoreProtocol(item) {
|
|
2615
2801
|
}
|
2616
2802
|
if (storeFactory.has(protocol)) {
|
2617
2803
|
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
2618
|
-
|
2619
|
-
logger.Warn().Msg(`protocol ${protocol} already registered`);
|
2804
|
+
throw new Error(`we need a logger here`);
|
2620
2805
|
return () => {
|
2621
2806
|
};
|
2622
2807
|
}
|
@@ -2631,106 +2816,83 @@ function registerStoreProtocol(item) {
|
|
2631
2816
|
storeFactory.delete(protocol);
|
2632
2817
|
};
|
2633
2818
|
}
|
2634
|
-
function runStoreFactory(url2, logger, run) {
|
2635
|
-
const item = storeFactory.get(url2.protocol);
|
2636
|
-
if (!item) {
|
2637
|
-
throw logger.Error().Url(url2).Str("protocol", url2.protocol).Any("keys", Array(storeFactory.keys())).Msg(`unsupported protocol`).AsError();
|
2638
|
-
}
|
2639
|
-
logger.Debug().Str("protocol", url2.protocol).Msg("run");
|
2640
|
-
return run(item);
|
2641
|
-
}
|
2642
|
-
var onceLoadDataGateway = new import_cement8.KeyedResolvOnce();
|
2643
|
-
function loadDataGateway(url2, logger) {
|
2644
|
-
return onceLoadDataGateway.get(url2.protocol).once(async () => {
|
2645
|
-
return runStoreFactory(url2, logger, async (item) => item.data(logger));
|
2646
|
-
});
|
2647
|
-
}
|
2648
|
-
var onceDataStoreFactory = new import_cement8.KeyedResolvOnce();
|
2649
2819
|
async function dataStoreFactory(loader) {
|
2650
|
-
const
|
2651
|
-
|
2652
|
-
const
|
2653
|
-
|
2654
|
-
|
2655
|
-
|
2656
|
-
|
2657
|
-
|
2658
|
-
|
2659
|
-
|
2660
|
-
}
|
2661
|
-
logger.Debug().Str("prepared", store.url.toString()).Msg("produced");
|
2662
|
-
return store;
|
2663
|
-
});
|
2664
|
-
}
|
2665
|
-
var onceLoadMetaGateway = new import_cement8.KeyedResolvOnce();
|
2666
|
-
function loadMetaGateway(url2, logger) {
|
2667
|
-
return onceLoadMetaGateway.get(url2.protocol).once(async () => {
|
2668
|
-
return runStoreFactory(url2, logger, async (item) => item.meta(logger));
|
2820
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
2821
|
+
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
2822
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2823
|
+
if (!gateway) {
|
2824
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2825
|
+
}
|
2826
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2827
|
+
gateway: gateway.gateway,
|
2828
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2829
|
+
...loader.ebOpts.keyBag
|
2830
|
+
})
|
2669
2831
|
});
|
2832
|
+
return store;
|
2670
2833
|
}
|
2671
|
-
var onceMetaStoreFactory = new import_cement8.KeyedResolvOnce();
|
2672
2834
|
async function metaStoreFactory(loader) {
|
2673
|
-
const
|
2674
|
-
|
2675
|
-
|
2676
|
-
|
2677
|
-
|
2678
|
-
logger.
|
2679
|
-
|
2680
|
-
|
2681
|
-
|
2682
|
-
|
2683
|
-
|
2684
|
-
}
|
2685
|
-
return store;
|
2686
|
-
});
|
2687
|
-
}
|
2688
|
-
var onceWalGateway = new import_cement8.KeyedResolvOnce();
|
2689
|
-
function loadWalGateway(url2, logger) {
|
2690
|
-
return onceWalGateway.get(url2.protocol).once(async () => {
|
2691
|
-
return runStoreFactory(url2, logger, async (item) => item.wal(logger));
|
2835
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
2836
|
+
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
2837
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2838
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2839
|
+
if (!gateway) {
|
2840
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2841
|
+
}
|
2842
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2843
|
+
gateway: gateway.gateway,
|
2844
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2845
|
+
...loader.ebOpts.keyBag
|
2846
|
+
})
|
2692
2847
|
});
|
2848
|
+
return store;
|
2693
2849
|
}
|
2694
|
-
var onceRemoteWalFactory = new import_cement8.KeyedResolvOnce();
|
2695
2850
|
async function remoteWalFactory(loader) {
|
2696
|
-
const
|
2697
|
-
|
2698
|
-
const
|
2699
|
-
|
2700
|
-
|
2701
|
-
|
2702
|
-
|
2703
|
-
|
2704
|
-
|
2705
|
-
|
2706
|
-
|
2707
|
-
}
|
2708
|
-
return store;
|
2851
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
2852
|
+
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
2853
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2854
|
+
if (!gateway) {
|
2855
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2856
|
+
}
|
2857
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2858
|
+
const store = new WALStoreImpl(loader, url, {
|
2859
|
+
gateway: gateway.gateway,
|
2860
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2861
|
+
...loader.ebOpts.keyBag
|
2862
|
+
})
|
2709
2863
|
});
|
2864
|
+
return store;
|
2710
2865
|
}
|
2711
|
-
async function testStoreFactory(
|
2712
|
-
|
2713
|
-
|
2714
|
-
|
2715
|
-
|
2716
|
-
|
2717
|
-
|
2718
|
-
|
2866
|
+
async function testStoreFactory(url, sthis) {
|
2867
|
+
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
2868
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2869
|
+
if (!gateway) {
|
2870
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2871
|
+
}
|
2872
|
+
return gateway.test;
|
2873
|
+
}
|
2874
|
+
async function ensureStart(store, logger) {
|
2875
|
+
const ret = await store.start();
|
2876
|
+
if (ret.isErr()) {
|
2877
|
+
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2878
|
+
}
|
2879
|
+
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2880
|
+
return store;
|
2719
2881
|
}
|
2720
|
-
function toStoreRuntime(opts,
|
2721
|
-
const logger = ensureLogger(
|
2882
|
+
function toStoreRuntime(opts, sthis) {
|
2883
|
+
const logger = ensureLogger(sthis, "toStoreRuntime", {});
|
2722
2884
|
return {
|
2723
|
-
makeMetaStore: (loader) => {
|
2885
|
+
makeMetaStore: async (loader) => {
|
2724
2886
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
|
2725
|
-
return (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader);
|
2887
|
+
return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
|
2726
2888
|
},
|
2727
|
-
makeDataStore: (loader) => {
|
2889
|
+
makeDataStore: async (loader) => {
|
2728
2890
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
|
2729
|
-
return (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader);
|
2891
|
+
return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
|
2730
2892
|
},
|
2731
|
-
|
2732
|
-
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.
|
2733
|
-
return (loader.ebOpts.store.
|
2893
|
+
makeWALStore: async (loader) => {
|
2894
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
|
2895
|
+
return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
|
2734
2896
|
},
|
2735
2897
|
encodeFile: opts.encodeFile || encodeFile,
|
2736
2898
|
decodeFile: opts.decodeFile || decodeFile
|
@@ -2738,48 +2900,202 @@ function toStoreRuntime(opts, ilogger) {
|
|
2738
2900
|
}
|
2739
2901
|
registerStoreProtocol({
|
2740
2902
|
protocol: "file:",
|
2741
|
-
|
2742
|
-
const {
|
2743
|
-
return new
|
2744
|
-
},
|
2745
|
-
meta: async (logger) => {
|
2746
|
-
const { FileMetaGateway: FileMetaGateway2 } = await Promise.resolve().then(() => (init_store_file(), store_file_exports));
|
2747
|
-
return new FileMetaGateway2(logger);
|
2748
|
-
},
|
2749
|
-
wal: async (logger) => {
|
2750
|
-
const { FileWALGateway: FileWALGateway2 } = await Promise.resolve().then(() => (init_store_file(), store_file_exports));
|
2751
|
-
return new FileWALGateway2(logger);
|
2903
|
+
gateway: async (logger) => {
|
2904
|
+
const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
2905
|
+
return new FileGateway2(logger);
|
2752
2906
|
},
|
2753
2907
|
test: async (logger) => {
|
2754
|
-
const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (
|
2908
|
+
const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
2755
2909
|
return new FileTestStore2(logger);
|
2756
2910
|
}
|
2757
2911
|
});
|
2758
2912
|
registerStoreProtocol({
|
2759
2913
|
protocol: "indexdb:",
|
2760
|
-
|
2761
|
-
const {
|
2762
|
-
return new
|
2763
|
-
},
|
2764
|
-
meta: async (logger) => {
|
2765
|
-
const { IndexDBMetaGateway: IndexDBMetaGateway2 } = await Promise.resolve().then(() => (init_store_indexdb(), store_indexdb_exports));
|
2766
|
-
return new IndexDBMetaGateway2(logger);
|
2767
|
-
},
|
2768
|
-
wal: async (logger) => {
|
2769
|
-
const { IndexDBMetaGateway: IndexDBMetaGateway2 } = await Promise.resolve().then(() => (init_store_indexdb(), store_indexdb_exports));
|
2770
|
-
return new IndexDBMetaGateway2(logger);
|
2914
|
+
gateway: async (logger) => {
|
2915
|
+
const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
2916
|
+
return new IndexDBGateway2(logger);
|
2771
2917
|
},
|
2772
2918
|
test: async (logger) => {
|
2773
|
-
const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (
|
2919
|
+
const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
2774
2920
|
return new IndexDBTestStore2(logger);
|
2775
2921
|
}
|
2776
2922
|
});
|
2777
2923
|
|
2778
|
-
// src/blockstore/
|
2779
|
-
|
2924
|
+
// src/blockstore/connection-base.ts
|
2925
|
+
var import_clock = require("@web3-storage/pail/clock");
|
2926
|
+
var import_block6 = require("@web3-storage/pail/block");
|
2927
|
+
|
2928
|
+
// src/blockstore/task-manager.ts
|
2929
|
+
init_utils();
|
2930
|
+
var TaskManager = class {
|
2931
|
+
constructor(loader) {
|
2932
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
2933
|
+
this.queue = [];
|
2934
|
+
this.isProcessing = false;
|
2935
|
+
this.loader = loader;
|
2936
|
+
this.logger = ensureLogger(loader.sthis, "TaskManager");
|
2937
|
+
}
|
2938
|
+
async handleEvent(eventBlock) {
|
2939
|
+
const cid = eventBlock.cid.toString();
|
2940
|
+
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
2941
|
+
for (const parent of parents) {
|
2942
|
+
this.eventsWeHandled.add(parent);
|
2943
|
+
}
|
2944
|
+
this.queue.push({ cid, eventBlock, retries: 0 });
|
2945
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
2946
|
+
void this.processQueue();
|
2947
|
+
}
|
2948
|
+
async processQueue() {
|
2949
|
+
if (this.isProcessing) return;
|
2950
|
+
this.isProcessing = true;
|
2951
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2952
|
+
const first = filteredQueue[0];
|
2953
|
+
if (!first) {
|
2954
|
+
return;
|
2955
|
+
}
|
2956
|
+
try {
|
2957
|
+
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
2958
|
+
this.eventsWeHandled.add(first.cid);
|
2959
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2960
|
+
} catch (err) {
|
2961
|
+
if (first.retries++ > 3) {
|
2962
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
2963
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
2964
|
+
}
|
2965
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
2966
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
2967
|
+
} finally {
|
2968
|
+
this.isProcessing = false;
|
2969
|
+
if (this.queue.length > 0) {
|
2970
|
+
void this.processQueue();
|
2971
|
+
}
|
2972
|
+
}
|
2973
|
+
}
|
2974
|
+
};
|
2975
|
+
|
2976
|
+
// src/blockstore/store-remote.ts
|
2977
|
+
async function RemoteDataStore(sthis, name, url, opts) {
|
2978
|
+
const ds = new DataStoreImpl(sthis, name, url, opts);
|
2979
|
+
await ds.start();
|
2980
|
+
return ds;
|
2981
|
+
}
|
2982
|
+
async function RemoteMetaStore(sthis, name, url, opts) {
|
2983
|
+
const ms = new MetaStoreImpl(sthis, name, url, opts);
|
2984
|
+
await ms.start();
|
2985
|
+
return ms;
|
2986
|
+
}
|
2987
|
+
|
2988
|
+
// src/blockstore/connection-base.ts
|
2989
|
+
var ConnectionBase = class {
|
2990
|
+
constructor(url, logger) {
|
2991
|
+
// readonly ready: Promise<unknown>;
|
2992
|
+
// todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
|
2993
|
+
this.eventBlocks = new import_block6.MemoryBlockstore();
|
2994
|
+
this.parents = [];
|
2995
|
+
this.loaded = Promise.resolve();
|
2996
|
+
this.logger = logger;
|
2997
|
+
this.url = url;
|
2998
|
+
}
|
2999
|
+
async refresh() {
|
3000
|
+
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load("main");
|
3001
|
+
await (await throwFalsy(this.loader).WALStore()).process();
|
3002
|
+
}
|
3003
|
+
async connect_X({ loader }) {
|
3004
|
+
if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
|
3005
|
+
await this.connectMeta_X({ loader });
|
3006
|
+
await this.connectStorage_X({ loader });
|
3007
|
+
}
|
3008
|
+
async connectMeta_X({ loader }) {
|
3009
|
+
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
3010
|
+
this.loader = loader;
|
3011
|
+
this.taskManager = new TaskManager(loader);
|
3012
|
+
await this.onConnect();
|
3013
|
+
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
3014
|
+
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
3015
|
+
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
3016
|
+
const name = metaUrl.toString();
|
3017
|
+
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
3018
|
+
gateway: gateway.gateway,
|
3019
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag)
|
3020
|
+
});
|
3021
|
+
remote.onLoad("main", async (metas) => {
|
3022
|
+
if (metas) {
|
3023
|
+
this.logger.Debug().Any("metas", metas).Bool("loader", this.loader).Msg("connectMeta_X: handleDbMetasFromStore pre");
|
3024
|
+
await throwFalsy(this.loader).handleDbMetasFromStore(metas);
|
3025
|
+
this.logger.Debug().Any("metas", metas).Msg("connectMeta_X: handleDbMetasFromStore post");
|
3026
|
+
}
|
3027
|
+
});
|
3028
|
+
this.loader.remoteMetaStore = remote;
|
3029
|
+
this.loaded = this.loader.ready().then(async () => {
|
3030
|
+
remote.load("main").then(async () => {
|
3031
|
+
(await throwFalsy(this.loader).WALStore()).process();
|
3032
|
+
});
|
3033
|
+
});
|
3034
|
+
}
|
3035
|
+
async connectStorage_X({ loader }) {
|
3036
|
+
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
3037
|
+
this.loader = loader;
|
3038
|
+
const dataUrl = this.url.build().defParam("store", "data").URI();
|
3039
|
+
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
3040
|
+
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
3041
|
+
const name = dataUrl.toString();
|
3042
|
+
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
3043
|
+
gateway: gateway.gateway,
|
3044
|
+
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
3045
|
+
});
|
3046
|
+
loader.remoteFileStore = loader.remoteCarStore;
|
3047
|
+
}
|
3048
|
+
async createEventBlock(bytes) {
|
3049
|
+
const data = {
|
3050
|
+
dbMeta: bytes
|
3051
|
+
};
|
3052
|
+
const event = await import_clock.EventBlock.create(
|
3053
|
+
data,
|
3054
|
+
this.parents
|
3055
|
+
);
|
3056
|
+
await this.eventBlocks.put(event.cid, event.bytes);
|
3057
|
+
return event;
|
3058
|
+
}
|
3059
|
+
async decodeEventBlock(bytes) {
|
3060
|
+
const event = await (0, import_clock.decodeEventBlock)(bytes);
|
3061
|
+
return event;
|
3062
|
+
}
|
3063
|
+
// move this stuff to connect
|
3064
|
+
// async getDashboardURL(compact = true) {
|
3065
|
+
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
3066
|
+
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
3067
|
+
// // if (compact) {
|
3068
|
+
// // await this.compact()
|
3069
|
+
// // }
|
3070
|
+
// const currents = await this.loader?.metaStore?.load()
|
3071
|
+
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
3072
|
+
// if (currents.length > 1)
|
3073
|
+
// throw new Error("Can't sync database with split heads: make an update first")
|
3074
|
+
// const current = currents[0]
|
3075
|
+
// const params = {
|
3076
|
+
// car: current.car.toString()
|
3077
|
+
// }
|
3078
|
+
// if (current.key) {
|
3079
|
+
// // @ts-ignore
|
3080
|
+
// params.key = current.key.toString()
|
3081
|
+
// }
|
3082
|
+
// // @ts-ignore
|
3083
|
+
// if (this.name) {
|
3084
|
+
// // @ts-ignore
|
3085
|
+
// params.name = this.name
|
3086
|
+
// }
|
3087
|
+
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
3088
|
+
// console.log('Import to dashboard: ' + url.toString())
|
3089
|
+
// return url
|
3090
|
+
// }
|
3091
|
+
// openDashboard() {
|
3092
|
+
// void this.getDashboardURL().then(url => {
|
3093
|
+
// if (url) window.open(url.toString(), '_blank')
|
3094
|
+
// })
|
3095
|
+
// }
|
3096
|
+
};
|
2780
3097
|
|
2781
3098
|
// src/crdt-helpers.ts
|
2782
|
-
init_types();
|
2783
3099
|
function time(tag) {
|
2784
3100
|
}
|
2785
3101
|
function timeEnd(tag) {
|
@@ -2827,7 +3143,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
2827
3143
|
await processFiles(store, blocks, update.value, logger);
|
2828
3144
|
value = { doc: update.value };
|
2829
3145
|
}
|
2830
|
-
const block = await
|
3146
|
+
const block = await encode({ value, hasher: import_sha25.sha256, codec });
|
2831
3147
|
blocks.putSync(block.cid, block.bytes);
|
2832
3148
|
return block.cid;
|
2833
3149
|
}
|
@@ -2836,10 +3152,16 @@ async function processFiles(store, blocks, doc, logger) {
|
|
2836
3152
|
await processFileset(logger, store, blocks, doc._files);
|
2837
3153
|
}
|
2838
3154
|
if (doc._publicFiles) {
|
2839
|
-
await processFileset(
|
3155
|
+
await processFileset(
|
3156
|
+
logger,
|
3157
|
+
store,
|
3158
|
+
blocks,
|
3159
|
+
doc._publicFiles
|
3160
|
+
/*, true*/
|
3161
|
+
);
|
2840
3162
|
}
|
2841
3163
|
}
|
2842
|
-
async function processFileset(logger, store, blocks, files
|
3164
|
+
async function processFileset(logger, store, blocks, files) {
|
2843
3165
|
const dbBlockstore = blocks.parent;
|
2844
3166
|
if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
|
2845
3167
|
const t = new CarTransaction(dbBlockstore);
|
@@ -2861,9 +3183,10 @@ async function processFileset(logger, store, blocks, files, publicFiles = false)
|
|
2861
3183
|
}
|
2862
3184
|
}
|
2863
3185
|
if (didPut.length) {
|
2864
|
-
const car = await dbBlockstore.loader.commitFiles(
|
2865
|
-
|
2866
|
-
|
3186
|
+
const car = await dbBlockstore.loader.commitFiles(
|
3187
|
+
t,
|
3188
|
+
{ files }
|
3189
|
+
);
|
2867
3190
|
if (car) {
|
2868
3191
|
for (const name of didPut) {
|
2869
3192
|
files[name] = { car, ...files[name] };
|
@@ -2897,7 +3220,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2897
3220
|
fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
|
2898
3221
|
{
|
2899
3222
|
get: async (cid) => {
|
2900
|
-
return await blocks.getFile(throwFalsy(fileMeta.car), cid
|
3223
|
+
return await blocks.getFile(throwFalsy(fileMeta.car), cid);
|
2901
3224
|
}
|
2902
3225
|
},
|
2903
3226
|
fileMeta.cid,
|
@@ -2911,7 +3234,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2911
3234
|
async function getValueFromLink(blocks, link, logger) {
|
2912
3235
|
const block = await blocks.get(link);
|
2913
3236
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
2914
|
-
const { value } = await
|
3237
|
+
const { value } = await decode({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
|
2915
3238
|
const cvalue = {
|
2916
3239
|
...value,
|
2917
3240
|
cid: link
|
@@ -2920,17 +3243,21 @@ async function getValueFromLink(blocks, link, logger) {
|
|
2920
3243
|
return cvalue;
|
2921
3244
|
}
|
2922
3245
|
var DirtyEventFetcher = class extends import_clock2.EventFetcher {
|
3246
|
+
constructor(logger, blocks) {
|
3247
|
+
super(blocks);
|
3248
|
+
this.logger = logger;
|
3249
|
+
}
|
2923
3250
|
async get(link) {
|
2924
3251
|
try {
|
2925
3252
|
return super.get(link);
|
2926
3253
|
} catch (e) {
|
2927
|
-
|
3254
|
+
this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
|
2928
3255
|
return { value: void 0 };
|
2929
3256
|
}
|
2930
3257
|
}
|
2931
3258
|
};
|
2932
3259
|
async function clockChangesSince(blocks, head, since, opts, logger) {
|
2933
|
-
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new import_clock2.EventFetcher(blocks);
|
3260
|
+
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new import_clock2.EventFetcher(blocks);
|
2934
3261
|
const keys = /* @__PURE__ */ new Set();
|
2935
3262
|
const updates = await gatherUpdates(
|
2936
3263
|
blocks,
|
@@ -3028,38 +3355,34 @@ async function doCompact(blockLog, head, logger) {
|
|
3028
3355
|
async function getBlock(blocks, cidString) {
|
3029
3356
|
const block = await blocks.get((0, import_link.parse)(cidString));
|
3030
3357
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
3031
|
-
const { cid, value } = await
|
3032
|
-
return new
|
3358
|
+
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
|
3359
|
+
return new Block({ cid, value, bytes: block.bytes });
|
3033
3360
|
}
|
3034
3361
|
|
3035
|
-
// src/indexer.ts
|
3036
|
-
init_types();
|
3037
|
-
|
3038
3362
|
// src/indexer-helpers.ts
|
3039
|
-
var
|
3040
|
-
var
|
3041
|
-
var codec3 = __toESM(require("@ipld/dag-cbor"), 1);
|
3363
|
+
var import_sha26 = require("multiformats/hashes/sha2");
|
3364
|
+
var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
3042
3365
|
var import_charwise = __toESM(require("charwise"), 1);
|
3043
3366
|
var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
|
3044
|
-
var
|
3045
|
-
var
|
3367
|
+
var import_utils15 = require("prolly-trees/utils");
|
3368
|
+
var import_cache = require("prolly-trees/cache");
|
3046
3369
|
var IndexTree = class {
|
3047
3370
|
};
|
3048
3371
|
function refCompare(aRef, bRef) {
|
3049
3372
|
if (Number.isNaN(aRef)) return -1;
|
3050
3373
|
if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
|
3051
3374
|
if (aRef === Infinity) return 1;
|
3052
|
-
return (0,
|
3375
|
+
return (0, import_utils15.simpleCompare)(aRef, bRef);
|
3053
3376
|
}
|
3054
3377
|
function compare(a, b) {
|
3055
3378
|
const [aKey, aRef] = a;
|
3056
3379
|
const [bKey, bRef] = b;
|
3057
|
-
const comp = (0,
|
3380
|
+
const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
|
3058
3381
|
if (comp !== 0) return comp;
|
3059
3382
|
return refCompare(aRef, bRef);
|
3060
3383
|
}
|
3061
|
-
var byKeyOpts = { cache:
|
3062
|
-
var byIdOpts = { cache:
|
3384
|
+
var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
|
3385
|
+
var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
|
3063
3386
|
function indexEntriesForChanges(changes, mapFn) {
|
3064
3387
|
const indexEntries = [];
|
3065
3388
|
changes.forEach(({ id: key, value, del }) => {
|
@@ -3087,7 +3410,7 @@ function makeProllyGetBlock(blocks) {
|
|
3087
3410
|
const block = await blocks.get(address);
|
3088
3411
|
if (!block) throw new Error(`Missing block ${address.toString()}`);
|
3089
3412
|
const { cid, bytes } = block;
|
3090
|
-
return
|
3413
|
+
return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
|
3091
3414
|
};
|
3092
3415
|
}
|
3093
3416
|
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
@@ -3159,25 +3482,25 @@ function encodeKey(key) {
|
|
3159
3482
|
|
3160
3483
|
// src/indexer.ts
|
3161
3484
|
init_utils();
|
3162
|
-
function index({ _crdt }, name, mapFn, meta) {
|
3485
|
+
function index(sthis, { _crdt }, name, mapFn, meta) {
|
3163
3486
|
if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
3164
3487
|
if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
3165
3488
|
if (_crdt.indexers.has(name)) {
|
3166
3489
|
const idx = _crdt.indexers.get(name);
|
3167
3490
|
idx.applyMapFn(name, mapFn, meta);
|
3168
3491
|
} else {
|
3169
|
-
const idx = new Index(_crdt, name, mapFn, meta);
|
3492
|
+
const idx = new Index(sthis, _crdt, name, mapFn, meta);
|
3170
3493
|
_crdt.indexers.set(name, idx);
|
3171
3494
|
}
|
3172
3495
|
return _crdt.indexers.get(name);
|
3173
3496
|
}
|
3174
3497
|
var Index = class {
|
3175
|
-
constructor(crdt, name, mapFn, meta) {
|
3498
|
+
constructor(sthis, crdt, name, mapFn, meta) {
|
3176
3499
|
this.mapFnString = "";
|
3177
3500
|
this.byKey = new IndexTree();
|
3178
3501
|
this.byId = new IndexTree();
|
3179
3502
|
this.includeDocsDefault = false;
|
3180
|
-
this.logger = ensureLogger(
|
3503
|
+
this.logger = ensureLogger(sthis, "Index");
|
3181
3504
|
this.blockstore = crdt.indexBlockstore;
|
3182
3505
|
this.crdt = crdt;
|
3183
3506
|
this.applyMapFn(name, mapFn, meta);
|
@@ -3232,7 +3555,7 @@ var Index = class {
|
|
3232
3555
|
}
|
3233
3556
|
if (this.mapFnString) {
|
3234
3557
|
if (this.mapFnString !== mapFn.toString()) {
|
3235
|
-
throw this.logger.Error().Msg("cannot apply different mapFn app").AsError();
|
3558
|
+
throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
|
3236
3559
|
}
|
3237
3560
|
} else {
|
3238
3561
|
this.mapFnString = mapFn.toString();
|
@@ -3367,8 +3690,7 @@ var Index = class {
|
|
3367
3690
|
// src/crdt-clock.ts
|
3368
3691
|
var import_clock3 = require("@web3-storage/pail/clock");
|
3369
3692
|
var import_crdt2 = require("@web3-storage/pail/crdt");
|
3370
|
-
var
|
3371
|
-
init_types();
|
3693
|
+
var import_cement11 = require("@adviser/cement");
|
3372
3694
|
|
3373
3695
|
// src/apply-head-queue.ts
|
3374
3696
|
function applyHeadQueue(worker, logger) {
|
@@ -3426,9 +3748,9 @@ var CRDTClock = class {
|
|
3426
3748
|
this.zoomers = /* @__PURE__ */ new Set();
|
3427
3749
|
this.watchers = /* @__PURE__ */ new Set();
|
3428
3750
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
3429
|
-
this._ready = new
|
3751
|
+
this._ready = new import_cement11.ResolveOnce();
|
3430
3752
|
this.blockstore = blockstore;
|
3431
|
-
this.logger = ensureLogger(blockstore.
|
3753
|
+
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
3432
3754
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
3433
3755
|
}
|
3434
3756
|
async ready() {
|
@@ -3501,7 +3823,7 @@ var CRDTClock = class {
|
|
3501
3823
|
}
|
3502
3824
|
return { head: advancedHead };
|
3503
3825
|
},
|
3504
|
-
{ noLoader }
|
3826
|
+
{ noLoader, add: false }
|
3505
3827
|
);
|
3506
3828
|
this.setHead(meta.head);
|
3507
3829
|
}
|
@@ -3536,13 +3858,14 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3536
3858
|
// src/crdt.ts
|
3537
3859
|
init_utils();
|
3538
3860
|
var CRDT = class {
|
3539
|
-
constructor(name, opts = {}) {
|
3540
|
-
this.onceReady = new import_cement10.ResolveOnce();
|
3861
|
+
constructor(sthis, name, opts = {}) {
|
3541
3862
|
this.indexers = /* @__PURE__ */ new Map();
|
3863
|
+
this.onceReady = new import_cement12.ResolveOnce();
|
3864
|
+
this.sthis = sthis;
|
3542
3865
|
this.name = name;
|
3543
|
-
this.logger = ensureLogger(
|
3866
|
+
this.logger = ensureLogger(sthis, "CRDT");
|
3544
3867
|
this.opts = opts;
|
3545
|
-
this.blockstore = blockstoreFactory({
|
3868
|
+
this.blockstore = blockstoreFactory(sthis, {
|
3546
3869
|
name,
|
3547
3870
|
applyMeta: async (meta) => {
|
3548
3871
|
const crdtMeta = meta;
|
@@ -3554,22 +3877,20 @@ var CRDT = class {
|
|
3554
3877
|
return { head: this.clock.head };
|
3555
3878
|
},
|
3556
3879
|
autoCompact: this.opts.autoCompact || 100,
|
3557
|
-
crypto: this.opts.crypto,
|
3558
3880
|
store: { ...this.opts.store, isIndex: void 0 },
|
3559
3881
|
public: this.opts.public,
|
3560
3882
|
meta: this.opts.meta,
|
3561
3883
|
threshold: this.opts.threshold
|
3562
3884
|
});
|
3563
|
-
this.indexBlockstore = blockstoreFactory({
|
3885
|
+
this.indexBlockstore = blockstoreFactory(sthis, {
|
3564
3886
|
name,
|
3565
3887
|
applyMeta: async (meta) => {
|
3566
3888
|
const idxCarMeta = meta;
|
3567
3889
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
3568
3890
|
for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
|
3569
|
-
index({ _crdt: this }, name2, void 0, idx);
|
3891
|
+
index(this.sthis, { _crdt: this }, name2, void 0, idx);
|
3570
3892
|
}
|
3571
3893
|
},
|
3572
|
-
crypto: this.opts.crypto,
|
3573
3894
|
store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
|
3574
3895
|
public: this.opts.public
|
3575
3896
|
});
|
@@ -3580,17 +3901,6 @@ var CRDT = class {
|
|
3580
3901
|
}
|
3581
3902
|
});
|
3582
3903
|
}
|
3583
|
-
async ready() {
|
3584
|
-
return this.onceReady.once(async () => {
|
3585
|
-
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3586
|
-
});
|
3587
|
-
}
|
3588
|
-
async close() {
|
3589
|
-
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
3590
|
-
}
|
3591
|
-
async destroy() {
|
3592
|
-
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
3593
|
-
}
|
3594
3904
|
async bulk(updates) {
|
3595
3905
|
await this.ready();
|
3596
3906
|
const prevHead = [...this.clock.head];
|
@@ -3611,6 +3921,21 @@ var CRDT = class {
|
|
3611
3921
|
await this.clock.applyHead(done.meta.head, prevHead, updates);
|
3612
3922
|
return done.meta;
|
3613
3923
|
}
|
3924
|
+
async ready() {
|
3925
|
+
return this.onceReady.once(async () => {
|
3926
|
+
try {
|
3927
|
+
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3928
|
+
} catch (e) {
|
3929
|
+
throw this.logger.Error().Err(e).Msg("CRDT not ready").AsError();
|
3930
|
+
}
|
3931
|
+
});
|
3932
|
+
}
|
3933
|
+
async close() {
|
3934
|
+
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
3935
|
+
}
|
3936
|
+
async destroy() {
|
3937
|
+
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
3938
|
+
}
|
3614
3939
|
// if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
|
3615
3940
|
async allDocs() {
|
3616
3941
|
await this.ready();
|
@@ -3649,20 +3974,19 @@ var CRDT = class {
|
|
3649
3974
|
};
|
3650
3975
|
|
3651
3976
|
// src/database.ts
|
3652
|
-
init_sys_container();
|
3653
3977
|
init_utils();
|
3654
|
-
init_gateway();
|
3655
3978
|
var Database = class {
|
3656
3979
|
constructor(name, opts) {
|
3657
3980
|
this.opts = {};
|
3658
3981
|
this._listening = false;
|
3659
3982
|
this._listeners = /* @__PURE__ */ new Set();
|
3660
3983
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3661
|
-
this._ready = new
|
3984
|
+
this._ready = new import_cement13.ResolveOnce();
|
3662
3985
|
this.name = name;
|
3663
3986
|
this.opts = opts || this.opts;
|
3664
|
-
this.
|
3665
|
-
this.
|
3987
|
+
this.sthis = ensureSuperThis(this.opts);
|
3988
|
+
this.logger = ensureLogger(this.sthis, "Database");
|
3989
|
+
this._crdt = new CRDT(this.sthis, name, this.opts);
|
3666
3990
|
this.blockstore = this._crdt.blockstore;
|
3667
3991
|
this._writeQueue = writeQueue(async (updates) => {
|
3668
3992
|
return await this._crdt.bulk(updates);
|
@@ -3686,7 +4010,7 @@ var Database = class {
|
|
3686
4010
|
}
|
3687
4011
|
async ready() {
|
3688
4012
|
return this._ready.once(async () => {
|
3689
|
-
await
|
4013
|
+
await this.sthis.start();
|
3690
4014
|
await this._crdt.ready();
|
3691
4015
|
await this.blockstore.ready();
|
3692
4016
|
});
|
@@ -3706,7 +4030,7 @@ var Database = class {
|
|
3706
4030
|
await this.ready();
|
3707
4031
|
this.logger.Debug().Str("id", doc._id).Msg("put");
|
3708
4032
|
const { _id, ...value } = doc;
|
3709
|
-
const docId = _id ||
|
4033
|
+
const docId = _id || this.sthis.nextId().str;
|
3710
4034
|
const result = await this._writeQueue.push({
|
3711
4035
|
id: docId,
|
3712
4036
|
value: {
|
@@ -3771,7 +4095,7 @@ var Database = class {
|
|
3771
4095
|
await this.ready();
|
3772
4096
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
3773
4097
|
const _crdt = this._crdt;
|
3774
|
-
const idx = typeof field === "string" ? index({ _crdt }, field) : index({ _crdt }, makeName(field.toString()), field);
|
4098
|
+
const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
|
3775
4099
|
return await idx.query(opts);
|
3776
4100
|
}
|
3777
4101
|
async compact() {
|
@@ -3808,12 +4132,7 @@ function fireproof(name, opts) {
|
|
3808
4132
|
const key = JSON.stringify(
|
3809
4133
|
toSortedArray({
|
3810
4134
|
name,
|
3811
|
-
stores: toSortedArray(opts?.store?.stores)
|
3812
|
-
makeMetaStore: !!opts?.store?.makeMetaStore,
|
3813
|
-
makeDataStore: !!opts?.store?.makeDataStore,
|
3814
|
-
makeRemoteWAL: !!opts?.store?.makeRemoteWAL,
|
3815
|
-
encodeFile: !!opts?.store?.encodeFile,
|
3816
|
-
decodeFile: !!opts?.store?.decodeFile
|
4135
|
+
stores: toSortedArray(opts?.store?.stores)
|
3817
4136
|
})
|
3818
4137
|
);
|
3819
4138
|
let db = Database.databases.get(key);
|
@@ -3828,7 +4147,10 @@ function makeName(fnString) {
|
|
3828
4147
|
let found = null;
|
3829
4148
|
const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
|
3830
4149
|
if (matches.length === 0) {
|
3831
|
-
found = /=>\s*(
|
4150
|
+
found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
|
4151
|
+
if (found && found[1].includes("return")) {
|
4152
|
+
found = null;
|
4153
|
+
}
|
3832
4154
|
}
|
3833
4155
|
if (!found) {
|
3834
4156
|
return fnString;
|
@@ -3837,14 +4159,41 @@ function makeName(fnString) {
|
|
3837
4159
|
}
|
3838
4160
|
}
|
3839
4161
|
|
4162
|
+
// src/runtime/index.ts
|
4163
|
+
var runtime_exports = {};
|
4164
|
+
__export(runtime_exports, {
|
4165
|
+
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
4166
|
+
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
4167
|
+
files: () => files_exports,
|
4168
|
+
getFileName: () => getFileName,
|
4169
|
+
getFileSystem: () => getFileSystem,
|
4170
|
+
getPath: () => getPath,
|
4171
|
+
kb: () => key_bag_exports,
|
4172
|
+
kc: () => keyed_crypto_exports,
|
4173
|
+
mf: () => wait_pr_multiformats_exports,
|
4174
|
+
toArrayBuffer: () => toArrayBuffer
|
4175
|
+
});
|
4176
|
+
init_utils2();
|
4177
|
+
|
4178
|
+
// src/runtime/wait-pr-multiformats/index.ts
|
4179
|
+
var wait_pr_multiformats_exports = {};
|
4180
|
+
__export(wait_pr_multiformats_exports, {
|
4181
|
+
block: () => block_exports,
|
4182
|
+
codec: () => codec_interface_exports
|
4183
|
+
});
|
4184
|
+
|
4185
|
+
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
4186
|
+
var codec_interface_exports = {};
|
4187
|
+
|
4188
|
+
// src/runtime/index.ts
|
4189
|
+
init_version();
|
4190
|
+
init_version2();
|
4191
|
+
|
3840
4192
|
// src/index.ts
|
3841
|
-
init_types();
|
3842
|
-
init_runtime();
|
3843
|
-
init_runtime();
|
3844
4193
|
init_utils();
|
3845
4194
|
|
3846
4195
|
// src/version.ts
|
3847
4196
|
var PACKAGE_VERSION = Object.keys({
|
3848
|
-
"0.19.
|
4197
|
+
"0.19.9-dev-frag": "xxxx"
|
3849
4198
|
})[0];
|
3850
4199
|
//# sourceMappingURL=index.cjs.map
|