@fireproof/core 0.19.8-dev-global → 0.19.9-dev-frag
Sign up to get free protection for your applications and to get access to all the features.
- package/README.md +7 -0
- package/chunk-7EWIAXTM.js +7 -0
- package/chunk-7EWIAXTM.js.map +1 -0
- package/chunk-JO5AVWG7.js +67 -0
- package/chunk-JO5AVWG7.js.map +1 -0
- package/chunk-PB4BKL4O.js +7 -0
- package/chunk-PB4BKL4O.js.map +1 -0
- package/chunk-YS4GL6OK.js +266 -0
- package/chunk-YS4GL6OK.js.map +1 -0
- package/{store-indexdb-WLRSICCB.js → gateway-IZRHJWPE.js} +48 -80
- package/gateway-IZRHJWPE.js.map +1 -0
- package/gateway-YSNUK2L3.js +145 -0
- package/gateway-YSNUK2L3.js.map +1 -0
- package/index.cjs +2132 -1783
- package/index.cjs.map +1 -1
- package/index.d.cts +613 -513
- package/index.d.ts +613 -513
- package/index.global.js +19366 -20107
- package/index.global.js.map +1 -1
- package/index.js +1512 -1022
- package/index.js.map +1 -1
- package/key-bag-file-NMEBFSPM.js +54 -0
- package/key-bag-file-NMEBFSPM.js.map +1 -0
- package/key-bag-indexdb-X5V6GNBZ.js +50 -0
- package/key-bag-indexdb-X5V6GNBZ.js.map +1 -0
- package/mem-filesystem-B6C6QOIP.js +41 -0
- package/mem-filesystem-B6C6QOIP.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/node-filesystem-5JLBSHKQ.js +41 -0
- package/node-filesystem-5JLBSHKQ.js.map +1 -0
- package/package.json +8 -7
- package/tests/blockstore/fragment-gateway.test.ts +107 -0
- package/tests/blockstore/keyed-crypto.test.ts +302 -0
- package/tests/blockstore/loader.test.ts +24 -19
- package/tests/blockstore/store.test.ts +34 -28
- package/tests/blockstore/transaction.test.ts +19 -15
- package/tests/fireproof/config.test.ts +94 -78
- package/tests/fireproof/crdt.test.ts +34 -28
- package/tests/fireproof/database.test.ts +22 -14
- package/tests/fireproof/fireproof.test.fixture.ts +133 -0
- package/tests/fireproof/fireproof.test.ts +331 -219
- package/tests/fireproof/hello.test.ts +6 -4
- package/tests/fireproof/indexer.test.ts +34 -27
- package/tests/fireproof/utils.test.ts +65 -0
- package/tests/helpers.ts +25 -57
- package/utils-IZPK4QS7.js +14 -0
- package/utils-IZPK4QS7.js.map +1 -0
- package/chunk-BNL4PVBF.js +0 -314
- package/chunk-BNL4PVBF.js.map +0 -1
- package/chunk-JW2QT6BF.js +0 -184
- package/chunk-JW2QT6BF.js.map +0 -1
- package/node-sys-container-MIEX6ELJ.js +0 -29
- package/node-sys-container-MIEX6ELJ.js.map +0 -1
- package/store-file-VJ6BI4II.js +0 -191
- package/store-file-VJ6BI4II.js.map +0 -1
- package/store-indexdb-WLRSICCB.js.map +0 -1
package/index.cjs
CHANGED
@@ -30,452 +30,50 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
30
30
|
));
|
31
31
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
32
32
|
|
33
|
-
// src/
|
34
|
-
function
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
return value;
|
48
|
-
}
|
49
|
-
var init_types = __esm({
|
50
|
-
"src/types.ts"() {
|
51
|
-
"use strict";
|
52
|
-
}
|
53
|
-
});
|
54
|
-
|
55
|
-
// src/runtime/node-sys-container.ts
|
56
|
-
var node_sys_container_exports = {};
|
57
|
-
__export(node_sys_container_exports, {
|
58
|
-
createNodeSysContainer: () => createNodeSysContainer
|
59
|
-
});
|
60
|
-
async function createNodeSysContainer() {
|
61
|
-
return {
|
62
|
-
state: "node",
|
63
|
-
...path,
|
64
|
-
// ...(await import("node:os")),
|
65
|
-
// ...(await import("node:url")),
|
66
|
-
...os,
|
67
|
-
...url,
|
68
|
-
...fs,
|
69
|
-
join,
|
70
|
-
stat: fs.stat,
|
71
|
-
readdir: fs.readdir,
|
72
|
-
readfile: fs.readFile,
|
73
|
-
writefile: fs.writeFile
|
74
|
-
};
|
75
|
-
}
|
76
|
-
var fs, path, os, url;
|
77
|
-
var init_node_sys_container = __esm({
|
78
|
-
"src/runtime/node-sys-container.ts"() {
|
79
|
-
"use strict";
|
80
|
-
init_sys_container();
|
81
|
-
fs = __toESM(require("fs/promises"), 1);
|
82
|
-
path = __toESM(require("path"), 1);
|
83
|
-
os = __toESM(require("os"), 1);
|
84
|
-
url = __toESM(require("url"), 1);
|
85
|
-
}
|
86
|
-
});
|
87
|
-
|
88
|
-
// src/runtime/sys-container.ts
|
89
|
-
function isSet(value, ref = globalThis) {
|
90
|
-
const [head, ...tail] = value.split(".");
|
91
|
-
if (["object", "function"].includes(typeof ref) && ref && ["object", "function"].includes(typeof ref[head]) && ref[head]) {
|
92
|
-
if (tail.length <= 1) {
|
93
|
-
return true;
|
94
|
-
}
|
95
|
-
return isSet(tail.join("."), ref[head]);
|
96
|
-
}
|
97
|
-
return false;
|
98
|
-
}
|
99
|
-
function runtimeFn() {
|
100
|
-
const isNodeIsh = isSet("process.versions.node");
|
101
|
-
const isDeno = isSet("Deno");
|
102
|
-
return {
|
103
|
-
isNodeIsh,
|
104
|
-
isBrowser: !(isNodeIsh || isDeno),
|
105
|
-
isDeno,
|
106
|
-
isReactNative: false
|
107
|
-
};
|
108
|
-
}
|
109
|
-
function join(...paths) {
|
110
|
-
return paths.map((i) => i.replace(/\/+$/, "")).join("/");
|
111
|
-
}
|
112
|
-
var import_uuidv7, import_cement, onceStart, envImpl, sysContainer, SysContainer;
|
113
|
-
var init_sys_container = __esm({
|
114
|
-
"src/runtime/sys-container.ts"() {
|
115
|
-
"use strict";
|
116
|
-
import_uuidv7 = require("uuidv7");
|
117
|
-
import_cement = require("@adviser/cement");
|
118
|
-
init_types();
|
119
|
-
onceStart = new import_cement.ResolveOnce();
|
120
|
-
envImpl = new import_cement.EnvImpl({
|
121
|
-
symbol: "FP_ENV",
|
122
|
-
presetEnv: /* @__PURE__ */ new Map([
|
123
|
-
// ["FP_DEBUG", "xxx"],
|
124
|
-
// ["FP_ENV", "development"],
|
125
|
-
])
|
126
|
-
});
|
127
|
-
sysContainer = class {
|
128
|
-
constructor() {
|
129
|
-
this.freight = {
|
130
|
-
state: "seeded",
|
131
|
-
join,
|
132
|
-
dirname: (path2) => path2.split("/").slice(0, -1).join("/"),
|
133
|
-
homedir: () => {
|
134
|
-
throw new Error("SysContainer:homedir is not available in seeded state");
|
135
|
-
},
|
136
|
-
fileURLToPath: (strurl) => {
|
137
|
-
let url2;
|
138
|
-
if (typeof strurl === "string") {
|
139
|
-
url2 = new URL(strurl);
|
140
|
-
} else {
|
141
|
-
url2 = strurl;
|
142
|
-
}
|
143
|
-
return url2.pathname;
|
144
|
-
},
|
145
|
-
// assert: (condition: unknown, message?: string | Error) => {
|
146
|
-
// if (!condition) {
|
147
|
-
// if (message instanceof Error) {
|
148
|
-
// throw message;
|
149
|
-
// } else {
|
150
|
-
// throw new Error(message);
|
151
|
-
// }
|
152
|
-
// }
|
153
|
-
// },
|
154
|
-
mkdir: () => Promise.reject(new Error("SysContainer:mkdir is not available in seeded state")),
|
155
|
-
readdir: () => Promise.reject(new Error("SysContainer:readdir is not available in seeded state")),
|
156
|
-
rm: () => Promise.reject(new Error("SysContainer:rm is not available in seeded state")),
|
157
|
-
copyFile: () => Promise.reject(new Error("SysContainer:copyFile is not available in seeded state")),
|
158
|
-
readfile: () => Promise.reject(new Error("SysContainer:readfile is not available in seeded state")),
|
159
|
-
unlink: () => Promise.reject(new Error("SysContainer:unlink is not available in seeded state")),
|
160
|
-
writefile: () => Promise.reject(new Error("SysContainer:writefile is not available in seeded state")),
|
161
|
-
stat: () => Promise.reject(new Error("SysContainer:stat is not available in seeded state"))
|
162
|
-
};
|
163
|
-
this.id = (0, import_uuidv7.uuidv4)();
|
164
|
-
this.homedir = () => {
|
165
|
-
this.logSeeded("homedir");
|
166
|
-
return throwFalsy(this.freight).homedir();
|
167
|
-
};
|
168
|
-
this.runtime = runtimeFn;
|
169
|
-
this.env = envImpl;
|
170
|
-
}
|
171
|
-
async start() {
|
172
|
-
await onceStart.once(async () => {
|
173
|
-
switch (this.freight.state) {
|
174
|
-
case "seeded":
|
175
|
-
if (this.runtime().isNodeIsh) {
|
176
|
-
const { createNodeSysContainer: createNodeSysContainer2 } = await Promise.resolve().then(() => (init_node_sys_container(), node_sys_container_exports));
|
177
|
-
this.freight = await createNodeSysContainer2();
|
178
|
-
} else {
|
179
|
-
this.freight.state = "browser";
|
180
|
-
}
|
181
|
-
return;
|
182
|
-
case "browser":
|
183
|
-
case "node":
|
184
|
-
return;
|
185
|
-
}
|
186
|
-
});
|
187
|
-
}
|
188
|
-
async readdir(path2, options) {
|
189
|
-
this.logSeeded("readdir");
|
190
|
-
return throwFalsy(this.freight).readdir(path2, options) || [];
|
191
|
-
}
|
192
|
-
async readdirent(path2, options) {
|
193
|
-
this.logSeeded("readdirent");
|
194
|
-
return throwFalsy(this.freight).readdir(path2, { ...options, withFileTypes: true }) || [];
|
195
|
-
}
|
196
|
-
async readfile(path2, options) {
|
197
|
-
this.logSeeded("readfile");
|
198
|
-
return throwFalsy(this.freight).readfile(path2, options);
|
199
|
-
}
|
200
|
-
async mkdir(path2, options) {
|
201
|
-
this.logSeeded("mkdir");
|
202
|
-
return throwFalsy(this.freight).mkdir(path2, options);
|
203
|
-
}
|
204
|
-
async rm(path2, options) {
|
205
|
-
this.logSeeded("rm");
|
206
|
-
return throwFalsy(this.freight).rm(path2, options);
|
207
|
-
}
|
208
|
-
async unlink(path2) {
|
209
|
-
this.logSeeded("unlink");
|
210
|
-
return throwFalsy(this.freight).unlink(path2);
|
211
|
-
}
|
212
|
-
async writefile(path2, data) {
|
213
|
-
this.logSeeded("writefile");
|
214
|
-
return throwFalsy(this.freight).writefile(path2, data);
|
215
|
-
}
|
216
|
-
async copyFile(source, destination) {
|
217
|
-
this.logSeeded("copyFile");
|
218
|
-
return throwFalsy(this.freight).copyFile(source, destination);
|
219
|
-
}
|
220
|
-
async stat(path2) {
|
221
|
-
this.logSeeded("stat");
|
222
|
-
return throwFalsy(this.freight).stat(path2);
|
223
|
-
}
|
224
|
-
fileURLToPath(url2) {
|
225
|
-
this.logSeeded("fileURLToPath");
|
226
|
-
return throwFalsy(this.freight).fileURLToPath(url2);
|
227
|
-
}
|
228
|
-
dirname(path2) {
|
229
|
-
this.logSeeded("dirname");
|
230
|
-
return throwFalsy(this.freight).dirname(path2);
|
231
|
-
}
|
232
|
-
join(...args) {
|
233
|
-
this.logSeeded("join");
|
234
|
-
return throwFalsy(this.freight).join(...args);
|
235
|
-
}
|
236
|
-
logSeeded(method) {
|
237
|
-
if (this.freight.state === "seeded") {
|
238
|
-
const err = new Error();
|
239
|
-
console.warn(`SysContainer.${method} is not available in seeded state:`, err.stack);
|
240
|
-
}
|
241
|
-
}
|
242
|
-
};
|
243
|
-
SysContainer = new sysContainer();
|
244
|
-
}
|
245
|
-
});
|
246
|
-
|
247
|
-
// src/runtime/data-dir.ts
|
248
|
-
function dataDir(name, base) {
|
249
|
-
if (!base) {
|
250
|
-
if (SysContainer.runtime().isNodeIsh || SysContainer.runtime().isDeno) {
|
251
|
-
base = SysContainer.env.get("FP_STORAGE_URL") || `file://${SysContainer.join(SysContainer.homedir(), ".fireproof")}`;
|
252
|
-
} else {
|
253
|
-
base = `indexdb://fp`;
|
254
|
-
}
|
255
|
-
}
|
256
|
-
let url2;
|
257
|
-
if (typeof base === "string") {
|
258
|
-
try {
|
259
|
-
url2 = new URL(base.toString());
|
260
|
-
} catch (e) {
|
261
|
-
try {
|
262
|
-
base = `file://${base}`;
|
263
|
-
url2 = new URL(base);
|
264
|
-
} catch (e2) {
|
265
|
-
throw new Error(`invalid base url: ${base}`);
|
266
|
-
}
|
267
|
-
}
|
268
|
-
} else {
|
269
|
-
url2 = base;
|
270
|
-
}
|
271
|
-
url2.searchParams.set("name", name || "");
|
272
|
-
return url2.toString();
|
273
|
-
}
|
274
|
-
var init_data_dir = __esm({
|
275
|
-
"src/runtime/data-dir.ts"() {
|
276
|
-
"use strict";
|
277
|
-
init_sys_container();
|
278
|
-
}
|
279
|
-
});
|
280
|
-
|
281
|
-
// src/runtime/store-file-utils.ts
|
282
|
-
async function getPath(url2, logger) {
|
283
|
-
const basePath = url2.toString().replace(new RegExp(`^${url2.protocol}//`), "").replace(/\?.*$/, "");
|
284
|
-
const name = url2.searchParams.get("name");
|
285
|
-
if (name) {
|
286
|
-
const version = url2.searchParams.get("version");
|
287
|
-
if (!version) throw logger.Error().Str("url", url2.toString()).Msg(`version not found`).AsError();
|
288
|
-
return SysContainer.join(basePath, version, name);
|
289
|
-
}
|
290
|
-
return SysContainer.join(basePath);
|
291
|
-
}
|
292
|
-
function getFileName(url2, key, logger) {
|
293
|
-
switch (getStore(url2, logger, (...a) => a.join("/"))) {
|
294
|
-
case "data":
|
295
|
-
return key + ".car";
|
296
|
-
case "meta":
|
297
|
-
return key + ".json";
|
298
|
-
default:
|
299
|
-
throw logger.Error().Str("url", url2.toString()).Msg(`unsupported store type`).AsError();
|
300
|
-
}
|
301
|
-
}
|
302
|
-
function ensureIndexName(url2, name) {
|
303
|
-
if (url2.searchParams.has("index")) {
|
304
|
-
name = (url2.searchParams.get("index")?.replace(/[^a-zA-Z0-9]/g, "") || "idx") + "-" + name;
|
305
|
-
}
|
306
|
-
return name;
|
307
|
-
}
|
308
|
-
var init_store_file_utils = __esm({
|
309
|
-
"src/runtime/store-file-utils.ts"() {
|
310
|
-
"use strict";
|
311
|
-
init_utils();
|
312
|
-
init_sys_container();
|
313
|
-
}
|
314
|
-
});
|
315
|
-
|
316
|
-
// src/runtime/crypto.ts
|
317
|
-
var crypto_exports = {};
|
318
|
-
__export(crypto_exports, {
|
319
|
-
toCryptoOpts: () => toCryptoOpts
|
320
|
-
});
|
321
|
-
function randomBytes(size) {
|
322
|
-
const bytes = new Uint8Array(size);
|
323
|
-
if (size > 0) {
|
324
|
-
crypto.getRandomValues(bytes);
|
325
|
-
}
|
326
|
-
return bytes;
|
327
|
-
}
|
328
|
-
function digestSHA256(data) {
|
329
|
-
return Promise.resolve(crypto.subtle.digest("SHA-256", data));
|
330
|
-
}
|
331
|
-
function toCryptoOpts(cryptoOpts = {}) {
|
332
|
-
const opts = {
|
333
|
-
importKey: cryptoOpts.importKey || crypto.subtle.importKey.bind(crypto.subtle),
|
334
|
-
encrypt: cryptoOpts.encrypt || crypto.subtle.encrypt.bind(crypto.subtle),
|
335
|
-
decrypt: cryptoOpts.decrypt || crypto.subtle.decrypt.bind(crypto.subtle),
|
336
|
-
randomBytes: cryptoOpts.randomBytes || randomBytes,
|
337
|
-
digestSHA256: cryptoOpts.digestSHA256 || digestSHA256
|
338
|
-
};
|
339
|
-
return opts;
|
340
|
-
}
|
341
|
-
var init_crypto = __esm({
|
342
|
-
"src/runtime/crypto.ts"() {
|
343
|
-
"use strict";
|
344
|
-
}
|
345
|
-
});
|
346
|
-
|
347
|
-
// src/runtime/files.ts
|
348
|
-
var files_exports = {};
|
349
|
-
__export(files_exports, {
|
350
|
-
decodeFile: () => decodeFile,
|
351
|
-
encodeFile: () => encodeFile
|
352
|
-
});
|
353
|
-
async function collect(collectable) {
|
354
|
-
const chunks = [];
|
355
|
-
await collectable.pipeTo(
|
356
|
-
new WritableStream({
|
357
|
-
write(chunk) {
|
358
|
-
chunks.push(chunk);
|
359
|
-
}
|
360
|
-
})
|
361
|
-
);
|
362
|
-
return chunks;
|
363
|
-
}
|
364
|
-
async function encodeFile(blob) {
|
365
|
-
const readable = createFileEncoderStream(blob);
|
366
|
-
const blocks = await collect(readable);
|
367
|
-
return { cid: blocks.at(-1).cid, blocks };
|
33
|
+
// src/utils.ts
|
34
|
+
function presetEnv() {
|
35
|
+
const penv = new Map([
|
36
|
+
// ["FP_DEBUG", "xxx"],
|
37
|
+
// ["FP_ENV", "development"],
|
38
|
+
...Array.from(
|
39
|
+
Object.entries(
|
40
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
41
|
+
globalThis[Symbol.for("FP_PRESET_ENV")] || {}
|
42
|
+
)
|
43
|
+
)
|
44
|
+
// .map(([k, v]) => [k, v as string])
|
45
|
+
]);
|
46
|
+
return penv;
|
368
47
|
}
|
369
|
-
|
370
|
-
const
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
48
|
+
function ensureSuperThis(osthis) {
|
49
|
+
const env = (0, import_cement.envFactory)({
|
50
|
+
symbol: osthis?.env?.symbol || "FP_ENV",
|
51
|
+
presetEnv: osthis?.env?.presetEnv || presetEnv()
|
52
|
+
});
|
53
|
+
return new superThis({
|
54
|
+
logger: osthis?.logger || globalLogger,
|
55
|
+
env,
|
56
|
+
crypto: osthis?.crypto || (0, import_cement.toCryptoRuntime)(),
|
57
|
+
ctx: osthis?.ctx || {},
|
58
|
+
pathOps,
|
59
|
+
txt: osthis?.txt || txtOps
|
60
|
+
});
|
376
61
|
}
|
377
|
-
function
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
void (async () => {
|
382
|
-
await fileBuilder.finalize(unixfsWriter);
|
383
|
-
await unixfsWriter.close();
|
384
|
-
})();
|
385
|
-
return readable;
|
62
|
+
function ensureSuperLog(sthis, componentName, ctx) {
|
63
|
+
return sthis.clone({
|
64
|
+
logger: ensureLogger(sthis, componentName, ctx)
|
65
|
+
});
|
386
66
|
}
|
387
|
-
|
388
|
-
var init_files = __esm({
|
389
|
-
"src/runtime/files.ts"() {
|
390
|
-
"use strict";
|
391
|
-
UnixFS = __toESM(require("@ipld/unixfs"), 1);
|
392
|
-
raw = __toESM(require("multiformats/codecs/raw"), 1);
|
393
|
-
import_fixed = require("@ipld/unixfs/file/chunker/fixed");
|
394
|
-
import_balanced = require("@ipld/unixfs/file/layout/balanced");
|
395
|
-
import_ipfs_unixfs_exporter = require("ipfs-unixfs-exporter");
|
396
|
-
queuingStrategy = UnixFS.withCapacity();
|
397
|
-
settings = UnixFS.configure({
|
398
|
-
fileChunkEncoder: raw,
|
399
|
-
smallFileEncoder: raw,
|
400
|
-
chunker: (0, import_fixed.withMaxChunkSize)(1024 * 1024),
|
401
|
-
fileLayout: (0, import_balanced.withWidth)(1024)
|
402
|
-
});
|
403
|
-
UnixFSFileBuilder = class {
|
404
|
-
#file;
|
405
|
-
constructor(name, file) {
|
406
|
-
this.name = name;
|
407
|
-
this.#file = file;
|
408
|
-
}
|
409
|
-
async finalize(writer) {
|
410
|
-
const unixfsFileWriter = UnixFS.createFileWriter(writer);
|
411
|
-
await this.#file.stream().pipeTo(
|
412
|
-
new WritableStream({
|
413
|
-
async write(chunk) {
|
414
|
-
await unixfsFileWriter.write(chunk);
|
415
|
-
}
|
416
|
-
})
|
417
|
-
);
|
418
|
-
return await unixfsFileWriter.close();
|
419
|
-
}
|
420
|
-
};
|
421
|
-
}
|
422
|
-
});
|
423
|
-
|
424
|
-
// src/runtime/store-file-version.ts
|
425
|
-
var FILESTORE_VERSION;
|
426
|
-
var init_store_file_version = __esm({
|
427
|
-
"src/runtime/store-file-version.ts"() {
|
428
|
-
"use strict";
|
429
|
-
FILESTORE_VERSION = "v0.19-file";
|
430
|
-
}
|
431
|
-
});
|
432
|
-
|
433
|
-
// src/runtime/store-indexdb-version.ts
|
434
|
-
var INDEXDB_VERSION;
|
435
|
-
var init_store_indexdb_version = __esm({
|
436
|
-
"src/runtime/store-indexdb-version.ts"() {
|
437
|
-
"use strict";
|
438
|
-
INDEXDB_VERSION = "v0.19-indexdb";
|
439
|
-
}
|
440
|
-
});
|
441
|
-
|
442
|
-
// src/runtime/index.ts
|
443
|
-
var runtime_exports = {};
|
444
|
-
__export(runtime_exports, {
|
445
|
-
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
446
|
-
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
447
|
-
SysContainer: () => SysContainer,
|
448
|
-
crypto: () => crypto_exports,
|
449
|
-
dataDir: () => dataDir,
|
450
|
-
ensureIndexName: () => ensureIndexName,
|
451
|
-
files: () => files_exports,
|
452
|
-
getFileName: () => getFileName,
|
453
|
-
getPath: () => getPath,
|
454
|
-
join: () => join
|
455
|
-
});
|
456
|
-
var init_runtime = __esm({
|
457
|
-
"src/runtime/index.ts"() {
|
458
|
-
"use strict";
|
459
|
-
init_sys_container();
|
460
|
-
init_data_dir();
|
461
|
-
init_store_file_utils();
|
462
|
-
init_crypto();
|
463
|
-
init_files();
|
464
|
-
init_store_file_version();
|
465
|
-
init_store_indexdb_version();
|
466
|
-
}
|
467
|
-
});
|
468
|
-
|
469
|
-
// src/utils.ts
|
470
|
-
function ensureLogger(optsOrLogger, componentName, ctx) {
|
67
|
+
function ensureLogger(sthis, componentName, ctx) {
|
471
68
|
let logger = globalLogger;
|
472
|
-
if ((0,
|
473
|
-
logger =
|
474
|
-
} else if (
|
475
|
-
logger =
|
69
|
+
if ((0, import_cement.IsLogger)(sthis)) {
|
70
|
+
logger = sthis;
|
71
|
+
} else if (sthis && (0, import_cement.IsLogger)(sthis.logger)) {
|
72
|
+
logger = sthis.logger;
|
476
73
|
}
|
477
74
|
const cLogger = logger.With().Module(componentName);
|
478
75
|
const debug = [];
|
76
|
+
let exposeStack = false;
|
479
77
|
if (ctx) {
|
480
78
|
if ("debug" in ctx) {
|
481
79
|
if (typeof ctx.debug === "string" && ctx.debug.length > 0) {
|
@@ -485,8 +83,12 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
485
83
|
}
|
486
84
|
delete ctx.debug;
|
487
85
|
}
|
86
|
+
if ("exposeStack" in ctx) {
|
87
|
+
exposeStack = true;
|
88
|
+
delete ctx.exposeStack;
|
89
|
+
}
|
488
90
|
if ("this" in ctx) {
|
489
|
-
cLogger.Str("this",
|
91
|
+
cLogger.Str("this", sthis.nextId(4).str);
|
490
92
|
delete ctx.this;
|
491
93
|
}
|
492
94
|
for (const [key, value] of Object.entries(ctx)) {
|
@@ -500,7 +102,7 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
500
102
|
default:
|
501
103
|
if (value instanceof Date) {
|
502
104
|
cLogger.Str(key, value.toISOString());
|
503
|
-
} else if (
|
105
|
+
} else if ((0, import_cement.isURL)(value)) {
|
504
106
|
cLogger.Str(key, value.toString());
|
505
107
|
} else if (typeof value === "function") {
|
506
108
|
cLogger.Ref(key, value);
|
@@ -512,74 +114,152 @@ function ensureLogger(optsOrLogger, componentName, ctx) {
|
|
512
114
|
}
|
513
115
|
}
|
514
116
|
registerFP_DEBUG.once(async () => {
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
117
|
+
sthis.env.onSet(
|
118
|
+
(key, value) => {
|
119
|
+
switch (key) {
|
120
|
+
case "FP_DEBUG":
|
121
|
+
logger.SetDebug(value || []);
|
122
|
+
break;
|
123
|
+
case "FP_STACK":
|
124
|
+
logger.SetExposeStack(!!value);
|
125
|
+
break;
|
126
|
+
}
|
127
|
+
},
|
128
|
+
"FP_DEBUG",
|
129
|
+
"FP_STACK"
|
130
|
+
);
|
520
131
|
}).finally(() => {
|
521
132
|
});
|
522
133
|
if (debug.length > 0) {
|
523
134
|
logger.SetDebug(debug);
|
524
135
|
}
|
136
|
+
if (exposeStack) {
|
137
|
+
logger.SetExposeStack(true);
|
138
|
+
}
|
525
139
|
const out = cLogger.Logger();
|
526
140
|
return out;
|
527
141
|
}
|
528
|
-
function getStore(
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
142
|
+
function getStore(url, sthis, joiner) {
|
143
|
+
const store = url.getParam("store");
|
144
|
+
switch (store) {
|
145
|
+
case "data":
|
146
|
+
case "wal":
|
147
|
+
case "meta":
|
148
|
+
break;
|
149
|
+
default:
|
150
|
+
throw sthis.logger.Error().Url(url).Msg(`store not found`).AsError();
|
533
151
|
}
|
534
|
-
|
152
|
+
let name = store;
|
153
|
+
if (url.hasParam("index")) {
|
154
|
+
name = joiner(url.getParam("index") || "idx", name);
|
155
|
+
}
|
156
|
+
return { store, name };
|
535
157
|
}
|
536
|
-
function getKey(
|
537
|
-
const result =
|
538
|
-
if (!result) throw logger.Error().Str("url",
|
158
|
+
function getKey(url, logger) {
|
159
|
+
const result = url.getParam("key");
|
160
|
+
if (!result) throw logger.Error().Str("url", url.toString()).Msg(`key not found`).AsError();
|
539
161
|
return result;
|
540
162
|
}
|
541
|
-
function getName(
|
542
|
-
let result =
|
163
|
+
function getName(sthis, url) {
|
164
|
+
let result = url.getParam("name");
|
543
165
|
if (!result) {
|
544
|
-
result =
|
166
|
+
result = sthis.pathOps.dirname(url.pathname);
|
545
167
|
if (result.length === 0) {
|
546
|
-
throw logger.Error().Str("url",
|
168
|
+
throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
|
547
169
|
}
|
548
170
|
}
|
549
171
|
return result;
|
550
172
|
}
|
551
173
|
function exception2Result(fn) {
|
552
|
-
return fn().then((value) =>
|
174
|
+
return fn().then((value) => import_cement.Result.Ok(value)).catch((e) => import_cement.Result.Err(e));
|
553
175
|
}
|
554
176
|
async function exceptionWrapper(fn) {
|
555
|
-
return fn().catch((e) =>
|
177
|
+
return fn().catch((e) => import_cement.Result.Err(e));
|
556
178
|
}
|
557
|
-
var import_cement2, import_uuidv72, globalLogger, registerFP_DEBUG;
|
558
|
-
var init_utils = __esm({
|
559
|
-
"src/utils.ts"() {
|
560
|
-
"use strict";
|
561
|
-
import_cement2 = require("@adviser/cement");
|
562
|
-
init_runtime();
|
563
|
-
import_uuidv72 = require("uuidv7");
|
564
|
-
globalLogger = new import_cement2.LoggerImpl();
|
565
|
-
registerFP_DEBUG = new import_cement2.ResolveOnce();
|
566
|
-
}
|
567
|
-
});
|
568
|
-
|
569
|
-
// src/blockstore/gateway.ts
|
570
179
|
function isNotFoundError(e) {
|
571
|
-
if (
|
180
|
+
if (import_cement.Result.Is(e)) {
|
572
181
|
if (e.isOk()) return false;
|
573
182
|
e = e.Err();
|
574
183
|
}
|
575
184
|
if (e.code === "ENOENT") return true;
|
576
185
|
return false;
|
577
186
|
}
|
578
|
-
|
579
|
-
|
580
|
-
|
187
|
+
function dataDir(sthis, name, base) {
|
188
|
+
if (!base) {
|
189
|
+
if (!(0, import_cement.runtimeFn)().isBrowser) {
|
190
|
+
const home = sthis.env.get("HOME") || "./";
|
191
|
+
base = sthis.env.get("FP_STORAGE_URL") || `file://${sthis.pathOps.join(home, ".fireproof")}`;
|
192
|
+
} else {
|
193
|
+
base = sthis.env.get("FP_STORAGE_URL") || `indexdb://fp`;
|
194
|
+
}
|
195
|
+
}
|
196
|
+
return import_cement.URI.from(base.toString()).build().setParam("name", name || "").URI();
|
197
|
+
}
|
198
|
+
function UInt8ArrayEqual(a, b) {
|
199
|
+
if (a.length !== b.length) {
|
200
|
+
return false;
|
201
|
+
}
|
202
|
+
for (let i = 0; i < a.length; i++) {
|
203
|
+
if (a[i] !== b[i]) {
|
204
|
+
return false;
|
205
|
+
}
|
206
|
+
}
|
207
|
+
return true;
|
208
|
+
}
|
209
|
+
var import_cement, import_base58, globalLogger, registerFP_DEBUG, superThis, pathOpsImpl, pathOps, txtOps, NotFoundError;
|
210
|
+
var init_utils = __esm({
|
211
|
+
"src/utils.ts"() {
|
581
212
|
"use strict";
|
582
|
-
|
213
|
+
import_cement = require("@adviser/cement");
|
214
|
+
import_base58 = require("multiformats/bases/base58");
|
215
|
+
globalLogger = new import_cement.LoggerImpl();
|
216
|
+
registerFP_DEBUG = new import_cement.ResolveOnce();
|
217
|
+
superThis = class _superThis {
|
218
|
+
constructor(opts) {
|
219
|
+
this.logger = opts.logger;
|
220
|
+
this.env = opts.env;
|
221
|
+
this.crypto = opts.crypto;
|
222
|
+
this.pathOps = opts.pathOps;
|
223
|
+
this.txt = opts.txt;
|
224
|
+
this.ctx = { ...opts.ctx };
|
225
|
+
}
|
226
|
+
nextId(bytes = 6) {
|
227
|
+
const bin = this.crypto.randomBytes(bytes);
|
228
|
+
return {
|
229
|
+
str: import_base58.base58btc.encode(bin),
|
230
|
+
bin
|
231
|
+
};
|
232
|
+
}
|
233
|
+
start() {
|
234
|
+
return Promise.resolve();
|
235
|
+
}
|
236
|
+
clone(override) {
|
237
|
+
return new _superThis({
|
238
|
+
logger: override.logger || this.logger,
|
239
|
+
env: (0, import_cement.envFactory)(override.env) || this.env,
|
240
|
+
crypto: override.crypto || this.crypto,
|
241
|
+
pathOps: override.pathOps || this.pathOps,
|
242
|
+
txt: override.txt || this.txt,
|
243
|
+
ctx: { ...this.ctx, ...override.ctx }
|
244
|
+
});
|
245
|
+
}
|
246
|
+
};
|
247
|
+
pathOpsImpl = class {
|
248
|
+
join(...paths) {
|
249
|
+
return paths.map((i) => i.replace(/\/+$/, "")).join("/");
|
250
|
+
}
|
251
|
+
dirname(path) {
|
252
|
+
return path.split("/").slice(0, -1).join("/");
|
253
|
+
}
|
254
|
+
// homedir() {
|
255
|
+
// throw new Error("SysContainer:homedir is not available in seeded state");
|
256
|
+
// }
|
257
|
+
};
|
258
|
+
pathOps = new pathOpsImpl();
|
259
|
+
txtOps = {
|
260
|
+
encode: (input) => new TextEncoder().encode(input),
|
261
|
+
decode: (input) => new TextDecoder().decode(input)
|
262
|
+
};
|
583
263
|
NotFoundError = class extends Error {
|
584
264
|
constructor() {
|
585
265
|
super(...arguments);
|
@@ -589,38 +269,149 @@ var init_gateway = __esm({
|
|
589
269
|
}
|
590
270
|
});
|
591
271
|
|
592
|
-
// src/runtime/
|
593
|
-
var
|
594
|
-
__export(
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
272
|
+
// src/runtime/gateways/file/mem-filesystem.ts
|
273
|
+
var mem_filesystem_exports = {};
|
274
|
+
__export(mem_filesystem_exports, {
|
275
|
+
MemFileSystem: () => MemFileSystem
|
276
|
+
});
|
277
|
+
var import_memfs, MemFileSystem;
|
278
|
+
var init_mem_filesystem = __esm({
|
279
|
+
"src/runtime/gateways/file/mem-filesystem.ts"() {
|
280
|
+
"use strict";
|
281
|
+
import_memfs = require("memfs");
|
282
|
+
init_utils2();
|
283
|
+
MemFileSystem = class {
|
284
|
+
async start() {
|
285
|
+
return this;
|
286
|
+
}
|
287
|
+
mkdir(path, options) {
|
288
|
+
return import_memfs.fs.promises.mkdir(path, options);
|
289
|
+
}
|
290
|
+
readdir(path, options) {
|
291
|
+
return import_memfs.fs.promises.readdir(path, options);
|
292
|
+
}
|
293
|
+
rm(path, options) {
|
294
|
+
return import_memfs.fs.promises.rm(path, options);
|
295
|
+
}
|
296
|
+
copyFile(source, destination) {
|
297
|
+
return import_memfs.fs.promises.copyFile(source, destination);
|
298
|
+
}
|
299
|
+
async readfile(path, options) {
|
300
|
+
const ret = await import_memfs.fs.promises.readFile(path, options);
|
301
|
+
return toArrayBuffer(ret);
|
302
|
+
}
|
303
|
+
stat(path) {
|
304
|
+
return import_memfs.fs.promises.stat(path);
|
305
|
+
}
|
306
|
+
unlink(path) {
|
307
|
+
return import_memfs.fs.promises.unlink(path);
|
308
|
+
}
|
309
|
+
writefile(path, data) {
|
310
|
+
return import_memfs.fs.promises.writeFile(path, Buffer.from(data));
|
311
|
+
}
|
312
|
+
};
|
313
|
+
}
|
314
|
+
});
|
315
|
+
|
316
|
+
// src/runtime/gateways/file/node-filesystem.ts
|
317
|
+
var node_filesystem_exports = {};
|
318
|
+
__export(node_filesystem_exports, {
|
319
|
+
NodeFileSystem: () => NodeFileSystem
|
320
|
+
});
|
321
|
+
var NodeFileSystem;
|
322
|
+
var init_node_filesystem = __esm({
|
323
|
+
"src/runtime/gateways/file/node-filesystem.ts"() {
|
324
|
+
"use strict";
|
325
|
+
init_utils2();
|
326
|
+
NodeFileSystem = class {
|
327
|
+
async start() {
|
328
|
+
this.fs = await import("fs/promises");
|
329
|
+
return this;
|
330
|
+
}
|
331
|
+
async mkdir(path, options) {
|
332
|
+
return this.fs?.mkdir(path, options);
|
333
|
+
}
|
334
|
+
async readdir(path, options) {
|
335
|
+
return this.fs?.readdir(path, options);
|
336
|
+
}
|
337
|
+
async rm(path, options) {
|
338
|
+
return this.fs?.rm(path, options);
|
339
|
+
}
|
340
|
+
async copyFile(source, destination) {
|
341
|
+
return this.fs?.copyFile(source, destination);
|
342
|
+
}
|
343
|
+
async readfile(path, options) {
|
344
|
+
const ret = await this.fs?.readFile(path, options);
|
345
|
+
return toArrayBuffer(ret);
|
346
|
+
}
|
347
|
+
stat(path) {
|
348
|
+
return this.fs?.stat(path);
|
349
|
+
}
|
350
|
+
async unlink(path) {
|
351
|
+
return this.fs?.unlink(path);
|
352
|
+
}
|
353
|
+
async writefile(path, data) {
|
354
|
+
return this.fs?.writeFile(path, Buffer.from(data));
|
355
|
+
}
|
356
|
+
};
|
357
|
+
}
|
358
|
+
});
|
359
|
+
|
360
|
+
// src/runtime/gateways/file/utils.ts
|
361
|
+
var utils_exports = {};
|
362
|
+
__export(utils_exports, {
|
363
|
+
getFileName: () => getFileName,
|
364
|
+
getFileSystem: () => getFileSystem,
|
365
|
+
getPath: () => getPath,
|
366
|
+
toArrayBuffer: () => toArrayBuffer
|
599
367
|
});
|
600
|
-
async function
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
}
|
616
|
-
const v = await SysContainer.readfile(vFile);
|
617
|
-
if (v.toString() !== FILESTORE_VERSION) {
|
618
|
-
console.warn(`version mismatch:${vFile}: ${v.toString()}!=${FILESTORE_VERSION}`);
|
368
|
+
async function getFileSystem(url) {
|
369
|
+
const name = url.getParam("fs");
|
370
|
+
let fs2;
|
371
|
+
switch (name) {
|
372
|
+
case "mem":
|
373
|
+
{
|
374
|
+
const { MemFileSystem: MemFileSystem2 } = await Promise.resolve().then(() => (init_mem_filesystem(), mem_filesystem_exports));
|
375
|
+
fs2 = new MemFileSystem2();
|
376
|
+
}
|
377
|
+
break;
|
378
|
+
case "node":
|
379
|
+
case "sys":
|
380
|
+
default: {
|
381
|
+
const { NodeFileSystem: NodeFileSystem2 } = await Promise.resolve().then(() => (init_node_filesystem(), node_filesystem_exports));
|
382
|
+
fs2 = new NodeFileSystem2();
|
619
383
|
}
|
620
|
-
}
|
621
|
-
return
|
384
|
+
}
|
385
|
+
return fs2.start();
|
386
|
+
}
|
387
|
+
function getPath(url, sthis) {
|
388
|
+
const basePath = url.pathname;
|
389
|
+
const name = url.getParam("name");
|
390
|
+
if (name) {
|
391
|
+
const version = url.getParam("version");
|
392
|
+
if (!version) throw sthis.logger.Error().Url(url).Msg(`version not found`).AsError();
|
393
|
+
return sthis.pathOps.join(basePath, version, name);
|
394
|
+
}
|
395
|
+
return sthis.pathOps.join(basePath);
|
396
|
+
}
|
397
|
+
function getFileName(url, sthis) {
|
398
|
+
const key = url.getParam("key");
|
399
|
+
if (!key) throw sthis.logger.Error().Url(url).Msg(`key not found`).AsError();
|
400
|
+
const res = getStore(url, sthis, (...a) => a.join("-"));
|
401
|
+
switch (res.store) {
|
402
|
+
case "data":
|
403
|
+
return sthis.pathOps.join(res.name, key + ".car");
|
404
|
+
case "wal":
|
405
|
+
case "meta":
|
406
|
+
return sthis.pathOps.join(res.name, key + ".json");
|
407
|
+
default:
|
408
|
+
throw sthis.logger.Error().Url(url).Msg(`unsupported store type`).AsError();
|
409
|
+
}
|
622
410
|
}
|
623
411
|
function toArrayBuffer(buffer) {
|
412
|
+
if (typeof buffer === "string") {
|
413
|
+
buffer = Buffer.from(buffer);
|
414
|
+
}
|
624
415
|
const ab = new ArrayBuffer(buffer.length);
|
625
416
|
const view = new Uint8Array(ab);
|
626
417
|
for (let i = 0; i < buffer.length; ++i) {
|
@@ -628,167 +419,290 @@ function toArrayBuffer(buffer) {
|
|
628
419
|
}
|
629
420
|
return view;
|
630
421
|
}
|
631
|
-
var
|
632
|
-
|
633
|
-
|
422
|
+
var init_utils2 = __esm({
|
423
|
+
"src/runtime/gateways/file/utils.ts"() {
|
424
|
+
"use strict";
|
425
|
+
init_utils();
|
426
|
+
}
|
427
|
+
});
|
428
|
+
|
429
|
+
// src/runtime/key-bag-file.ts
|
430
|
+
var key_bag_file_exports = {};
|
431
|
+
__export(key_bag_file_exports, {
|
432
|
+
KeyBagProviderFile: () => KeyBagProviderFile
|
433
|
+
});
|
434
|
+
var KeyBagProviderFile;
|
435
|
+
var init_key_bag_file = __esm({
|
436
|
+
"src/runtime/key-bag-file.ts"() {
|
437
|
+
"use strict";
|
438
|
+
init_utils();
|
439
|
+
KeyBagProviderFile = class {
|
440
|
+
async _prepare(id) {
|
441
|
+
await this.sthis.start();
|
442
|
+
let sysFS;
|
443
|
+
switch (this.url.protocol) {
|
444
|
+
case "file:": {
|
445
|
+
const { getFileSystem: getFileSystem2 } = await Promise.resolve().then(() => (init_utils2(), utils_exports));
|
446
|
+
sysFS = await getFileSystem2(this.url);
|
447
|
+
break;
|
448
|
+
}
|
449
|
+
default:
|
450
|
+
throw this.logger.Error().Url(this.url).Msg("unsupported protocol").AsError();
|
451
|
+
}
|
452
|
+
const dirName = this.url.pathname;
|
453
|
+
await sysFS.mkdir(dirName, { recursive: true });
|
454
|
+
return {
|
455
|
+
dirName,
|
456
|
+
sysFS,
|
457
|
+
fName: this.sthis.pathOps.join(dirName, `${id.replace(/[^a-zA-Z0-9]/g, "_")}.json`)
|
458
|
+
};
|
459
|
+
}
|
460
|
+
constructor(url, sthis) {
|
461
|
+
this.url = url;
|
462
|
+
this.sthis = sthis;
|
463
|
+
this.logger = sthis.logger;
|
464
|
+
}
|
465
|
+
async get(id) {
|
466
|
+
const ctx = await this._prepare(id);
|
467
|
+
try {
|
468
|
+
const p = await ctx.sysFS.readfile(ctx.fName);
|
469
|
+
const ki = JSON.parse(this.sthis.txt.decode(p));
|
470
|
+
return ki;
|
471
|
+
} catch (e) {
|
472
|
+
if (isNotFoundError(e)) {
|
473
|
+
return void 0;
|
474
|
+
}
|
475
|
+
throw this.logger.Error().Err(e).Str("file", ctx.dirName).Msg("read bag failed").AsError();
|
476
|
+
}
|
477
|
+
}
|
478
|
+
async set(id, item) {
|
479
|
+
const ctx = await this._prepare(id);
|
480
|
+
const p = this.sthis.txt.encode(JSON.stringify(item, null, 2));
|
481
|
+
await ctx.sysFS.writefile(ctx.fName, p);
|
482
|
+
}
|
483
|
+
};
|
484
|
+
}
|
485
|
+
});
|
486
|
+
|
487
|
+
// src/runtime/key-bag-indexdb.ts
|
488
|
+
var key_bag_indexdb_exports = {};
|
489
|
+
__export(key_bag_indexdb_exports, {
|
490
|
+
KeyBagProviderIndexDB: () => KeyBagProviderIndexDB
|
491
|
+
});
|
492
|
+
var import_idb, import_cement4, KeyBagProviderIndexDB;
|
493
|
+
var init_key_bag_indexdb = __esm({
|
494
|
+
"src/runtime/key-bag-indexdb.ts"() {
|
495
|
+
"use strict";
|
496
|
+
import_idb = require("idb");
|
497
|
+
init_utils2();
|
498
|
+
import_cement4 = require("@adviser/cement");
|
499
|
+
KeyBagProviderIndexDB = class {
|
500
|
+
constructor(url, sthis) {
|
501
|
+
this._db = new import_cement4.ResolveOnce();
|
502
|
+
this.sthis = sthis;
|
503
|
+
this.logger = sthis.logger;
|
504
|
+
this.url = url;
|
505
|
+
this.dbName = getPath(this.url, this.sthis);
|
506
|
+
}
|
507
|
+
async _prepare() {
|
508
|
+
return this._db.once(async () => {
|
509
|
+
return await (0, import_idb.openDB)(this.dbName, 1, {
|
510
|
+
upgrade(db) {
|
511
|
+
["bag"].map((store) => {
|
512
|
+
db.createObjectStore(store, {
|
513
|
+
autoIncrement: false
|
514
|
+
});
|
515
|
+
});
|
516
|
+
}
|
517
|
+
});
|
518
|
+
});
|
519
|
+
}
|
520
|
+
async get(id) {
|
521
|
+
const db = await this._prepare();
|
522
|
+
const tx = db.transaction(["bag"], "readonly");
|
523
|
+
const keyItem = await tx.objectStore("bag").get(id);
|
524
|
+
await tx.done;
|
525
|
+
if (!keyItem) {
|
526
|
+
return void 0;
|
527
|
+
}
|
528
|
+
return keyItem;
|
529
|
+
}
|
530
|
+
async set(id, item) {
|
531
|
+
const db = await this._prepare();
|
532
|
+
const tx = db.transaction(["bag"], "readwrite");
|
533
|
+
await tx.objectStore("bag").put(item, id);
|
534
|
+
await tx.done;
|
535
|
+
}
|
536
|
+
};
|
537
|
+
}
|
538
|
+
});
|
539
|
+
|
540
|
+
// src/runtime/gateways/file/version.ts
|
541
|
+
var FILESTORE_VERSION;
|
542
|
+
var init_version = __esm({
|
543
|
+
"src/runtime/gateways/file/version.ts"() {
|
634
544
|
"use strict";
|
635
|
-
|
636
|
-
|
637
|
-
|
545
|
+
FILESTORE_VERSION = "v0.19-file";
|
546
|
+
}
|
547
|
+
});
|
548
|
+
|
549
|
+
// src/runtime/gateways/file/gateway.ts
|
550
|
+
var gateway_exports = {};
|
551
|
+
__export(gateway_exports, {
|
552
|
+
FileGateway: () => FileGateway,
|
553
|
+
FileTestStore: () => FileTestStore
|
554
|
+
});
|
555
|
+
var import_cement8, versionFiles, FileGateway, FileTestStore;
|
556
|
+
var init_gateway = __esm({
|
557
|
+
"src/runtime/gateways/file/gateway.ts"() {
|
558
|
+
"use strict";
|
559
|
+
init_version();
|
560
|
+
import_cement8 = require("@adviser/cement");
|
638
561
|
init_utils();
|
639
|
-
|
640
|
-
|
641
|
-
versionFiles = /* @__PURE__ */ new Map();
|
562
|
+
init_utils2();
|
563
|
+
versionFiles = new import_cement8.KeyedResolvOnce();
|
642
564
|
FileGateway = class {
|
643
|
-
|
644
|
-
this.logger
|
565
|
+
get fs() {
|
566
|
+
if (!this._fs) throw this.logger.Error().Msg("fs not initialized").AsError();
|
567
|
+
return this._fs;
|
568
|
+
}
|
569
|
+
constructor(sthis) {
|
570
|
+
this.sthis = sthis;
|
571
|
+
this.logger = sthis.logger;
|
572
|
+
}
|
573
|
+
async getVersionFromFile(path, logger) {
|
574
|
+
return versionFiles.get(path).once(async () => {
|
575
|
+
await this.fs.mkdir(path, { recursive: true });
|
576
|
+
const vFile = this.sthis.pathOps.join(path, "version");
|
577
|
+
const vFileStat = await this.fs.stat(vFile).catch(() => void 0);
|
578
|
+
if (!vFileStat) {
|
579
|
+
await this.fs.writefile(this.sthis.pathOps.join(path, "version"), FILESTORE_VERSION);
|
580
|
+
return FILESTORE_VERSION;
|
581
|
+
} else if (!vFileStat.isFile()) {
|
582
|
+
throw logger.Error().Str("file", vFile).Msg(`version file is a directory`).AsError();
|
583
|
+
}
|
584
|
+
const v = await this.fs.readfile(vFile);
|
585
|
+
const vStr = new TextDecoder().decode(v);
|
586
|
+
if (vStr !== FILESTORE_VERSION) {
|
587
|
+
logger.Warn().Str("file", vFile).Str("from", vStr).Str("expected", FILESTORE_VERSION).Msg(`version mismatch`);
|
588
|
+
}
|
589
|
+
return vStr;
|
590
|
+
});
|
645
591
|
}
|
646
592
|
start(baseURL) {
|
647
593
|
return exception2Result(async () => {
|
648
|
-
await
|
649
|
-
|
650
|
-
const
|
651
|
-
|
652
|
-
const
|
653
|
-
|
654
|
-
|
655
|
-
this.
|
656
|
-
|
594
|
+
this._fs = await getFileSystem(baseURL);
|
595
|
+
await this.fs.start();
|
596
|
+
const url = baseURL.build();
|
597
|
+
url.defParam("version", FILESTORE_VERSION);
|
598
|
+
const dbUrl = await this.buildUrl(url.URI(), "dummy");
|
599
|
+
const dbdirFile = this.getFilePath(dbUrl.Ok());
|
600
|
+
await this.fs.mkdir(this.sthis.pathOps.dirname(dbdirFile), { recursive: true });
|
601
|
+
const dbroot = this.sthis.pathOps.dirname(dbdirFile);
|
602
|
+
this.logger.Debug().Url(url.URI()).Str("dbroot", dbroot).Msg("start");
|
603
|
+
url.setParam("version", await this.getVersionFromFile(dbroot, this.logger));
|
604
|
+
return url.URI();
|
657
605
|
});
|
658
606
|
}
|
607
|
+
async buildUrl(baseUrl, key) {
|
608
|
+
return import_cement8.Result.Ok(baseUrl.build().setParam("key", key).URI());
|
609
|
+
}
|
659
610
|
async close() {
|
660
|
-
return
|
611
|
+
return import_cement8.Result.Ok(void 0);
|
661
612
|
}
|
662
|
-
|
663
|
-
|
664
|
-
|
665
|
-
|
613
|
+
// abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
|
614
|
+
getFilePath(url) {
|
615
|
+
const key = url.getParam("key");
|
616
|
+
if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
|
617
|
+
return this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
666
618
|
}
|
667
|
-
async put(
|
619
|
+
async put(url, body) {
|
668
620
|
return exception2Result(async () => {
|
669
|
-
const file = this.getFilePath(
|
670
|
-
this.logger.Debug().Str("url",
|
671
|
-
await
|
621
|
+
const file = await this.getFilePath(url);
|
622
|
+
this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
|
623
|
+
await this.fs.writefile(file, body);
|
672
624
|
});
|
673
625
|
}
|
674
|
-
async get(
|
626
|
+
async get(url) {
|
675
627
|
return exceptionWrapper(async () => {
|
676
|
-
const file = this.getFilePath(
|
628
|
+
const file = this.getFilePath(url);
|
677
629
|
try {
|
678
|
-
const res = await
|
679
|
-
this.logger.Debug().Url(
|
680
|
-
return
|
630
|
+
const res = await this.fs.readfile(file);
|
631
|
+
this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get");
|
632
|
+
return import_cement8.Result.Ok(new Uint8Array(res));
|
681
633
|
} catch (e) {
|
682
634
|
if (isNotFoundError(e)) {
|
683
|
-
return
|
635
|
+
return import_cement8.Result.Err(new NotFoundError(`file not found: ${file}`));
|
684
636
|
}
|
685
|
-
return
|
637
|
+
return import_cement8.Result.Err(e);
|
686
638
|
}
|
687
639
|
});
|
688
640
|
}
|
689
|
-
async delete(
|
641
|
+
async delete(url) {
|
690
642
|
return exception2Result(async () => {
|
691
|
-
await
|
643
|
+
await this.fs.unlink(this.getFilePath(url));
|
692
644
|
});
|
693
645
|
}
|
694
|
-
async
|
695
|
-
const
|
696
|
-
if (
|
697
|
-
const filepath =
|
698
|
-
let
|
646
|
+
async destroy(baseURL) {
|
647
|
+
const url = await this.buildUrl(baseURL, "x");
|
648
|
+
if (url.isErr()) return url;
|
649
|
+
const filepath = this.sthis.pathOps.dirname(this.getFilePath(url.Ok()));
|
650
|
+
let files = [];
|
699
651
|
try {
|
700
|
-
|
652
|
+
files = await this.fs.readdir(filepath);
|
701
653
|
} catch (e) {
|
702
654
|
if (!isNotFoundError(e)) {
|
703
655
|
throw this.logger.Error().Err(e).Str("dir", filepath).Msg("destroy:readdir").AsError();
|
704
656
|
}
|
705
657
|
}
|
706
|
-
for (const file of
|
707
|
-
const pathed =
|
658
|
+
for (const file of files) {
|
659
|
+
const pathed = this.sthis.pathOps.join(filepath, file);
|
708
660
|
try {
|
709
|
-
await
|
661
|
+
await this.fs.unlink(pathed);
|
710
662
|
} catch (e) {
|
711
663
|
if (!isNotFoundError(e)) {
|
712
664
|
throw this.logger.Error().Err(e).Str("file", pathed).Msg("destroy:unlink").AsError();
|
713
665
|
}
|
714
666
|
}
|
715
667
|
}
|
716
|
-
return
|
717
|
-
}
|
718
|
-
};
|
719
|
-
FileWALGateway = class extends FileGateway {
|
720
|
-
constructor(logger) {
|
721
|
-
super(ensureLogger(logger, "FileWALGateway"));
|
722
|
-
}
|
723
|
-
async destroy(baseURL) {
|
724
|
-
return this.destroyDir(baseURL);
|
725
|
-
}
|
726
|
-
async buildUrl(baseUrl, key) {
|
727
|
-
const url2 = new URL(baseUrl.toString());
|
728
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "wal"), key + ".json");
|
729
|
-
return import_cement6.Result.Ok(url2);
|
730
|
-
}
|
731
|
-
};
|
732
|
-
FileMetaGateway = class extends FileGateway {
|
733
|
-
constructor(logger) {
|
734
|
-
super(ensureLogger(logger, "FileMetaGateway"));
|
735
|
-
}
|
736
|
-
async destroy(baseURL) {
|
737
|
-
return this.destroyDir(baseURL);
|
738
|
-
}
|
739
|
-
async buildUrl(baseUrl, key) {
|
740
|
-
const url2 = new URL(baseUrl.toString());
|
741
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "meta"), key + ".json");
|
742
|
-
return import_cement6.Result.Ok(url2);
|
743
|
-
}
|
744
|
-
};
|
745
|
-
FileDataGateway = class extends FileGateway {
|
746
|
-
constructor(logger) {
|
747
|
-
super(ensureLogger(logger, "FileDataGateway"));
|
748
|
-
this.branches = /* @__PURE__ */ new Set();
|
749
|
-
}
|
750
|
-
async destroy(baseURL) {
|
751
|
-
return this.destroyDir(baseURL);
|
752
|
-
}
|
753
|
-
async buildUrl(baseUrl, key) {
|
754
|
-
const url2 = new URL(baseUrl.toString());
|
755
|
-
url2.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "data"), key + ".car");
|
756
|
-
return import_cement6.Result.Ok(url2);
|
668
|
+
return import_cement8.Result.Ok(void 0);
|
757
669
|
}
|
758
670
|
};
|
759
671
|
FileTestStore = class {
|
760
|
-
constructor(
|
761
|
-
this.logger = ensureLogger(
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
const
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
);
|
770
|
-
|
771
|
-
const buffer = await SysContainer.readfile(dbFile);
|
772
|
-
logger.Debug().Str("dbFile", dbFile).Len(buffer).Msg("got");
|
773
|
-
return toArrayBuffer(buffer);
|
672
|
+
constructor(sthis) {
|
673
|
+
this.logger = ensureLogger(sthis, "FileTestStore");
|
674
|
+
this.sthis = sthis;
|
675
|
+
}
|
676
|
+
async get(iurl, key) {
|
677
|
+
const url = iurl.build().setParam("key", key).URI();
|
678
|
+
const dbFile = this.sthis.pathOps.join(getPath(url, this.sthis), getFileName(url, this.sthis));
|
679
|
+
this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
|
680
|
+
const buffer = await (await getFileSystem(url)).readfile(dbFile);
|
681
|
+
this.logger.Debug().Url(url).Str("dbFile", dbFile).Len(buffer).Msg("got");
|
682
|
+
return buffer;
|
774
683
|
}
|
775
684
|
};
|
776
685
|
}
|
777
686
|
});
|
778
687
|
|
779
|
-
// src/runtime/
|
780
|
-
var
|
781
|
-
|
782
|
-
|
783
|
-
|
688
|
+
// src/runtime/gateways/indexdb/version.ts
|
689
|
+
var INDEXDB_VERSION;
|
690
|
+
var init_version2 = __esm({
|
691
|
+
"src/runtime/gateways/indexdb/version.ts"() {
|
692
|
+
"use strict";
|
693
|
+
INDEXDB_VERSION = "v0.19-indexdb";
|
694
|
+
}
|
695
|
+
});
|
696
|
+
|
697
|
+
// src/runtime/gateways/indexdb/gateway.ts
|
698
|
+
var gateway_exports2 = {};
|
699
|
+
__export(gateway_exports2, {
|
700
|
+
IndexDBGateway: () => IndexDBGateway,
|
784
701
|
IndexDBTestStore: () => IndexDBTestStore,
|
785
|
-
IndexDBWalGateway: () => IndexDBWalGateway,
|
786
702
|
getIndexDBName: () => getIndexDBName
|
787
703
|
});
|
788
|
-
function ensureVersion(
|
789
|
-
|
790
|
-
ret.searchParams.set("version", url2.searchParams.get("version") || INDEXDB_VERSION);
|
791
|
-
return ret;
|
704
|
+
function ensureVersion(url) {
|
705
|
+
return url.build().defParam("version", INDEXDB_VERSION).URI();
|
792
706
|
}
|
793
707
|
function sanitzeKey(key) {
|
794
708
|
if (key.length === 1) {
|
@@ -796,10 +710,10 @@ function sanitzeKey(key) {
|
|
796
710
|
}
|
797
711
|
return key;
|
798
712
|
}
|
799
|
-
async function connectIdb(
|
800
|
-
const dbName = getIndexDBName(
|
713
|
+
async function connectIdb(url, sthis) {
|
714
|
+
const dbName = getIndexDBName(url, sthis);
|
801
715
|
const once = await onceIndexDB.get(dbName.fullDb).once(async () => {
|
802
|
-
const db = await (0,
|
716
|
+
const db = await (0, import_idb2.openDB)(dbName.fullDb, 1, {
|
803
717
|
upgrade(db2) {
|
804
718
|
["version", "data", "wal", "meta", "idx.data", "idx.wal", "idx.meta"].map((store) => {
|
805
719
|
db2.createObjectStore(store, {
|
@@ -809,27 +723,29 @@ async function connectIdb(url2, logger) {
|
|
809
723
|
}
|
810
724
|
});
|
811
725
|
const found = await db.get("version", "version");
|
812
|
-
const version =
|
726
|
+
const version = ensureVersion(url).getParam("version");
|
813
727
|
if (!found) {
|
814
728
|
await db.put("version", { version }, "version");
|
815
729
|
} else if (found.version !== version) {
|
816
|
-
logger.Warn().Str("url",
|
730
|
+
sthis.logger.Warn().Str("url", url.toString()).Str("version", version).Str("found", found.version).Msg("version mismatch");
|
817
731
|
}
|
818
|
-
return { db, dbName, version };
|
732
|
+
return { db, dbName, version, url };
|
819
733
|
});
|
820
|
-
|
821
|
-
|
734
|
+
return {
|
735
|
+
...once,
|
736
|
+
url: url.build().setParam("version", once.version).URI()
|
737
|
+
};
|
822
738
|
}
|
823
739
|
function joinDBName(...names) {
|
824
740
|
return names.map((i) => i.replace(/^[^a-zA-Z0-9]+/g, "").replace(/[^a-zA-Z0-9]+/g, "_")).filter((i) => i.length).join(".");
|
825
741
|
}
|
826
|
-
function getIndexDBName(iurl,
|
827
|
-
const
|
828
|
-
const fullDb =
|
829
|
-
const dbName =
|
830
|
-
if (!dbName) throw logger.Error().Str("url",
|
742
|
+
function getIndexDBName(iurl, sthis) {
|
743
|
+
const url = ensureVersion(iurl);
|
744
|
+
const fullDb = url.pathname.replace(/^\/+/, "").replace(/\?.*$/, "");
|
745
|
+
const dbName = url.getParam("name");
|
746
|
+
if (!dbName) throw sthis.logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
|
831
747
|
const result = joinDBName(fullDb, dbName);
|
832
|
-
const objStore = getStore(
|
748
|
+
const objStore = getStore(url, sthis, joinDBName).name;
|
833
749
|
const connectionKey = [result, objStore].join(":");
|
834
750
|
return {
|
835
751
|
fullDb: result,
|
@@ -838,40 +754,38 @@ function getIndexDBName(iurl, logger) {
|
|
838
754
|
dbName
|
839
755
|
};
|
840
756
|
}
|
841
|
-
var
|
842
|
-
var
|
843
|
-
"src/runtime/
|
757
|
+
var import_idb2, import_cement9, onceIndexDB, IndexDBGateway, IndexDBTestStore;
|
758
|
+
var init_gateway2 = __esm({
|
759
|
+
"src/runtime/gateways/indexdb/gateway.ts"() {
|
844
760
|
"use strict";
|
845
|
-
|
846
|
-
|
847
|
-
|
761
|
+
import_idb2 = require("idb");
|
762
|
+
import_cement9 = require("@adviser/cement");
|
763
|
+
init_version2();
|
848
764
|
init_utils();
|
849
|
-
|
850
|
-
init_sys_container();
|
851
|
-
onceIndexDB = new import_cement7.KeyedResolvOnce();
|
765
|
+
onceIndexDB = new import_cement9.KeyedResolvOnce();
|
852
766
|
IndexDBGateway = class {
|
853
|
-
constructor(
|
854
|
-
this.
|
855
|
-
this.logger =
|
856
|
-
|
857
|
-
idb() {
|
858
|
-
this.db;
|
767
|
+
constructor(sthis) {
|
768
|
+
this._db = {};
|
769
|
+
this.logger = ensureLogger(sthis, "IndexDBGateway");
|
770
|
+
this.sthis = sthis;
|
859
771
|
}
|
860
772
|
async start(baseURL) {
|
861
773
|
return exception2Result(async () => {
|
862
774
|
this.logger.Debug().Url(baseURL).Msg("starting");
|
863
|
-
await
|
864
|
-
|
865
|
-
this.
|
775
|
+
await this.sthis.start();
|
776
|
+
const ic = await connectIdb(baseURL, this.sthis);
|
777
|
+
this._db = ic.db;
|
778
|
+
this.logger.Debug().Url(ic.url).Msg("started");
|
779
|
+
return ic.url;
|
866
780
|
});
|
867
781
|
}
|
868
782
|
async close() {
|
869
|
-
return
|
783
|
+
return import_cement9.Result.Ok(void 0);
|
870
784
|
}
|
871
785
|
async destroy(baseUrl) {
|
872
786
|
return exception2Result(async () => {
|
873
|
-
const type = getStore(baseUrl, this.
|
874
|
-
const idb = this.
|
787
|
+
const type = getStore(baseUrl, this.sthis, joinDBName).name;
|
788
|
+
const idb = this._db;
|
875
789
|
const trans = idb.transaction(type, "readwrite");
|
876
790
|
const object_store = trans.objectStore(type);
|
877
791
|
const toDelete = [];
|
@@ -884,87 +798,58 @@ var init_store_indexdb = __esm({
|
|
884
798
|
await trans.done;
|
885
799
|
});
|
886
800
|
}
|
887
|
-
|
801
|
+
buildUrl(baseUrl, key) {
|
802
|
+
return Promise.resolve(import_cement9.Result.Ok(baseUrl.build().setParam("key", key).URI()));
|
803
|
+
}
|
804
|
+
async get(url) {
|
888
805
|
return exceptionWrapper(async () => {
|
889
|
-
const key = getKey(
|
890
|
-
const store = getStore(
|
891
|
-
this.logger.Debug().Url(
|
892
|
-
const tx = this.
|
806
|
+
const key = getKey(url, this.logger);
|
807
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
808
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("getting");
|
809
|
+
const tx = this._db.transaction([store], "readonly");
|
893
810
|
const bytes = await tx.objectStore(store).get(sanitzeKey(key));
|
894
811
|
await tx.done;
|
895
812
|
if (!bytes) {
|
896
|
-
return
|
813
|
+
return import_cement9.Result.Err(new NotFoundError(`missing ${key}`));
|
897
814
|
}
|
898
|
-
return
|
815
|
+
return import_cement9.Result.Ok(bytes);
|
899
816
|
});
|
900
817
|
}
|
901
|
-
async put(
|
818
|
+
async put(url, value) {
|
902
819
|
return exception2Result(async () => {
|
903
|
-
const key = getKey(
|
904
|
-
const store = getStore(
|
905
|
-
this.logger.Debug().Url(
|
906
|
-
const tx = this.
|
820
|
+
const key = getKey(url, this.logger);
|
821
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
822
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
|
823
|
+
const tx = this._db.transaction([store], "readwrite");
|
907
824
|
await tx.objectStore(store).put(value, sanitzeKey(key));
|
908
825
|
await tx.done;
|
909
826
|
});
|
910
827
|
}
|
911
|
-
async delete(
|
828
|
+
async delete(url) {
|
912
829
|
return exception2Result(async () => {
|
913
|
-
const key = getKey(
|
914
|
-
const store = getStore(
|
915
|
-
this.logger.Debug().Url(
|
916
|
-
const tx = this.
|
830
|
+
const key = getKey(url, this.logger);
|
831
|
+
const store = getStore(url, this.sthis, joinDBName).name;
|
832
|
+
this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
|
833
|
+
const tx = this._db.transaction([store], "readwrite");
|
917
834
|
await tx.objectStore(store).delete(sanitzeKey(key));
|
918
835
|
await tx.done;
|
919
|
-
return
|
836
|
+
return import_cement9.Result.Ok(void 0);
|
920
837
|
});
|
921
838
|
}
|
922
839
|
};
|
923
|
-
IndexDBDataGateway = class extends IndexDBGateway {
|
924
|
-
constructor(logger) {
|
925
|
-
super(ensureLogger(logger, "IndexDBDataGateway", {}));
|
926
|
-
}
|
927
|
-
buildUrl(baseUrl, key) {
|
928
|
-
const url2 = new URL(baseUrl.toString());
|
929
|
-
url2.searchParams.set("key", key);
|
930
|
-
return Promise.resolve(import_cement7.Result.Ok(url2));
|
931
|
-
}
|
932
|
-
};
|
933
|
-
IndexDBWalGateway = class extends IndexDBGateway {
|
934
|
-
constructor(logger) {
|
935
|
-
super(ensureLogger(logger, "IndexDBWalGateway", {}));
|
936
|
-
}
|
937
|
-
buildUrl(baseUrl, key) {
|
938
|
-
const url2 = new URL(baseUrl.toString());
|
939
|
-
url2.searchParams.set("key", key);
|
940
|
-
return Promise.resolve(import_cement7.Result.Ok(url2));
|
941
|
-
}
|
942
|
-
};
|
943
|
-
IndexDBMetaGateway = class extends IndexDBGateway {
|
944
|
-
constructor(logger) {
|
945
|
-
super(ensureLogger(logger, "IndexDBDataGateway", {}));
|
946
|
-
this.branches = /* @__PURE__ */ new Set();
|
947
|
-
}
|
948
|
-
async buildUrl(baseUrl, key) {
|
949
|
-
const url2 = new URL(baseUrl.toString());
|
950
|
-
this.branches.add(key);
|
951
|
-
url2.searchParams.set("key", key);
|
952
|
-
return import_cement7.Result.Ok(url2);
|
953
|
-
}
|
954
|
-
};
|
955
|
-
txtEncoder = new TextEncoder();
|
956
840
|
IndexDBTestStore = class {
|
957
|
-
constructor(
|
958
|
-
this.
|
841
|
+
constructor(sthis) {
|
842
|
+
this.sthis = sthis;
|
843
|
+
this.logger = ensureLogger(sthis, "IndexDBTestStore", {});
|
959
844
|
}
|
960
|
-
async get(
|
961
|
-
const
|
962
|
-
const store = getStore(
|
845
|
+
async get(url, key) {
|
846
|
+
const ic = await connectIdb(url, this.sthis);
|
847
|
+
const store = getStore(ic.url, this.sthis, joinDBName).name;
|
963
848
|
this.logger.Debug().Str("key", key).Str("store", store).Msg("getting");
|
964
|
-
let bytes = await db.get(store, sanitzeKey(key));
|
849
|
+
let bytes = await ic.db.get(store, sanitzeKey(key));
|
965
850
|
this.logger.Debug().Str("key", key).Str("store", store).Int("len", bytes.length).Msg("got");
|
966
851
|
if (typeof bytes === "string") {
|
967
|
-
bytes =
|
852
|
+
bytes = this.sthis.txt.encode(bytes);
|
968
853
|
}
|
969
854
|
return bytes;
|
970
855
|
}
|
@@ -978,11 +863,16 @@ __export(src_exports, {
|
|
978
863
|
CRDT: () => CRDT,
|
979
864
|
Database: () => Database,
|
980
865
|
Index: () => Index,
|
866
|
+
NotFoundError: () => NotFoundError,
|
981
867
|
PACKAGE_VERSION: () => PACKAGE_VERSION,
|
982
|
-
Result: () =>
|
868
|
+
Result: () => import_cement.Result,
|
869
|
+
UInt8ArrayEqual: () => UInt8ArrayEqual,
|
983
870
|
blockstore: () => blockstore_exports,
|
984
871
|
bs: () => blockstore_exports,
|
872
|
+
dataDir: () => dataDir,
|
985
873
|
ensureLogger: () => ensureLogger,
|
874
|
+
ensureSuperLog: () => ensureSuperLog,
|
875
|
+
ensureSuperThis: () => ensureSuperThis,
|
986
876
|
exception2Result: () => exception2Result,
|
987
877
|
exceptionWrapper: () => exceptionWrapper,
|
988
878
|
falsyToUndef: () => falsyToUndef,
|
@@ -992,6 +882,7 @@ __export(src_exports, {
|
|
992
882
|
getStore: () => getStore,
|
993
883
|
index: () => index,
|
994
884
|
isFalsy: () => isFalsy,
|
885
|
+
isNotFoundError: () => isNotFoundError,
|
995
886
|
rt: () => runtime_exports,
|
996
887
|
runtime: () => runtime_exports,
|
997
888
|
throwFalsy: () => throwFalsy
|
@@ -999,8 +890,7 @@ __export(src_exports, {
|
|
999
890
|
module.exports = __toCommonJS(src_exports);
|
1000
891
|
|
1001
892
|
// src/database.ts
|
1002
|
-
var
|
1003
|
-
var import_cement11 = require("@adviser/cement");
|
893
|
+
var import_cement13 = require("@adviser/cement");
|
1004
894
|
|
1005
895
|
// src/write-queue.ts
|
1006
896
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -1043,13 +933,83 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
1043
933
|
}
|
1044
934
|
|
1045
935
|
// src/crdt.ts
|
1046
|
-
var
|
936
|
+
var import_cement12 = require("@adviser/cement");
|
937
|
+
|
938
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
939
|
+
var block_exports = {};
|
940
|
+
__export(block_exports, {
|
941
|
+
Block: () => Block,
|
942
|
+
create: () => create,
|
943
|
+
createUnsafe: () => createUnsafe,
|
944
|
+
decode: () => decode,
|
945
|
+
encode: () => encode
|
946
|
+
});
|
947
|
+
var import_multiformats = require("multiformats");
|
948
|
+
var import_block = require("multiformats/block");
|
949
|
+
var Block = import_block.Block;
|
950
|
+
async function decode({
|
951
|
+
bytes,
|
952
|
+
codec: codec3,
|
953
|
+
hasher: hasher7
|
954
|
+
}) {
|
955
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
956
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
957
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
958
|
+
const hash = await hasher7.digest(bytes);
|
959
|
+
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
960
|
+
return new import_block.Block({ value, bytes, cid });
|
961
|
+
}
|
962
|
+
async function encode({
|
963
|
+
value,
|
964
|
+
codec: codec3,
|
965
|
+
hasher: hasher7
|
966
|
+
}) {
|
967
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
968
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
969
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
970
|
+
const hash = await hasher7.digest(bytes);
|
971
|
+
const cid = import_multiformats.CID.create(1, codec3.code, hash);
|
972
|
+
return new import_block.Block({ value, bytes, cid });
|
973
|
+
}
|
974
|
+
async function create({
|
975
|
+
bytes,
|
976
|
+
cid,
|
977
|
+
hasher: hasher7,
|
978
|
+
codec: codec3
|
979
|
+
}) {
|
980
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
981
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
982
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
983
|
+
const hash = await hasher7.digest(bytes);
|
984
|
+
if (!import_multiformats.bytes.equals(cid.multihash.bytes, hash.bytes)) {
|
985
|
+
throw new Error("CID hash does not match bytes");
|
986
|
+
}
|
987
|
+
return createUnsafe({
|
988
|
+
bytes,
|
989
|
+
cid,
|
990
|
+
value,
|
991
|
+
codec: codec3
|
992
|
+
});
|
993
|
+
}
|
994
|
+
async function createUnsafe({
|
995
|
+
bytes,
|
996
|
+
cid,
|
997
|
+
value: maybeValue,
|
998
|
+
codec: codec3
|
999
|
+
}) {
|
1000
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
1001
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
1002
|
+
return new Block({
|
1003
|
+
cid,
|
1004
|
+
bytes,
|
1005
|
+
value
|
1006
|
+
});
|
1007
|
+
}
|
1047
1008
|
|
1048
1009
|
// src/crdt-helpers.ts
|
1049
|
-
var import_block6 = require("multiformats/block");
|
1050
1010
|
var import_link = require("multiformats/link");
|
1051
|
-
var
|
1052
|
-
var
|
1011
|
+
var import_sha25 = require("multiformats/hashes/sha2");
|
1012
|
+
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
1053
1013
|
var import_crdt = require("@web3-storage/pail/crdt");
|
1054
1014
|
var import_clock2 = require("@web3-storage/pail/clock");
|
1055
1015
|
var Batch = __toESM(require("@web3-storage/pail/crdt/batch"), 1);
|
@@ -1060,525 +1020,149 @@ __export(blockstore_exports, {
|
|
1060
1020
|
BaseBlockstore: () => BaseBlockstore,
|
1061
1021
|
CarTransaction: () => CarTransaction,
|
1062
1022
|
CompactionFetcher: () => CompactionFetcher,
|
1063
|
-
ConnectREST: () => ConnectREST,
|
1064
1023
|
ConnectionBase: () => ConnectionBase,
|
1065
|
-
DataStore: () => DataStore,
|
1066
1024
|
EncryptedBlockstore: () => EncryptedBlockstore,
|
1067
|
-
|
1025
|
+
FragmentGateway: () => FragmentGateway,
|
1068
1026
|
Loader: () => Loader,
|
1069
|
-
|
1070
|
-
|
1071
|
-
RemoteWAL: () => RemoteWAL,
|
1072
|
-
isNotFoundError: () => isNotFoundError,
|
1027
|
+
ensureStart: () => ensureStart,
|
1028
|
+
getGatewayFromURL: () => getGatewayFromURL,
|
1073
1029
|
parseCarFile: () => parseCarFile,
|
1074
1030
|
registerStoreProtocol: () => registerStoreProtocol,
|
1075
1031
|
testStoreFactory: () => testStoreFactory,
|
1076
|
-
|
1077
|
-
|
1032
|
+
toCIDBlock: () => toCIDBlock,
|
1033
|
+
toStoreRuntime: () => toStoreRuntime
|
1078
1034
|
});
|
1079
1035
|
|
1080
|
-
// src/blockstore/
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1036
|
+
// src/blockstore/types.ts
|
1037
|
+
function toCIDBlock(block) {
|
1038
|
+
return block;
|
1039
|
+
}
|
1084
1040
|
|
1085
|
-
// src/blockstore/
|
1086
|
-
|
1087
|
-
var TaskManager = class {
|
1088
|
-
constructor(loader) {
|
1089
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
1090
|
-
this.queue = [];
|
1091
|
-
this.isProcessing = false;
|
1092
|
-
this.loader = loader;
|
1093
|
-
this.logger = ensureLogger(loader.logger, "TaskManager");
|
1094
|
-
}
|
1095
|
-
async handleEvent(eventBlock) {
|
1096
|
-
const cid = eventBlock.cid.toString();
|
1097
|
-
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
1098
|
-
for (const parent of parents) {
|
1099
|
-
this.eventsWeHandled.add(parent);
|
1100
|
-
}
|
1101
|
-
this.queue.push({ cid, eventBlock, retries: 0 });
|
1102
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
1103
|
-
void this.processQueue();
|
1104
|
-
}
|
1105
|
-
async processQueue() {
|
1106
|
-
if (this.isProcessing) return;
|
1107
|
-
this.isProcessing = true;
|
1108
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1109
|
-
const first = filteredQueue[0];
|
1110
|
-
if (!first) {
|
1111
|
-
return;
|
1112
|
-
}
|
1113
|
-
try {
|
1114
|
-
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
1115
|
-
this.eventsWeHandled.add(first.cid);
|
1116
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
1117
|
-
} catch (err) {
|
1118
|
-
if (first.retries++ > 3) {
|
1119
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
1120
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
1121
|
-
}
|
1122
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
1123
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
1124
|
-
} finally {
|
1125
|
-
this.isProcessing = false;
|
1126
|
-
if (this.queue.length > 0) {
|
1127
|
-
void this.processQueue();
|
1128
|
-
}
|
1129
|
-
}
|
1130
|
-
}
|
1131
|
-
};
|
1041
|
+
// src/blockstore/store-factory.ts
|
1042
|
+
var import_cement10 = require("@adviser/cement");
|
1132
1043
|
|
1133
|
-
// src/
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
|
1166
|
-
|
1044
|
+
// src/runtime/files.ts
|
1045
|
+
var files_exports = {};
|
1046
|
+
__export(files_exports, {
|
1047
|
+
decodeFile: () => decodeFile,
|
1048
|
+
encodeFile: () => encodeFile
|
1049
|
+
});
|
1050
|
+
var UnixFS = __toESM(require("@ipld/unixfs"), 1);
|
1051
|
+
var raw = __toESM(require("multiformats/codecs/raw"), 1);
|
1052
|
+
var import_fixed = require("@ipld/unixfs/file/chunker/fixed");
|
1053
|
+
var import_balanced = require("@ipld/unixfs/file/layout/balanced");
|
1054
|
+
var import_ipfs_unixfs_exporter = require("ipfs-unixfs-exporter");
|
1055
|
+
var queuingStrategy = UnixFS.withCapacity();
|
1056
|
+
var settings = UnixFS.configure({
|
1057
|
+
fileChunkEncoder: raw,
|
1058
|
+
smallFileEncoder: raw,
|
1059
|
+
chunker: (0, import_fixed.withMaxChunkSize)(1024 * 1024),
|
1060
|
+
fileLayout: (0, import_balanced.withWidth)(1024)
|
1061
|
+
});
|
1062
|
+
async function collect(collectable) {
|
1063
|
+
const chunks = [];
|
1064
|
+
await collectable.pipeTo(
|
1065
|
+
new WritableStream({
|
1066
|
+
write(chunk) {
|
1067
|
+
chunks.push(chunk);
|
1068
|
+
}
|
1069
|
+
})
|
1070
|
+
);
|
1071
|
+
return chunks;
|
1072
|
+
}
|
1073
|
+
async function encodeFile(blob) {
|
1074
|
+
const readable = createFileEncoderStream(blob);
|
1075
|
+
const blocks = await collect(readable);
|
1076
|
+
return { cid: blocks.at(-1).cid, blocks };
|
1077
|
+
}
|
1078
|
+
async function decodeFile(blocks, cid, meta) {
|
1079
|
+
const entry = await (0, import_ipfs_unixfs_exporter.exporter)(cid.toString(), blocks, { length: meta.size });
|
1080
|
+
const chunks = [];
|
1081
|
+
for await (const chunk of entry.content()) {
|
1082
|
+
chunks.push(chunk);
|
1167
1083
|
}
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1084
|
+
return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
|
1085
|
+
}
|
1086
|
+
function createFileEncoderStream(blob) {
|
1087
|
+
const { readable, writable } = new TransformStream({}, queuingStrategy);
|
1088
|
+
const unixfsWriter = UnixFS.createWriter({ writable, settings });
|
1089
|
+
const fileBuilder = new UnixFSFileBuilder("", blob);
|
1090
|
+
void (async () => {
|
1091
|
+
await fileBuilder.finalize(unixfsWriter);
|
1092
|
+
await unixfsWriter.close();
|
1093
|
+
})();
|
1094
|
+
return readable;
|
1095
|
+
}
|
1096
|
+
var UnixFSFileBuilder = class {
|
1097
|
+
#file;
|
1098
|
+
constructor(name, file) {
|
1099
|
+
this.name = name;
|
1100
|
+
this.#file = file;
|
1101
|
+
}
|
1102
|
+
async finalize(writer) {
|
1103
|
+
const unixfsFileWriter = UnixFS.createFileWriter(writer);
|
1104
|
+
await this.#file.stream().pipeTo(
|
1105
|
+
new WritableStream({
|
1106
|
+
async write(chunk) {
|
1107
|
+
await unixfsFileWriter.write(chunk);
|
1108
|
+
}
|
1109
|
+
})
|
1175
1110
|
);
|
1176
|
-
await
|
1177
|
-
return event;
|
1178
|
-
}
|
1179
|
-
async decodeEventBlock(bytes) {
|
1180
|
-
const event = await (0, import_clock.decodeEventBlock)(bytes);
|
1181
|
-
return event;
|
1111
|
+
return await unixfsFileWriter.close();
|
1182
1112
|
}
|
1183
|
-
// move this stuff to connect
|
1184
|
-
// async getDashboardURL(compact = true) {
|
1185
|
-
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
1186
|
-
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
1187
|
-
// // if (compact) {
|
1188
|
-
// // await this.compact()
|
1189
|
-
// // }
|
1190
|
-
// const currents = await this.loader?.metaStore?.load()
|
1191
|
-
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
1192
|
-
// if (currents.length > 1)
|
1193
|
-
// throw new Error("Can't sync database with split heads: make an update first")
|
1194
|
-
// const current = currents[0]
|
1195
|
-
// const params = {
|
1196
|
-
// car: current.car.toString()
|
1197
|
-
// }
|
1198
|
-
// if (current.key) {
|
1199
|
-
// // @ts-ignore
|
1200
|
-
// params.key = current.key.toString()
|
1201
|
-
// }
|
1202
|
-
// // @ts-ignore
|
1203
|
-
// if (this.name) {
|
1204
|
-
// // @ts-ignore
|
1205
|
-
// params.name = this.name
|
1206
|
-
// }
|
1207
|
-
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
1208
|
-
// console.log('Import to dashboard: ' + url.toString())
|
1209
|
-
// return url
|
1210
|
-
// }
|
1211
|
-
// openDashboard() {
|
1212
|
-
// void this.getDashboardURL().then(url => {
|
1213
|
-
// if (url) window.open(url.toString(), '_blank')
|
1214
|
-
// })
|
1215
|
-
// }
|
1216
1113
|
};
|
1217
1114
|
|
1218
|
-
// src/blockstore/
|
1219
|
-
|
1220
|
-
var
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
throw this.logger.Error().Msg("failed to upload data " + done.statusText);
|
1231
|
-
}
|
1232
|
-
}
|
1233
|
-
async dataDownload(params) {
|
1234
|
-
const { car } = params;
|
1235
|
-
const fetchFromUrl = new URL(`/cars/${car.toString()}.car`, this.baseUrl);
|
1236
|
-
const response = await fetch(fetchFromUrl);
|
1237
|
-
if (!response.ok) {
|
1238
|
-
return void 0;
|
1239
|
-
}
|
1240
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
1241
|
-
return bytes;
|
1115
|
+
// src/blockstore/store.ts
|
1116
|
+
var import_p_limit2 = __toESM(require("p-limit"), 1);
|
1117
|
+
var import_dag_json = require("@ipld/dag-json");
|
1118
|
+
var import_cement7 = require("@adviser/cement");
|
1119
|
+
|
1120
|
+
// src/types.ts
|
1121
|
+
function isFalsy(value) {
|
1122
|
+
return value === false && value === null && value === void 0;
|
1123
|
+
}
|
1124
|
+
function throwFalsy(value) {
|
1125
|
+
if (isFalsy(value)) {
|
1126
|
+
throw new Error("value is Falsy");
|
1242
1127
|
}
|
1243
|
-
|
1244
|
-
|
1128
|
+
return value;
|
1129
|
+
}
|
1130
|
+
function falsyToUndef(value) {
|
1131
|
+
if (isFalsy(value)) {
|
1245
1132
|
return void 0;
|
1246
1133
|
}
|
1247
|
-
|
1248
|
-
|
1249
|
-
return [];
|
1250
|
-
}
|
1251
|
-
};
|
1252
|
-
|
1253
|
-
// src/blockstore/store-factory.ts
|
1254
|
-
var import_cement8 = require("@adviser/cement");
|
1255
|
-
init_data_dir();
|
1256
|
-
init_files();
|
1134
|
+
return value;
|
1135
|
+
}
|
1257
1136
|
|
1258
1137
|
// src/blockstore/store.ts
|
1259
|
-
var import_p_limit2 = __toESM(require("p-limit"), 1);
|
1260
|
-
var import_dag_json = require("@ipld/dag-json");
|
1261
|
-
var import_cement5 = require("@adviser/cement");
|
1262
|
-
init_types();
|
1263
|
-
init_gateway();
|
1264
1138
|
init_utils();
|
1265
1139
|
|
1266
1140
|
// src/blockstore/loader.ts
|
1267
1141
|
var import_p_limit = __toESM(require("p-limit"), 1);
|
1268
1142
|
var import_car = require("@ipld/car");
|
1269
|
-
var
|
1270
|
-
|
1271
|
-
// src/blockstore/types.ts
|
1272
|
-
function toCIDBlock(block) {
|
1273
|
-
return block;
|
1274
|
-
}
|
1143
|
+
var import_cement6 = require("@adviser/cement");
|
1275
1144
|
|
1276
1145
|
// src/blockstore/loader-helpers.ts
|
1277
|
-
var import_block2 = require("multiformats/block");
|
1278
1146
|
var import_sha2 = require("multiformats/hashes/sha2");
|
1279
|
-
var
|
1280
|
-
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
1281
|
-
var codec = __toESM(require("@ipld/dag-cbor"), 1);
|
1282
|
-
async function encodeCarFile(roots, t) {
|
1283
|
-
let size = 0;
|
1284
|
-
const headerSize = CBW.headerLength({ roots });
|
1285
|
-
size += headerSize;
|
1286
|
-
for (const { cid, bytes } of t.entries()) {
|
1287
|
-
size += CBW.blockLength({ cid, bytes });
|
1288
|
-
}
|
1289
|
-
const buffer = new Uint8Array(size);
|
1290
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
1291
|
-
for (const r of roots) {
|
1292
|
-
writer.addRoot(r);
|
1293
|
-
}
|
1294
|
-
for (const { cid, bytes } of t.entries()) {
|
1295
|
-
writer.write({ cid, bytes });
|
1296
|
-
}
|
1297
|
-
writer.close();
|
1298
|
-
return await (0, import_block2.encode)({ value: writer.bytes, hasher: import_sha2.sha256, codec: raw2 });
|
1299
|
-
}
|
1300
|
-
async function encodeCarHeader(fp) {
|
1301
|
-
return await (0, import_block2.encode)({
|
1302
|
-
value: { fp },
|
1303
|
-
hasher: import_sha2.sha256,
|
1304
|
-
codec
|
1305
|
-
});
|
1306
|
-
}
|
1147
|
+
var dagCodec = __toESM(require("@ipld/dag-cbor"), 1);
|
1307
1148
|
async function parseCarFile(reader, logger) {
|
1308
1149
|
const roots = await reader.getRoots();
|
1309
1150
|
const header = await reader.get(roots[0]);
|
1310
1151
|
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
1311
|
-
const
|
1312
|
-
const fpvalue = value;
|
1152
|
+
const dec = await decode({ bytes: header.bytes, hasher: import_sha2.sha256, codec: dagCodec });
|
1153
|
+
const fpvalue = dec.value;
|
1313
1154
|
if (fpvalue && !fpvalue.fp) {
|
1314
1155
|
throw logger.Error().Msg("missing fp").AsError();
|
1315
1156
|
}
|
1316
1157
|
return fpvalue.fp;
|
1317
1158
|
}
|
1318
1159
|
|
1319
|
-
// src/blockstore/encrypt-helpers.ts
|
1320
|
-
var import_sha22 = require("multiformats/hashes/sha2");
|
1321
|
-
var import_multiformats2 = require("multiformats");
|
1322
|
-
var import_block3 = require("multiformats/block");
|
1323
|
-
var dagcbor = __toESM(require("@ipld/dag-cbor"), 1);
|
1324
|
-
var import_block4 = require("@web3-storage/pail/block");
|
1325
|
-
var import_utils6 = require("prolly-trees/utils");
|
1326
|
-
var import_cache = require("prolly-trees/cache");
|
1327
|
-
var import_cid_set = require("prolly-trees/cid-set");
|
1328
|
-
|
1329
|
-
// src/blockstore/encrypt-codec.ts
|
1330
|
-
var import_multiformats = require("multiformats");
|
1331
|
-
init_utils();
|
1332
|
-
function makeCodec(ilogger, crypto2, randomBytes2) {
|
1333
|
-
const logger = ensureLogger(ilogger, "makeCodec");
|
1334
|
-
const enc32 = (value) => {
|
1335
|
-
value = +value;
|
1336
|
-
const buff = new Uint8Array(4);
|
1337
|
-
buff[3] = value >>> 24;
|
1338
|
-
buff[2] = value >>> 16;
|
1339
|
-
buff[1] = value >>> 8;
|
1340
|
-
buff[0] = value & 255;
|
1341
|
-
return buff;
|
1342
|
-
};
|
1343
|
-
const readUInt32LE = (buffer) => {
|
1344
|
-
const offset = buffer.byteLength - 4;
|
1345
|
-
return (buffer[offset] | buffer[offset + 1] << 8 | buffer[offset + 2] << 16) + buffer[offset + 3] * 16777216;
|
1346
|
-
};
|
1347
|
-
const concat = (buffers) => {
|
1348
|
-
const uint8Arrays = buffers.map((b) => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
|
1349
|
-
const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
|
1350
|
-
const result = new Uint8Array(totalLength);
|
1351
|
-
let offset = 0;
|
1352
|
-
for (const arr of uint8Arrays) {
|
1353
|
-
result.set(arr, offset);
|
1354
|
-
offset += arr.length;
|
1355
|
-
}
|
1356
|
-
return result;
|
1357
|
-
};
|
1358
|
-
const encode4 = ({ iv, bytes }) => concat([iv, bytes]);
|
1359
|
-
const decode4 = (bytes) => {
|
1360
|
-
const iv = bytes.subarray(0, 12);
|
1361
|
-
bytes = bytes.slice(12);
|
1362
|
-
return { iv, bytes };
|
1363
|
-
};
|
1364
|
-
const code = 3145728 + 1337;
|
1365
|
-
async function subtleKey(key) {
|
1366
|
-
return await crypto2.importKey(
|
1367
|
-
"raw",
|
1368
|
-
// raw or jwk
|
1369
|
-
key,
|
1370
|
-
// raw data
|
1371
|
-
"AES-GCM",
|
1372
|
-
false,
|
1373
|
-
// extractable
|
1374
|
-
["encrypt", "decrypt"]
|
1375
|
-
);
|
1376
|
-
}
|
1377
|
-
const decrypt = async ({ key, value }) => {
|
1378
|
-
const { bytes: inBytes, iv } = value;
|
1379
|
-
const cryKey = await subtleKey(key);
|
1380
|
-
const deBytes = await crypto2.decrypt(
|
1381
|
-
{
|
1382
|
-
name: "AES-GCM",
|
1383
|
-
iv,
|
1384
|
-
tagLength: 128
|
1385
|
-
},
|
1386
|
-
cryKey,
|
1387
|
-
inBytes
|
1388
|
-
);
|
1389
|
-
const bytes = new Uint8Array(deBytes);
|
1390
|
-
const len = readUInt32LE(bytes.subarray(0, 4));
|
1391
|
-
const cid = import_multiformats.CID.decode(bytes.subarray(4, 4 + len));
|
1392
|
-
return { cid, bytes: bytes.subarray(4 + len) };
|
1393
|
-
};
|
1394
|
-
const encrypt = async ({ key, cid, bytes }) => {
|
1395
|
-
const len = enc32(cid.bytes.byteLength);
|
1396
|
-
const iv = randomBytes2(12);
|
1397
|
-
const msg = concat([len, cid.bytes, bytes]);
|
1398
|
-
try {
|
1399
|
-
const cryKey = await subtleKey(key);
|
1400
|
-
const deBytes = await crypto2.encrypt(
|
1401
|
-
{
|
1402
|
-
name: "AES-GCM",
|
1403
|
-
iv,
|
1404
|
-
tagLength: 128
|
1405
|
-
},
|
1406
|
-
cryKey,
|
1407
|
-
msg
|
1408
|
-
);
|
1409
|
-
bytes = new Uint8Array(deBytes);
|
1410
|
-
} catch (e) {
|
1411
|
-
throw logger.Error().Err(e).Msg("encrypt failed").AsError();
|
1412
|
-
}
|
1413
|
-
return { value: { bytes, iv } };
|
1414
|
-
};
|
1415
|
-
const cryptoFn = (key) => {
|
1416
|
-
return { encrypt: (opts) => encrypt({ ...opts, key }), decrypt: (opts) => decrypt({ ...opts, key }) };
|
1417
|
-
};
|
1418
|
-
const name = "jchris@encrypted-block:aes-gcm";
|
1419
|
-
return { encode: encode4, decode: decode4, code, name, encrypt, decrypt, crypto: cryptoFn };
|
1420
|
-
}
|
1421
|
-
|
1422
|
-
// src/blockstore/encrypt-helpers.ts
|
1423
|
-
function carLogIncludesGroup(list, cidMatch) {
|
1424
|
-
return list.some((cid) => {
|
1425
|
-
return cid.toString() === cidMatch.toString();
|
1426
|
-
});
|
1427
|
-
}
|
1428
|
-
function makeEncDec(logger, crypto2, randomBytes2) {
|
1429
|
-
const codec4 = makeCodec(logger, crypto2, randomBytes2);
|
1430
|
-
const encrypt = async function* ({
|
1431
|
-
get: get2,
|
1432
|
-
cids,
|
1433
|
-
hasher: hasher4,
|
1434
|
-
key,
|
1435
|
-
cache: cache3,
|
1436
|
-
chunker: chunker2,
|
1437
|
-
root: root3
|
1438
|
-
}) {
|
1439
|
-
const set = /* @__PURE__ */ new Set();
|
1440
|
-
let eroot;
|
1441
|
-
if (!carLogIncludesGroup(cids, root3)) cids.push(root3);
|
1442
|
-
for (const cid of cids) {
|
1443
|
-
const unencrypted = await get2(cid);
|
1444
|
-
if (!unencrypted) throw logger.Error().Ref("cid", cid).Msg("missing cid block").AsError();
|
1445
|
-
const encrypted = await codec4.encrypt({ ...unencrypted, key });
|
1446
|
-
const block2 = await (0, import_block3.encode)({ ...encrypted, codec: codec4, hasher: hasher4 });
|
1447
|
-
yield block2;
|
1448
|
-
set.add(block2.cid.toString());
|
1449
|
-
if (unencrypted.cid.equals(root3)) eroot = block2.cid;
|
1450
|
-
}
|
1451
|
-
if (!eroot) throw logger.Error().Msg("cids does not include root").AsError();
|
1452
|
-
const list = [...set].map((s) => import_multiformats2.CID.parse(s));
|
1453
|
-
let last;
|
1454
|
-
for await (const node of (0, import_cid_set.create)({ list, get: get2, cache: cache3, chunker: chunker2, hasher: hasher4, codec: dagcbor })) {
|
1455
|
-
const block2 = await node.block;
|
1456
|
-
yield block2;
|
1457
|
-
last = block2;
|
1458
|
-
}
|
1459
|
-
if (!last) throw logger.Error().Msg("missing last block").AsError();
|
1460
|
-
const head = [eroot, last.cid];
|
1461
|
-
const block = await (0, import_block3.encode)({ value: head, codec: dagcbor, hasher: hasher4 });
|
1462
|
-
yield block;
|
1463
|
-
};
|
1464
|
-
const decrypt = async function* ({
|
1465
|
-
root: root3,
|
1466
|
-
get: get2,
|
1467
|
-
key,
|
1468
|
-
cache: cache3,
|
1469
|
-
chunker: chunker2,
|
1470
|
-
hasher: hasher4
|
1471
|
-
}) {
|
1472
|
-
const getWithDecode = async (cid) => get2(cid).then(async (block) => {
|
1473
|
-
if (!block) return;
|
1474
|
-
const decoded = await (0, import_block3.decode)({ ...block, codec: dagcbor, hasher: hasher4 });
|
1475
|
-
return decoded;
|
1476
|
-
});
|
1477
|
-
const getWithDecrypt = async (cid) => get2(cid).then(async (block) => {
|
1478
|
-
if (!block) return;
|
1479
|
-
const decoded = await (0, import_block3.decode)({ ...block, codec: codec4, hasher: hasher4 });
|
1480
|
-
return decoded;
|
1481
|
-
});
|
1482
|
-
const decodedRoot = await getWithDecode(root3);
|
1483
|
-
if (!decodedRoot) throw logger.Error().Msg("missing root").AsError();
|
1484
|
-
if (!decodedRoot.bytes) throw logger.Error().Msg("missing bytes").AsError();
|
1485
|
-
const {
|
1486
|
-
value: [eroot, tree]
|
1487
|
-
} = decodedRoot;
|
1488
|
-
const rootBlock = await get2(eroot);
|
1489
|
-
if (!rootBlock) throw logger.Error().Msg("missing root block").AsError();
|
1490
|
-
const cidset = await (0, import_cid_set.load)({ cid: tree, get: getWithDecode, cache: cache3, chunker: chunker2, codec: codec4, hasher: hasher4 });
|
1491
|
-
const { result: nodes } = await cidset.getAllEntries();
|
1492
|
-
const unwrap = async (eblock) => {
|
1493
|
-
if (!eblock) throw logger.Error().Msg("missing block").AsError();
|
1494
|
-
if (!eblock.value) {
|
1495
|
-
eblock = await (0, import_block3.decode)({ ...eblock, codec: codec4, hasher: hasher4 });
|
1496
|
-
if (!eblock.value) throw logger.Error().Msg("missing value").AsError();
|
1497
|
-
}
|
1498
|
-
const { bytes, cid } = await codec4.decrypt({ ...eblock, key }).catch((e) => {
|
1499
|
-
throw e;
|
1500
|
-
});
|
1501
|
-
const block = await (0, import_block3.create)({ cid, bytes, hasher: hasher4, codec: codec4 });
|
1502
|
-
return block;
|
1503
|
-
};
|
1504
|
-
const promises = [];
|
1505
|
-
for (const { cid } of nodes) {
|
1506
|
-
if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap));
|
1507
|
-
}
|
1508
|
-
yield* promises;
|
1509
|
-
yield unwrap(rootBlock);
|
1510
|
-
};
|
1511
|
-
return { encrypt, decrypt };
|
1512
|
-
}
|
1513
|
-
var chunker = (0, import_utils6.bf)(30);
|
1514
|
-
function hexStringToUint8Array(hexString) {
|
1515
|
-
const length = hexString.length;
|
1516
|
-
const uint8Array = new Uint8Array(length / 2);
|
1517
|
-
for (let i = 0; i < length; i += 2) {
|
1518
|
-
uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
|
1519
|
-
}
|
1520
|
-
return uint8Array;
|
1521
|
-
}
|
1522
|
-
async function encryptedEncodeCarFile(logger, crypto2, key, rootCid, t) {
|
1523
|
-
const encryptionKey = hexStringToUint8Array(key);
|
1524
|
-
const encryptedBlocks = new import_block4.MemoryBlockstore();
|
1525
|
-
const cidsToEncrypt = [];
|
1526
|
-
for (const { cid, bytes } of t.entries()) {
|
1527
|
-
cidsToEncrypt.push(cid);
|
1528
|
-
const g = await t.get(cid);
|
1529
|
-
if (!g) throw logger.Error().Ref("cid", cid).Int("bytes", bytes.length).Msg("missing cid block").AsError();
|
1530
|
-
}
|
1531
|
-
let last = null;
|
1532
|
-
const { encrypt } = makeEncDec(logger, crypto2, crypto2.randomBytes);
|
1533
|
-
for await (const block of encrypt({
|
1534
|
-
cids: cidsToEncrypt,
|
1535
|
-
get: t.get.bind(t),
|
1536
|
-
key: encryptionKey,
|
1537
|
-
hasher: import_sha22.sha256,
|
1538
|
-
chunker,
|
1539
|
-
cache: import_cache.nocache,
|
1540
|
-
root: rootCid
|
1541
|
-
})) {
|
1542
|
-
await encryptedBlocks.put(block.cid, block.bytes);
|
1543
|
-
last = block;
|
1544
|
-
}
|
1545
|
-
if (!last) throw logger.Error().Msg("no blocks encrypted").AsError();
|
1546
|
-
const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
|
1547
|
-
return encryptedCar;
|
1548
|
-
}
|
1549
|
-
async function decodeEncryptedCar(logger, crypto2, key, reader) {
|
1550
|
-
const roots = await reader.getRoots();
|
1551
|
-
const root3 = roots[0];
|
1552
|
-
return await decodeCarBlocks(logger, crypto2, root3, reader.get.bind(reader), key);
|
1553
|
-
}
|
1554
|
-
async function decodeCarBlocks(logger, crypto2, root3, get2, keyMaterial) {
|
1555
|
-
const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
|
1556
|
-
const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength);
|
1557
|
-
const decryptedBlocks = new import_block4.MemoryBlockstore();
|
1558
|
-
let last = null;
|
1559
|
-
const { decrypt } = makeEncDec(logger, crypto2, crypto2.randomBytes);
|
1560
|
-
for await (const block of decrypt({
|
1561
|
-
root: root3,
|
1562
|
-
get: get2,
|
1563
|
-
key: decryptionKey,
|
1564
|
-
hasher: import_sha22.sha256,
|
1565
|
-
chunker,
|
1566
|
-
cache: import_cache.nocache
|
1567
|
-
})) {
|
1568
|
-
await decryptedBlocks.put(block.cid, block.bytes);
|
1569
|
-
last = block;
|
1570
|
-
}
|
1571
|
-
if (!last) throw logger.Error().Msg("no blocks decrypted").AsError();
|
1572
|
-
return { blocks: decryptedBlocks, root: last.cid };
|
1573
|
-
}
|
1574
|
-
|
1575
1160
|
// src/blockstore/transaction.ts
|
1576
|
-
var
|
1577
|
-
|
1578
|
-
init_crypto();
|
1161
|
+
var import_block3 = require("@web3-storage/pail/block");
|
1162
|
+
var import_cement2 = require("@adviser/cement");
|
1579
1163
|
init_utils();
|
1580
|
-
var CarTransaction = class extends
|
1581
|
-
constructor(parent, opts = { add: true }) {
|
1164
|
+
var CarTransaction = class extends import_block3.MemoryBlockstore {
|
1165
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
1582
1166
|
super();
|
1583
1167
|
if (opts.add) {
|
1584
1168
|
parent.transactions.add(this);
|
@@ -1592,8 +1176,8 @@ var CarTransaction = class extends import_block5.MemoryBlockstore {
|
|
1592
1176
|
return super.get(cid);
|
1593
1177
|
}
|
1594
1178
|
};
|
1595
|
-
function defaultedBlockstoreRuntime(opts, component, ctx) {
|
1596
|
-
const logger = ensureLogger(
|
1179
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
1180
|
+
const logger = ensureLogger(sthis, component, ctx);
|
1597
1181
|
const store = opts.store || {};
|
1598
1182
|
return {
|
1599
1183
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
@@ -1610,22 +1194,24 @@ function defaultedBlockstoreRuntime(opts, component, ctx) {
|
|
1610
1194
|
threshold: 1e3 * 1e3,
|
1611
1195
|
...opts,
|
1612
1196
|
logger,
|
1613
|
-
|
1197
|
+
keyBag: opts.keyBag || {},
|
1198
|
+
crypto: (0, import_cement2.toCryptoRuntime)(opts.crypto),
|
1614
1199
|
store,
|
1615
|
-
storeRuntime: toStoreRuntime(store,
|
1200
|
+
storeRuntime: toStoreRuntime(store, sthis)
|
1616
1201
|
};
|
1617
1202
|
}
|
1618
|
-
|
1203
|
+
function blockstoreFactory(sthis, opts) {
|
1619
1204
|
if (opts.name) {
|
1620
|
-
return new EncryptedBlockstore(opts);
|
1205
|
+
return new EncryptedBlockstore(sthis, opts);
|
1621
1206
|
} else {
|
1622
1207
|
return new BaseBlockstore(opts);
|
1623
1208
|
}
|
1624
|
-
}
|
1209
|
+
}
|
1625
1210
|
var BaseBlockstore = class {
|
1626
1211
|
constructor(ebOpts = {}) {
|
1627
1212
|
this.transactions = /* @__PURE__ */ new Set();
|
1628
|
-
this.
|
1213
|
+
this.sthis = ensureSuperThis(ebOpts);
|
1214
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
1629
1215
|
this.logger = this.ebOpts.logger;
|
1630
1216
|
}
|
1631
1217
|
// ready: Promise<void>;
|
@@ -1648,8 +1234,8 @@ var BaseBlockstore = class {
|
|
1648
1234
|
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
1649
1235
|
}
|
1650
1236
|
// TransactionMeta
|
1651
|
-
async transaction(fn, _opts
|
1652
|
-
const t = new CarTransaction(this);
|
1237
|
+
async transaction(fn, _opts) {
|
1238
|
+
const t = new CarTransaction(this, _opts);
|
1653
1239
|
const done = await fn(t);
|
1654
1240
|
this.lastTxMeta = done;
|
1655
1241
|
return { t, meta: done };
|
@@ -1666,16 +1252,16 @@ var BaseBlockstore = class {
|
|
1666
1252
|
}
|
1667
1253
|
};
|
1668
1254
|
var EncryptedBlockstore = class extends BaseBlockstore {
|
1669
|
-
constructor(ebOpts) {
|
1255
|
+
constructor(sthis, ebOpts) {
|
1670
1256
|
super(ebOpts);
|
1671
1257
|
this.compacting = false;
|
1672
|
-
this.logger = ensureLogger(
|
1258
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
1673
1259
|
const { name } = ebOpts;
|
1674
1260
|
if (!name) {
|
1675
1261
|
throw this.logger.Error().Msg("name required").AsError();
|
1676
1262
|
}
|
1677
1263
|
this.name = name;
|
1678
|
-
this.loader = new Loader(this.name, ebOpts);
|
1264
|
+
this.loader = new Loader(this.name, ebOpts, sthis);
|
1679
1265
|
}
|
1680
1266
|
ready() {
|
1681
1267
|
return this.loader.ready();
|
@@ -1706,10 +1292,13 @@ var EncryptedBlockstore = class extends BaseBlockstore {
|
|
1706
1292
|
}
|
1707
1293
|
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
1708
1294
|
}
|
1709
|
-
async getFile(car, cid
|
1295
|
+
async getFile(car, cid) {
|
1710
1296
|
await this.ready();
|
1711
1297
|
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
1712
|
-
const reader = await this.loader.loadFileCar(
|
1298
|
+
const reader = await this.loader.loadFileCar(
|
1299
|
+
car
|
1300
|
+
/*, isPublic */
|
1301
|
+
);
|
1713
1302
|
const block = await reader.get(cid);
|
1714
1303
|
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
1715
1304
|
return block.bytes;
|
@@ -1765,43 +1354,345 @@ var CompactionFetcher = class {
|
|
1765
1354
|
};
|
1766
1355
|
|
1767
1356
|
// src/blockstore/commit-queue.ts
|
1357
|
+
var import_cement3 = require("@adviser/cement");
|
1768
1358
|
var CommitQueue = class {
|
1769
1359
|
constructor() {
|
1770
1360
|
this.queue = [];
|
1771
1361
|
this.processing = false;
|
1362
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
1363
|
+
}
|
1364
|
+
waitIdle() {
|
1365
|
+
if (this.queue.length === 0 && !this.processing) {
|
1366
|
+
return Promise.resolve();
|
1367
|
+
}
|
1368
|
+
const fn = new import_cement3.Future();
|
1369
|
+
this._waitIdleItems.add(fn);
|
1370
|
+
return fn.asPromise();
|
1371
|
+
}
|
1372
|
+
async enqueue(fn) {
|
1373
|
+
return new Promise((resolve, reject) => {
|
1374
|
+
const queueFn = async () => {
|
1375
|
+
try {
|
1376
|
+
resolve(await fn());
|
1377
|
+
} catch (e) {
|
1378
|
+
reject(e);
|
1379
|
+
} finally {
|
1380
|
+
this.processing = false;
|
1381
|
+
this.processNext();
|
1382
|
+
}
|
1383
|
+
};
|
1384
|
+
this.queue.push(queueFn);
|
1385
|
+
if (!this.processing) {
|
1386
|
+
this.processNext();
|
1387
|
+
}
|
1388
|
+
});
|
1389
|
+
}
|
1390
|
+
processNext() {
|
1391
|
+
if (this.queue.length > 0 && !this.processing) {
|
1392
|
+
this.processing = true;
|
1393
|
+
const queueFn = this.queue.shift();
|
1394
|
+
if (queueFn) {
|
1395
|
+
queueFn().finally(() => {
|
1396
|
+
});
|
1397
|
+
}
|
1398
|
+
}
|
1399
|
+
if (this.queue.length === 0 && !this.processing) {
|
1400
|
+
const toResolve = Array.from(this._waitIdleItems);
|
1401
|
+
this._waitIdleItems.clear();
|
1402
|
+
toResolve.map((fn) => fn.resolve());
|
1403
|
+
}
|
1404
|
+
}
|
1405
|
+
};
|
1406
|
+
|
1407
|
+
// src/runtime/key-bag.ts
|
1408
|
+
var key_bag_exports = {};
|
1409
|
+
__export(key_bag_exports, {
|
1410
|
+
KeyBag: () => KeyBag,
|
1411
|
+
getKeyBag: () => getKeyBag,
|
1412
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
1413
|
+
});
|
1414
|
+
var import_cement5 = require("@adviser/cement");
|
1415
|
+
init_utils();
|
1416
|
+
var import_base582 = require("multiformats/bases/base58");
|
1417
|
+
var KeyBag = class {
|
1418
|
+
constructor(rt) {
|
1419
|
+
this.rt = rt;
|
1420
|
+
this._seq = new import_cement5.ResolveSeq();
|
1421
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag", {
|
1422
|
+
id: rt.id()
|
1423
|
+
});
|
1424
|
+
this.logger.Debug().Msg("KeyBag created");
|
1425
|
+
}
|
1426
|
+
async subtleKey(key) {
|
1427
|
+
return await this.rt.crypto.importKey(
|
1428
|
+
"raw",
|
1429
|
+
// raw or jwk
|
1430
|
+
import_base582.base58btc.decode(key),
|
1431
|
+
// hexStringToUint8Array(key), // raw data
|
1432
|
+
"AES-GCM",
|
1433
|
+
false,
|
1434
|
+
// extractable
|
1435
|
+
["encrypt", "decrypt"]
|
1436
|
+
);
|
1437
|
+
}
|
1438
|
+
async ensureKeyFromUrl(url, keyFactory) {
|
1439
|
+
const storeKey = url.getParam("storekey");
|
1440
|
+
if (storeKey === "insecure") {
|
1441
|
+
return import_cement5.Result.Ok(url);
|
1442
|
+
}
|
1443
|
+
if (!storeKey) {
|
1444
|
+
const keyName = `@${keyFactory()}@`;
|
1445
|
+
const ret = await this.getNamedKey(keyName);
|
1446
|
+
if (ret.isErr()) {
|
1447
|
+
return ret;
|
1448
|
+
}
|
1449
|
+
const urb = url.build().setParam("storekey", keyName);
|
1450
|
+
return import_cement5.Result.Ok(urb.URI());
|
1451
|
+
}
|
1452
|
+
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
1453
|
+
const ret = await this.getNamedKey(storeKey);
|
1454
|
+
if (ret.isErr()) {
|
1455
|
+
return ret;
|
1456
|
+
}
|
1457
|
+
}
|
1458
|
+
return import_cement5.Result.Ok(url);
|
1459
|
+
}
|
1460
|
+
async toKeyWithFingerPrint(keyStr) {
|
1461
|
+
const material = import_base582.base58btc.decode(keyStr);
|
1462
|
+
const key = await this.subtleKey(keyStr);
|
1463
|
+
const fpr = await this.rt.crypto.digestSHA256(material);
|
1464
|
+
return import_cement5.Result.Ok({
|
1465
|
+
key,
|
1466
|
+
fingerPrint: import_base582.base58btc.encode(new Uint8Array(fpr))
|
1467
|
+
});
|
1468
|
+
}
|
1469
|
+
async setNamedKey(name, key) {
|
1470
|
+
return this._seq.add(() => this._setNamedKey(name, key));
|
1471
|
+
}
|
1472
|
+
// avoid deadlock
|
1473
|
+
async _setNamedKey(name, key) {
|
1474
|
+
const item = {
|
1475
|
+
name,
|
1476
|
+
key
|
1477
|
+
};
|
1478
|
+
const bag = await this.rt.getBag();
|
1479
|
+
this.logger.Debug().Str("name", name).Msg("setNamedKey");
|
1480
|
+
await bag.set(name, item);
|
1481
|
+
return await this.toKeyWithFingerPrint(item.key);
|
1482
|
+
}
|
1483
|
+
async getNamedKey(name, failIfNotFound = false) {
|
1484
|
+
const id = this.rt.sthis.nextId(4).str;
|
1485
|
+
return this._seq.add(async () => {
|
1486
|
+
const bag = await this.rt.getBag();
|
1487
|
+
const named = await bag.get(name);
|
1488
|
+
if (named) {
|
1489
|
+
const fpr = await this.toKeyWithFingerPrint(named.key);
|
1490
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
|
1491
|
+
return fpr;
|
1492
|
+
}
|
1493
|
+
if (failIfNotFound) {
|
1494
|
+
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
1495
|
+
return import_cement5.Result.Err(new Error(`Key not found: ${name}`));
|
1496
|
+
}
|
1497
|
+
const ret = await this._setNamedKey(name, import_base582.base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
1498
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
1499
|
+
return ret;
|
1500
|
+
});
|
1501
|
+
}
|
1502
|
+
};
|
1503
|
+
var keyBagProviderFactories = new Map(
|
1504
|
+
[
|
1505
|
+
{
|
1506
|
+
protocol: "file:",
|
1507
|
+
factory: async (url, sthis) => {
|
1508
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1509
|
+
return new KeyBagProviderFile2(url, sthis);
|
1510
|
+
}
|
1511
|
+
},
|
1512
|
+
{
|
1513
|
+
protocol: "indexdb:",
|
1514
|
+
factory: async (url, sthis) => {
|
1515
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1516
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
1517
|
+
}
|
1518
|
+
}
|
1519
|
+
].map((i) => [i.protocol, i])
|
1520
|
+
);
|
1521
|
+
function registerKeyBagProviderFactory(item) {
|
1522
|
+
const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
|
1523
|
+
keyBagProviderFactories.set(protocol, {
|
1524
|
+
...item,
|
1525
|
+
protocol
|
1526
|
+
});
|
1527
|
+
}
|
1528
|
+
function defaultKeyBagOpts(sthis, kbo) {
|
1529
|
+
if (kbo.keyRuntime) {
|
1530
|
+
return kbo.keyRuntime;
|
1531
|
+
}
|
1532
|
+
const logger = ensureLogger(sthis, "KeyBag");
|
1533
|
+
let url;
|
1534
|
+
if (kbo.url) {
|
1535
|
+
url = import_cement5.URI.from(kbo.url);
|
1536
|
+
logger.Debug().Url(url).Msg("from opts");
|
1537
|
+
} else {
|
1538
|
+
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
1539
|
+
if ((0, import_cement5.runtimeFn)().isBrowser) {
|
1540
|
+
url = import_cement5.URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
1541
|
+
} else {
|
1542
|
+
if (!bagFnameOrUrl) {
|
1543
|
+
const home = sthis.env.get("HOME");
|
1544
|
+
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
1545
|
+
url = import_cement5.URI.from(`file://${bagFnameOrUrl}`);
|
1546
|
+
} else {
|
1547
|
+
url = import_cement5.URI.from(bagFnameOrUrl);
|
1548
|
+
}
|
1549
|
+
}
|
1550
|
+
logger.Debug().Url(url).Msg("from env");
|
1551
|
+
}
|
1552
|
+
let keyProviderFactory;
|
1553
|
+
switch (url.protocol) {
|
1554
|
+
case "file:":
|
1555
|
+
keyProviderFactory = async () => {
|
1556
|
+
const { KeyBagProviderFile: KeyBagProviderFile2 } = await Promise.resolve().then(() => (init_key_bag_file(), key_bag_file_exports));
|
1557
|
+
return new KeyBagProviderFile2(url, sthis);
|
1558
|
+
};
|
1559
|
+
break;
|
1560
|
+
case "indexdb:":
|
1561
|
+
keyProviderFactory = async () => {
|
1562
|
+
const { KeyBagProviderIndexDB: KeyBagProviderIndexDB2 } = await Promise.resolve().then(() => (init_key_bag_indexdb(), key_bag_indexdb_exports));
|
1563
|
+
return new KeyBagProviderIndexDB2(url, sthis);
|
1564
|
+
};
|
1565
|
+
break;
|
1566
|
+
default:
|
1567
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
1568
|
+
}
|
1569
|
+
if (url.hasParam("masterkey")) {
|
1570
|
+
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
1571
|
+
}
|
1572
|
+
return {
|
1573
|
+
url,
|
1574
|
+
crypto: kbo.crypto || (0, import_cement5.toCryptoRuntime)({}),
|
1575
|
+
sthis,
|
1576
|
+
logger,
|
1577
|
+
keyLength: kbo.keyLength || 16,
|
1578
|
+
getBag: keyProviderFactory,
|
1579
|
+
id: () => {
|
1580
|
+
return url.toString();
|
1581
|
+
}
|
1582
|
+
};
|
1583
|
+
}
|
1584
|
+
var _keyBags = new import_cement5.KeyedResolvOnce();
|
1585
|
+
async function getKeyBag(sthis, kbo = {}) {
|
1586
|
+
await sthis.start();
|
1587
|
+
const rt = defaultKeyBagOpts(sthis, kbo);
|
1588
|
+
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
1589
|
+
}
|
1590
|
+
|
1591
|
+
// src/blockstore/commitor.ts
|
1592
|
+
var CBW = __toESM(require("@ipld/car/buffer-writer"), 1);
|
1593
|
+
var import_sha22 = require("multiformats/hashes/sha2");
|
1594
|
+
var dagCodec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
1595
|
+
async function encodeCarFile(roots, t, codec3) {
|
1596
|
+
let size = 0;
|
1597
|
+
const headerSize = CBW.headerLength({ roots });
|
1598
|
+
size += headerSize;
|
1599
|
+
for (const { cid, bytes } of t.entries()) {
|
1600
|
+
size += CBW.blockLength({ cid, bytes });
|
1601
|
+
}
|
1602
|
+
const buffer = new Uint8Array(size);
|
1603
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
1604
|
+
for (const r of roots) {
|
1605
|
+
writer.addRoot(r);
|
1772
1606
|
}
|
1773
|
-
|
1774
|
-
|
1775
|
-
const queueFn = async () => {
|
1776
|
-
try {
|
1777
|
-
resolve(await fn());
|
1778
|
-
} catch (e) {
|
1779
|
-
reject(e);
|
1780
|
-
} finally {
|
1781
|
-
this.processing = false;
|
1782
|
-
this.processNext();
|
1783
|
-
}
|
1784
|
-
};
|
1785
|
-
this.queue.push(queueFn);
|
1786
|
-
if (!this.processing) {
|
1787
|
-
this.processNext();
|
1788
|
-
}
|
1789
|
-
});
|
1607
|
+
for (const { cid, bytes } of t.entries()) {
|
1608
|
+
writer.write({ cid, bytes });
|
1790
1609
|
}
|
1791
|
-
|
1792
|
-
|
1793
|
-
|
1794
|
-
|
1795
|
-
|
1796
|
-
|
1797
|
-
|
1610
|
+
writer.close();
|
1611
|
+
return await encode({ value: writer.bytes, hasher: import_sha22.sha256, codec: codec3 });
|
1612
|
+
}
|
1613
|
+
async function createCarFile(encoder, cid, t) {
|
1614
|
+
return encodeCarFile([cid], t, encoder);
|
1615
|
+
}
|
1616
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
1617
|
+
const { files: roots } = makeFileCarHeader(done);
|
1618
|
+
const cids = [];
|
1619
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
1620
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
1621
|
+
for (const car of cars) {
|
1622
|
+
const { cid, bytes } = car;
|
1623
|
+
await fileStore.save({ cid, bytes });
|
1624
|
+
await walStore.enqueueFile(
|
1625
|
+
cid
|
1626
|
+
/*, !!opts.public*/
|
1627
|
+
);
|
1628
|
+
cids.push(cid);
|
1629
|
+
}
|
1630
|
+
return cids;
|
1631
|
+
}
|
1632
|
+
function makeFileCarHeader(result) {
|
1633
|
+
const files = [];
|
1634
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
1635
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1636
|
+
files.push(meta.cid);
|
1798
1637
|
}
|
1799
1638
|
}
|
1800
|
-
};
|
1639
|
+
return { ...result, files };
|
1640
|
+
}
|
1641
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
1642
|
+
return [await encodeCarFile(roots, t, encoder)];
|
1643
|
+
}
|
1644
|
+
function makeCarHeader(meta, cars, compact = false) {
|
1645
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1646
|
+
return { ...coreHeader, meta };
|
1647
|
+
}
|
1648
|
+
async function encodeCarHeader(fp) {
|
1649
|
+
return await encode({
|
1650
|
+
value: { fp },
|
1651
|
+
hasher: import_sha22.sha256,
|
1652
|
+
codec: dagCodec2
|
1653
|
+
});
|
1654
|
+
}
|
1655
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
1656
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
1657
|
+
const rootBlock = await encodeCarHeader(fp);
|
1658
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
1659
|
+
const cids = [];
|
1660
|
+
for (const car of cars) {
|
1661
|
+
const { cid, bytes } = car;
|
1662
|
+
await params.carStore.save({ cid, bytes });
|
1663
|
+
cids.push(cid);
|
1664
|
+
}
|
1665
|
+
const newDbMeta = { cars: cids };
|
1666
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
1667
|
+
await params.metaStore.save(newDbMeta);
|
1668
|
+
return { cgrp: cids, header: fp };
|
1669
|
+
}
|
1670
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
1671
|
+
const carFiles = [];
|
1672
|
+
threshold = threshold || 1e3 * 1e3;
|
1673
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1674
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1675
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
1676
|
+
let cidRootBlock = rootBlock;
|
1677
|
+
for (const { cid, bytes } of t.entries()) {
|
1678
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1679
|
+
if (newsize >= threshold) {
|
1680
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1681
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
1682
|
+
clonedt.putSync(cid, bytes);
|
1683
|
+
cidRootBlock = { cid, bytes };
|
1684
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
1685
|
+
} else {
|
1686
|
+
clonedt.putSync(cid, bytes);
|
1687
|
+
}
|
1688
|
+
}
|
1689
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
1690
|
+
return carFiles;
|
1691
|
+
}
|
1801
1692
|
|
1802
1693
|
// src/blockstore/loader.ts
|
1803
|
-
var
|
1804
|
-
function
|
1694
|
+
var import_sha23 = require("multiformats/hashes/sha2");
|
1695
|
+
function carLogIncludesGroup(list, cids) {
|
1805
1696
|
return list.some((arr) => {
|
1806
1697
|
return arr.toString() === cids.toString();
|
1807
1698
|
});
|
@@ -1814,17 +1705,8 @@ function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
|
1814
1705
|
}
|
1815
1706
|
return [...byString.values()];
|
1816
1707
|
}
|
1817
|
-
function toHexString(byteArray) {
|
1818
|
-
return Array.from(byteArray).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
1819
|
-
}
|
1820
|
-
var Loadable = class {
|
1821
|
-
constructor() {
|
1822
|
-
this.name = "";
|
1823
|
-
this.carLog = new Array();
|
1824
|
-
}
|
1825
|
-
};
|
1826
1708
|
var Loader = class {
|
1827
|
-
constructor(name, ebOpts) {
|
1709
|
+
constructor(name, ebOpts, sthis) {
|
1828
1710
|
this.commitQueue = new CommitQueue();
|
1829
1711
|
this.isCompacting = false;
|
1830
1712
|
this.carReaders = /* @__PURE__ */ new Map();
|
@@ -1834,9 +1716,11 @@ var Loader = class {
|
|
1834
1716
|
this.getBlockCache = /* @__PURE__ */ new Map();
|
1835
1717
|
this.seenMeta = /* @__PURE__ */ new Set();
|
1836
1718
|
this.writeLimit = (0, import_p_limit.default)(1);
|
1837
|
-
this.onceReady = new
|
1719
|
+
this.onceReady = new import_cement6.ResolveOnce();
|
1838
1720
|
this.name = name;
|
1721
|
+
this.sthis = sthis;
|
1839
1722
|
this.ebOpts = defaultedBlockstoreRuntime(
|
1723
|
+
sthis,
|
1840
1724
|
{
|
1841
1725
|
...ebOpts,
|
1842
1726
|
name
|
@@ -1846,14 +1730,17 @@ var Loader = class {
|
|
1846
1730
|
this.logger = this.ebOpts.logger;
|
1847
1731
|
}
|
1848
1732
|
// readonly id = uuidv4();
|
1733
|
+
async keyBag() {
|
1734
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
1735
|
+
}
|
1849
1736
|
async carStore() {
|
1850
1737
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1851
1738
|
}
|
1852
1739
|
async fileStore() {
|
1853
1740
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
1854
1741
|
}
|
1855
|
-
async
|
1856
|
-
return this.ebOpts.storeRuntime.
|
1742
|
+
async WALStore() {
|
1743
|
+
return this.ebOpts.storeRuntime.makeWALStore(this);
|
1857
1744
|
}
|
1858
1745
|
async metaStore() {
|
1859
1746
|
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
@@ -1867,11 +1754,11 @@ var Loader = class {
|
|
1867
1754
|
});
|
1868
1755
|
}
|
1869
1756
|
async close() {
|
1870
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
1757
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1871
1758
|
await Promise.all(toClose.map((store) => store.close()));
|
1872
1759
|
}
|
1873
1760
|
async destroy() {
|
1874
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
1761
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
1875
1762
|
await Promise.all(toDestroy.map((store) => store.destroy()));
|
1876
1763
|
}
|
1877
1764
|
// async snapToCar(carCid: AnyLink | string) {
|
@@ -1885,6 +1772,7 @@ var Loader = class {
|
|
1885
1772
|
// await this._applyCarHeader(carHeader, true)
|
1886
1773
|
// }
|
1887
1774
|
async handleDbMetasFromStore(metas) {
|
1775
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
1888
1776
|
for (const meta of metas) {
|
1889
1777
|
await this.writeLimit(async () => {
|
1890
1778
|
await this.mergeDbMetaIntoClock(meta);
|
@@ -1897,10 +1785,7 @@ var Loader = class {
|
|
1897
1785
|
}
|
1898
1786
|
if (this.seenMeta.has(meta.cars.toString())) return;
|
1899
1787
|
this.seenMeta.add(meta.cars.toString());
|
1900
|
-
if (meta.
|
1901
|
-
await this.setKey(meta.key);
|
1902
|
-
}
|
1903
|
-
if (carLogIncludesGroup2(this.carLog, meta.cars)) {
|
1788
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
1904
1789
|
return;
|
1905
1790
|
}
|
1906
1791
|
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
@@ -1909,45 +1794,59 @@ var Loader = class {
|
|
1909
1794
|
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
1910
1795
|
await this.ebOpts.applyMeta?.(carHeader.meta);
|
1911
1796
|
}
|
1912
|
-
async ingestKeyFromMeta(meta) {
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
}
|
1797
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1798
|
+
// const { key } = meta;
|
1799
|
+
// if (key) {
|
1800
|
+
// await this.setKey(key);
|
1801
|
+
// }
|
1802
|
+
// }
|
1918
1803
|
async loadCarHeaderFromMeta({ cars: cids }) {
|
1919
1804
|
const reader = await this.loadCar(cids[0]);
|
1920
1805
|
return await parseCarFile(reader, this.logger);
|
1921
1806
|
}
|
1922
|
-
async _getKey() {
|
1923
|
-
|
1924
|
-
|
1925
|
-
|
1926
|
-
|
1927
|
-
|
1928
|
-
|
1929
|
-
|
1930
|
-
|
1931
|
-
}
|
1932
|
-
// can these skip the queue? or have a file queue?
|
1933
|
-
async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
|
1807
|
+
// async _getKey(): Promise<string | undefined> {
|
1808
|
+
// if (this.key) return this.key;
|
1809
|
+
// // generate a random key
|
1810
|
+
// if (!this.ebOpts.public) {
|
1811
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1812
|
+
// }
|
1813
|
+
// return this.key || undefined;
|
1814
|
+
// }
|
1815
|
+
async commitFiles(t, done) {
|
1934
1816
|
await this.ready();
|
1935
|
-
const
|
1936
|
-
const
|
1937
|
-
|
1938
|
-
for (const car of cars) {
|
1939
|
-
const { cid, bytes } = car;
|
1940
|
-
await (await this.fileStore()).save({ cid, bytes });
|
1941
|
-
await (await this.remoteWAL()).enqueueFile(cid, !!opts.public);
|
1942
|
-
cids.push(cid);
|
1943
|
-
}
|
1944
|
-
return cids;
|
1817
|
+
const fstore = await this.fileStore();
|
1818
|
+
const wstore = await this.WALStore();
|
1819
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
1945
1820
|
}
|
1946
|
-
async loadFileCar(cid
|
1947
|
-
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore
|
1821
|
+
async loadFileCar(cid) {
|
1822
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
1948
1823
|
}
|
1949
1824
|
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
1950
|
-
|
1825
|
+
await this.ready();
|
1826
|
+
const fstore = await this.fileStore();
|
1827
|
+
const params = {
|
1828
|
+
encoder: (await fstore.keyedCrypto()).codec(),
|
1829
|
+
carLog: this.carLog,
|
1830
|
+
carStore: fstore,
|
1831
|
+
WALStore: await this.WALStore(),
|
1832
|
+
metaStore: await this.metaStore()
|
1833
|
+
};
|
1834
|
+
return this.commitQueue.enqueue(async () => {
|
1835
|
+
await this.cacheTransaction(t);
|
1836
|
+
const ret = await commit(params, t, done, opts);
|
1837
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1838
|
+
return ret.cgrp;
|
1839
|
+
});
|
1840
|
+
}
|
1841
|
+
async updateCarLog(cids, fp, compact) {
|
1842
|
+
if (compact) {
|
1843
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1844
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1845
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1846
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1847
|
+
} else {
|
1848
|
+
this.carLog.unshift(cids);
|
1849
|
+
}
|
1951
1850
|
}
|
1952
1851
|
async cacheTransaction(t) {
|
1953
1852
|
for await (const block of t.entries()) {
|
@@ -1967,79 +1866,6 @@ var Loader = class {
|
|
1967
1866
|
}
|
1968
1867
|
}
|
1969
1868
|
}
|
1970
|
-
async _commitInternal(t, done, opts = { noLoader: false, compact: false }) {
|
1971
|
-
await this.ready();
|
1972
|
-
const fp = this.makeCarHeader(done, this.carLog, !!opts.compact);
|
1973
|
-
const rootBlock = await encodeCarHeader(fp);
|
1974
|
-
const cars = await this.prepareCarFiles(rootBlock, t, !!opts.public);
|
1975
|
-
const cids = [];
|
1976
|
-
for (const car of cars) {
|
1977
|
-
const { cid, bytes } = car;
|
1978
|
-
await (await this.carStore()).save({ cid, bytes });
|
1979
|
-
cids.push(cid);
|
1980
|
-
}
|
1981
|
-
await this.cacheTransaction(t);
|
1982
|
-
const newDbMeta = { cars: cids, key: this.key || null };
|
1983
|
-
await (await this.remoteWAL()).enqueue(newDbMeta, opts);
|
1984
|
-
await (await this.metaStore()).save(newDbMeta);
|
1985
|
-
await this.updateCarLog(cids, fp, !!opts.compact);
|
1986
|
-
return cids;
|
1987
|
-
}
|
1988
|
-
async prepareCarFilesFiles(roots, t, isPublic) {
|
1989
|
-
const theKey = isPublic ? null : await this._getKey();
|
1990
|
-
const car = theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, roots[0], t) : await encodeCarFile(roots, t);
|
1991
|
-
return [car];
|
1992
|
-
}
|
1993
|
-
async prepareCarFiles(rootBlock, t, isPublic) {
|
1994
|
-
const theKey = isPublic ? void 0 : await this._getKey();
|
1995
|
-
const carFiles = [];
|
1996
|
-
const threshold = this.ebOpts.threshold || 1e3 * 1e3;
|
1997
|
-
let clonedt = new CarTransaction(t.parent, { add: false });
|
1998
|
-
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1999
|
-
let newsize = CBW2.blockLength(toCIDBlock(rootBlock));
|
2000
|
-
let cidRootBlock = rootBlock;
|
2001
|
-
for (const { cid, bytes } of t.entries()) {
|
2002
|
-
newsize += CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
2003
|
-
if (newsize >= threshold) {
|
2004
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
2005
|
-
clonedt = new CarTransaction(t.parent, { add: false });
|
2006
|
-
clonedt.putSync(cid, bytes);
|
2007
|
-
cidRootBlock = { cid, bytes };
|
2008
|
-
newsize = CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
2009
|
-
} else {
|
2010
|
-
clonedt.putSync(cid, bytes);
|
2011
|
-
}
|
2012
|
-
}
|
2013
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
2014
|
-
return carFiles;
|
2015
|
-
}
|
2016
|
-
async createCarFile(theKey, cid, t) {
|
2017
|
-
try {
|
2018
|
-
return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t) : await encodeCarFile([cid], t);
|
2019
|
-
} catch (e) {
|
2020
|
-
console.error("error creating car file", e);
|
2021
|
-
throw e;
|
2022
|
-
}
|
2023
|
-
}
|
2024
|
-
makeFileCarHeader(result) {
|
2025
|
-
const files = [];
|
2026
|
-
for (const [, meta] of Object.entries(result.files || {})) {
|
2027
|
-
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
2028
|
-
files.push(meta.cid);
|
2029
|
-
}
|
2030
|
-
}
|
2031
|
-
return { ...result, files };
|
2032
|
-
}
|
2033
|
-
async updateCarLog(cids, fp, compact) {
|
2034
|
-
if (compact) {
|
2035
|
-
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
2036
|
-
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
2037
|
-
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
2038
|
-
await this.removeCidsForCompact(previousCompactCid[0]);
|
2039
|
-
} else {
|
2040
|
-
this.carLog.unshift(cids);
|
2041
|
-
}
|
2042
|
-
}
|
2043
1869
|
async removeCidsForCompact(cid) {
|
2044
1870
|
const carHeader = await this.loadCarHeaderFromMeta({
|
2045
1871
|
cars: [cid]
|
@@ -2058,9 +1884,9 @@ var Loader = class {
|
|
2058
1884
|
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
2059
1885
|
// }
|
2060
1886
|
// }
|
2061
|
-
async *entries(
|
1887
|
+
async *entries(cache2 = true) {
|
2062
1888
|
await this.ready();
|
2063
|
-
if (
|
1889
|
+
if (cache2) {
|
2064
1890
|
for (const [, block] of this.getBlockCache) {
|
2065
1891
|
yield block;
|
2066
1892
|
}
|
@@ -2142,10 +1968,6 @@ var Loader = class {
|
|
2142
1968
|
}
|
2143
1969
|
return got;
|
2144
1970
|
}
|
2145
|
-
makeCarHeader(meta, cars, compact = false) {
|
2146
|
-
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
2147
|
-
return { ...coreHeader, meta };
|
2148
|
-
}
|
2149
1971
|
async loadCar(cid) {
|
2150
1972
|
if (!this.carStore) {
|
2151
1973
|
throw this.logger.Error().Msg("car store not initialized").AsError();
|
@@ -2153,72 +1975,52 @@ var Loader = class {
|
|
2153
1975
|
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
2154
1976
|
return loaded;
|
2155
1977
|
}
|
2156
|
-
|
2157
|
-
async storesLoadCar(cid, local, remote, publicFiles) {
|
1978
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
2158
1979
|
const cidsString = cid.toString();
|
2159
|
-
|
2160
|
-
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2164
|
-
|
2165
|
-
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2175
|
-
|
2176
|
-
|
2177
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
2178
|
-
}
|
2179
|
-
}
|
2180
|
-
if (!loadedCar) {
|
2181
|
-
throw this.logger.Error().Url(local.url).Str("cid", cidsString).Msg("missing car files").AsError();
|
2182
|
-
}
|
2183
|
-
const rawReader = await import_car.CarReader.fromBytes(loadedCar.bytes);
|
2184
|
-
const readerP = publicFiles ? Promise.resolve(rawReader) : this.ensureDecryptedReader(rawReader);
|
2185
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
2186
|
-
await this.cacheCarReader(cidsString, reader).catch(() => {
|
2187
|
-
return;
|
2188
|
-
});
|
2189
|
-
return reader;
|
2190
|
-
});
|
2191
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
2192
|
-
return readerP;
|
2193
|
-
})().catch((e) => {
|
2194
|
-
this.carReaders.delete(cidsString);
|
2195
|
-
throw e;
|
2196
|
-
})
|
2197
|
-
);
|
1980
|
+
let loadedCar = void 0;
|
1981
|
+
let activeStore = local;
|
1982
|
+
try {
|
1983
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1984
|
+
loadedCar = await local.load(cid);
|
1985
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1986
|
+
} catch (e) {
|
1987
|
+
if (remote) {
|
1988
|
+
const remoteCar = await remote.load(cid);
|
1989
|
+
if (remoteCar) {
|
1990
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1991
|
+
await local.save(remoteCar);
|
1992
|
+
loadedCar = remoteCar;
|
1993
|
+
activeStore = remote;
|
1994
|
+
}
|
1995
|
+
} else {
|
1996
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1997
|
+
}
|
2198
1998
|
}
|
2199
|
-
|
2200
|
-
|
2201
|
-
async ensureDecryptedReader(reader) {
|
2202
|
-
const theKey = await this._getKey();
|
2203
|
-
if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) {
|
2204
|
-
return reader;
|
1999
|
+
if (!loadedCar) {
|
2000
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
2205
2001
|
}
|
2206
|
-
const
|
2207
|
-
|
2208
|
-
|
2209
|
-
|
2210
|
-
|
2211
|
-
|
2002
|
+
const bytes = await decode({ bytes: loadedCar.bytes, hasher: import_sha23.sha256, codec: (await activeStore.keyedCrypto()).codec() });
|
2003
|
+
const rawReader = await import_car.CarReader.fromBytes(bytes.value);
|
2004
|
+
const readerP = Promise.resolve(rawReader);
|
2005
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
2006
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
2007
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
2008
|
+
return;
|
2009
|
+
});
|
2010
|
+
return reader;
|
2011
|
+
});
|
2012
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
2013
|
+
return readerP;
|
2212
2014
|
}
|
2213
|
-
|
2214
|
-
|
2215
|
-
|
2216
|
-
|
2217
|
-
|
2218
|
-
|
2219
|
-
|
2220
|
-
|
2221
|
-
|
2015
|
+
//What if instead it returns an Array of CarHeader
|
2016
|
+
async storesLoadCar(cid, local, remote) {
|
2017
|
+
const cidsString = cid.toString();
|
2018
|
+
let dacr = this.carReaders.get(cidsString);
|
2019
|
+
if (!dacr) {
|
2020
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
2021
|
+
this.carReaders.set(cidsString, dacr);
|
2022
|
+
}
|
2023
|
+
return dacr;
|
2222
2024
|
}
|
2223
2025
|
async getMoreReaders(cids) {
|
2224
2026
|
const limit = (0, import_p_limit.default)(5);
|
@@ -2227,20 +2029,345 @@ var Loader = class {
|
|
2227
2029
|
}
|
2228
2030
|
};
|
2229
2031
|
|
2032
|
+
// src/runtime/keyed-crypto.ts
|
2033
|
+
var keyed_crypto_exports = {};
|
2034
|
+
__export(keyed_crypto_exports, {
|
2035
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
2036
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
2037
|
+
});
|
2038
|
+
init_utils();
|
2039
|
+
var import_base583 = require("multiformats/bases/base58");
|
2040
|
+
var import_sha24 = require("multiformats/hashes/sha2");
|
2041
|
+
var CBOR = __toESM(require("cborg"), 1);
|
2042
|
+
var generateIV = {
|
2043
|
+
random: {
|
2044
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2045
|
+
calc: async (ko, crypto, data) => {
|
2046
|
+
return crypto.randomBytes(ko.ivLength);
|
2047
|
+
},
|
2048
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2049
|
+
verify: async (ko, crypto, iv, data) => {
|
2050
|
+
return true;
|
2051
|
+
}
|
2052
|
+
},
|
2053
|
+
hash: {
|
2054
|
+
calc: async (ko, crypto, data) => {
|
2055
|
+
const hash = await import_sha24.sha256.digest(data);
|
2056
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
2057
|
+
const hashArray = new Uint8Array(ko.ivLength * 8);
|
2058
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
2059
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
2060
|
+
}
|
2061
|
+
return hashArray;
|
2062
|
+
},
|
2063
|
+
verify: async function(ko, crypto, iv, data) {
|
2064
|
+
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
2065
|
+
}
|
2066
|
+
}
|
2067
|
+
};
|
2068
|
+
function getGenerateIVFn(url, opts) {
|
2069
|
+
const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
|
2070
|
+
return generateIV[ivhash] || generateIV["hash"];
|
2071
|
+
}
|
2072
|
+
var BlockIvKeyIdCodec = class {
|
2073
|
+
constructor(ko, iv, opts) {
|
2074
|
+
this.code = 3147065;
|
2075
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
2076
|
+
this.ko = ko;
|
2077
|
+
this.iv = iv;
|
2078
|
+
this.opts = opts || {};
|
2079
|
+
}
|
2080
|
+
async encode(data) {
|
2081
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
2082
|
+
const { iv } = this.ko.algo(calcIv);
|
2083
|
+
const fprt = await this.ko.fingerPrint();
|
2084
|
+
const keyId = import_base583.base58btc.decode(fprt);
|
2085
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
2086
|
+
return CBOR.encode({
|
2087
|
+
iv,
|
2088
|
+
keyId,
|
2089
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
2090
|
+
});
|
2091
|
+
}
|
2092
|
+
async decode(abytes) {
|
2093
|
+
let bytes;
|
2094
|
+
if (abytes instanceof Uint8Array) {
|
2095
|
+
bytes = abytes;
|
2096
|
+
} else {
|
2097
|
+
bytes = new Uint8Array(abytes);
|
2098
|
+
}
|
2099
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
2100
|
+
const fprt = await this.ko.fingerPrint();
|
2101
|
+
this.ko.logger.Debug().Str("fp", import_base583.base58btc.encode(keyId)).Msg("decode");
|
2102
|
+
if (import_base583.base58btc.encode(keyId) !== fprt) {
|
2103
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", import_base583.base58btc.encode(keyId)).Msg("keyId mismatch").AsError();
|
2104
|
+
}
|
2105
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
2106
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
2107
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
2108
|
+
}
|
2109
|
+
return result;
|
2110
|
+
}
|
2111
|
+
};
|
2112
|
+
var keyedCrypto = class {
|
2113
|
+
constructor(url, key, cyopt, sthis) {
|
2114
|
+
this.ivLength = 12;
|
2115
|
+
this.isEncrypting = true;
|
2116
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
2117
|
+
this.crypto = cyopt;
|
2118
|
+
this.key = key;
|
2119
|
+
this.url = url;
|
2120
|
+
}
|
2121
|
+
fingerPrint() {
|
2122
|
+
return Promise.resolve(this.key.fingerPrint);
|
2123
|
+
}
|
2124
|
+
codec(iv, opts) {
|
2125
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
2126
|
+
}
|
2127
|
+
algo(iv) {
|
2128
|
+
return {
|
2129
|
+
name: "AES-GCM",
|
2130
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
2131
|
+
tagLength: 128
|
2132
|
+
};
|
2133
|
+
}
|
2134
|
+
async _decrypt(data) {
|
2135
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("decrypting");
|
2136
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
2137
|
+
}
|
2138
|
+
async _encrypt(data) {
|
2139
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
2140
|
+
const a = this.algo(data.iv);
|
2141
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
2142
|
+
}
|
2143
|
+
};
|
2144
|
+
var nullCodec = class {
|
2145
|
+
constructor() {
|
2146
|
+
this.code = 0;
|
2147
|
+
this.name = "Fireproof@unencrypted-block";
|
2148
|
+
}
|
2149
|
+
encode(data) {
|
2150
|
+
return data;
|
2151
|
+
}
|
2152
|
+
decode(data) {
|
2153
|
+
return data;
|
2154
|
+
}
|
2155
|
+
};
|
2156
|
+
var noCrypto = class {
|
2157
|
+
constructor(url, cyrt, sthis) {
|
2158
|
+
this.ivLength = 0;
|
2159
|
+
this.code = 0;
|
2160
|
+
this.name = "Fireproof@unencrypted-block";
|
2161
|
+
this.isEncrypting = false;
|
2162
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
2163
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
2164
|
+
this.crypto = cyrt;
|
2165
|
+
this.url = url;
|
2166
|
+
}
|
2167
|
+
fingerPrint() {
|
2168
|
+
return Promise.resolve(this._fingerPrint);
|
2169
|
+
}
|
2170
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2171
|
+
codec(iv) {
|
2172
|
+
return new nullCodec();
|
2173
|
+
}
|
2174
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2175
|
+
algo(iv) {
|
2176
|
+
return {
|
2177
|
+
name: "noCrypto",
|
2178
|
+
iv: new Uint8Array(),
|
2179
|
+
tagLength: 0
|
2180
|
+
};
|
2181
|
+
}
|
2182
|
+
_decrypt() {
|
2183
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
2184
|
+
}
|
2185
|
+
_encrypt() {
|
2186
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
2187
|
+
}
|
2188
|
+
};
|
2189
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
2190
|
+
const storekey = url.getParam("storekey");
|
2191
|
+
if (storekey && storekey !== "insecure") {
|
2192
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
2193
|
+
if (rkey.isErr()) {
|
2194
|
+
try {
|
2195
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
2196
|
+
} catch (e) {
|
2197
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
2198
|
+
}
|
2199
|
+
}
|
2200
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
2201
|
+
}
|
2202
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
2203
|
+
}
|
2204
|
+
|
2205
|
+
// src/blockstore/fragment-gateway.ts
|
2206
|
+
var import_base584 = require("multiformats/bases/base58");
|
2207
|
+
var import_cborg = require("cborg");
|
2208
|
+
function getFragSize(url) {
|
2209
|
+
const fragSize = url.getParam("fragSize");
|
2210
|
+
let ret = 0;
|
2211
|
+
if (fragSize) {
|
2212
|
+
ret = parseInt(fragSize);
|
2213
|
+
}
|
2214
|
+
if (isNaN(ret) || ret <= 0) {
|
2215
|
+
ret = 0;
|
2216
|
+
}
|
2217
|
+
return ret;
|
2218
|
+
}
|
2219
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
2220
|
+
const fragSize = getFragSize(url);
|
2221
|
+
if (!fragSize) {
|
2222
|
+
const res = await innerGW.get(url);
|
2223
|
+
if (res.isErr()) {
|
2224
|
+
return [res];
|
2225
|
+
}
|
2226
|
+
const data = res.unwrap();
|
2227
|
+
return [
|
2228
|
+
import_cement.Result.Ok({
|
2229
|
+
fid: new Uint8Array(0),
|
2230
|
+
ofs: 0,
|
2231
|
+
len: data.length,
|
2232
|
+
data
|
2233
|
+
})
|
2234
|
+
];
|
2235
|
+
}
|
2236
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
2237
|
+
if (firstRaw.isErr()) {
|
2238
|
+
return [firstRaw];
|
2239
|
+
}
|
2240
|
+
const firstFragment = (0, import_cborg.decode)(firstRaw.unwrap());
|
2241
|
+
const blockSize = firstFragment.data.length;
|
2242
|
+
const ops = [Promise.resolve(import_cement.Result.Ok(firstFragment))];
|
2243
|
+
const fidStr = import_base584.base58btc.encode(firstFragment.fid);
|
2244
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
2245
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
2246
|
+
ops.push(
|
2247
|
+
(async (furl, ofs2) => {
|
2248
|
+
const raw2 = await innerGW.get(furl);
|
2249
|
+
if (raw2.isErr()) {
|
2250
|
+
return raw2;
|
2251
|
+
}
|
2252
|
+
const fragment = (0, import_cborg.decode)(raw2.unwrap());
|
2253
|
+
if (import_base584.base58btc.encode(fragment.fid) !== fidStr) {
|
2254
|
+
return import_cement.Result.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
2255
|
+
}
|
2256
|
+
if (fragment.ofs !== ofs2) {
|
2257
|
+
return import_cement.Result.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
2258
|
+
}
|
2259
|
+
return import_cement.Result.Ok(fragment);
|
2260
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
2261
|
+
);
|
2262
|
+
}
|
2263
|
+
return Promise.all(ops);
|
2264
|
+
}
|
2265
|
+
var FragmentGateway = class {
|
2266
|
+
constructor(sthis, innerGW) {
|
2267
|
+
this.fidLength = 4;
|
2268
|
+
this.headerSize = 32;
|
2269
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
2270
|
+
this.logger = this.sthis.logger;
|
2271
|
+
this.innerGW = innerGW;
|
2272
|
+
}
|
2273
|
+
slicer(url, body) {
|
2274
|
+
const fragSize = getFragSize(url);
|
2275
|
+
if (!fragSize) {
|
2276
|
+
return [this.innerGW.put(url, body)];
|
2277
|
+
}
|
2278
|
+
const blocksize = fragSize - this.headerSize;
|
2279
|
+
if (blocksize <= 0) {
|
2280
|
+
throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
|
2281
|
+
}
|
2282
|
+
const ops = [];
|
2283
|
+
const fid = this.sthis.nextId(this.fidLength);
|
2284
|
+
const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
|
2285
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
2286
|
+
const block = (0, import_cborg.encode)({
|
2287
|
+
fid: fid.bin,
|
2288
|
+
ofs,
|
2289
|
+
len: body.length,
|
2290
|
+
data: body.slice(ofs, ofs + blocksize)
|
2291
|
+
});
|
2292
|
+
if (block.length > fragSize) {
|
2293
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
2294
|
+
}
|
2295
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
2296
|
+
}
|
2297
|
+
return ops;
|
2298
|
+
}
|
2299
|
+
buildUrl(baseUrl, key) {
|
2300
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
2301
|
+
}
|
2302
|
+
async destroy(iurl) {
|
2303
|
+
return this.innerGW.destroy(iurl);
|
2304
|
+
}
|
2305
|
+
async start(url) {
|
2306
|
+
this.headerSize = (0, import_cborg.encode)({
|
2307
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
2308
|
+
ofs: 1024 * 1024,
|
2309
|
+
// 32bit
|
2310
|
+
len: 16 * 1024 * 1024,
|
2311
|
+
// 32bit
|
2312
|
+
data: new Uint8Array(1024)
|
2313
|
+
}).length - 1024;
|
2314
|
+
return this.innerGW.start(url);
|
2315
|
+
}
|
2316
|
+
async close(url) {
|
2317
|
+
return this.innerGW.close(url);
|
2318
|
+
}
|
2319
|
+
async put(url, body) {
|
2320
|
+
await Promise.all(this.slicer(url, body));
|
2321
|
+
return import_cement.Result.Ok(void 0);
|
2322
|
+
}
|
2323
|
+
async get(url) {
|
2324
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
2325
|
+
let buffer = void 0;
|
2326
|
+
for (const rfrag of rfrags) {
|
2327
|
+
if (rfrag.isErr()) {
|
2328
|
+
return import_cement.Result.Err(rfrag.Err());
|
2329
|
+
}
|
2330
|
+
const frag = rfrag.Ok();
|
2331
|
+
buffer = buffer || new Uint8Array(frag.len);
|
2332
|
+
buffer.set(frag.data, frag.ofs);
|
2333
|
+
}
|
2334
|
+
return import_cement.Result.Ok(buffer || new Uint8Array(0));
|
2335
|
+
}
|
2336
|
+
async delete(url) {
|
2337
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
2338
|
+
for (const rfrag of rfrags) {
|
2339
|
+
if (rfrag.isErr()) {
|
2340
|
+
return import_cement.Result.Err(rfrag.Err());
|
2341
|
+
}
|
2342
|
+
const frag = rfrag.Ok();
|
2343
|
+
const fidStr = import_base584.base58btc.encode(frag.fid);
|
2344
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
2345
|
+
await this.innerGW.delete(fragUrl);
|
2346
|
+
}
|
2347
|
+
return import_cement.Result.Ok(void 0);
|
2348
|
+
}
|
2349
|
+
};
|
2350
|
+
|
2230
2351
|
// src/blockstore/store.ts
|
2231
|
-
function guardVersion(
|
2232
|
-
if (!
|
2233
|
-
return
|
2352
|
+
function guardVersion(url) {
|
2353
|
+
if (!url.hasParam("version")) {
|
2354
|
+
return import_cement7.Result.Err(`missing version: ${url.toString()}`);
|
2234
2355
|
}
|
2235
|
-
return
|
2356
|
+
return import_cement7.Result.Ok(url);
|
2236
2357
|
}
|
2237
|
-
var
|
2238
|
-
constructor(name,
|
2358
|
+
var BaseStoreImpl = class {
|
2359
|
+
constructor(name, url, opts, sthis, logger) {
|
2239
2360
|
this._onStarted = [];
|
2240
2361
|
this._onClosed = [];
|
2241
2362
|
this.name = name;
|
2242
|
-
this.
|
2243
|
-
this.
|
2363
|
+
this._url = url;
|
2364
|
+
this.keybag = opts.keybag;
|
2365
|
+
this.sthis = sthis;
|
2366
|
+
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
2367
|
+
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
2368
|
+
}
|
2369
|
+
url() {
|
2370
|
+
return this._url;
|
2244
2371
|
}
|
2245
2372
|
onStarted(fn) {
|
2246
2373
|
this._onStarted.push(fn);
|
@@ -2248,110 +2375,168 @@ var VersionedStore = class {
|
|
2248
2375
|
onClosed(fn) {
|
2249
2376
|
this._onClosed.push(fn);
|
2250
2377
|
}
|
2378
|
+
async keyedCrypto() {
|
2379
|
+
return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
|
2380
|
+
}
|
2381
|
+
async start() {
|
2382
|
+
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
2383
|
+
this._url = this._url.build().setParam("store", this.storeType).URI();
|
2384
|
+
const res = await this.gateway.start(this._url);
|
2385
|
+
if (res.isErr()) {
|
2386
|
+
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
2387
|
+
return res;
|
2388
|
+
}
|
2389
|
+
this._url = res.Ok();
|
2390
|
+
const kb = await this.keybag();
|
2391
|
+
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
2392
|
+
const idx = this._url.getParam("index");
|
2393
|
+
const storeKeyName = [this.name];
|
2394
|
+
if (idx) {
|
2395
|
+
storeKeyName.push(idx);
|
2396
|
+
}
|
2397
|
+
storeKeyName.push(this.storeType);
|
2398
|
+
return storeKeyName.join(":");
|
2399
|
+
});
|
2400
|
+
if (skRes.isErr()) {
|
2401
|
+
return skRes;
|
2402
|
+
}
|
2403
|
+
this._url = skRes.Ok();
|
2404
|
+
const version = guardVersion(this._url);
|
2405
|
+
if (version.isErr()) {
|
2406
|
+
this.logger.Error().Result("version", version).Msg("guardVersion");
|
2407
|
+
await this.close();
|
2408
|
+
return version;
|
2409
|
+
}
|
2410
|
+
if (this.ready) {
|
2411
|
+
const fn = this.ready.bind(this);
|
2412
|
+
const ready = await exception2Result(fn);
|
2413
|
+
if (ready.isErr()) {
|
2414
|
+
await this.close();
|
2415
|
+
return ready;
|
2416
|
+
}
|
2417
|
+
}
|
2418
|
+
this._onStarted.forEach((fn) => fn());
|
2419
|
+
this.logger.Debug().Msg("started");
|
2420
|
+
return version;
|
2421
|
+
}
|
2251
2422
|
};
|
2252
|
-
var
|
2253
|
-
|
2254
|
-
|
2255
|
-
|
2256
|
-
|
2257
|
-
|
2258
|
-
|
2259
|
-
|
2260
|
-
|
2423
|
+
var MetaStoreImpl = class extends BaseStoreImpl {
|
2424
|
+
constructor(sthis, name, url, opts) {
|
2425
|
+
super(
|
2426
|
+
name,
|
2427
|
+
url,
|
2428
|
+
{
|
2429
|
+
...opts
|
2430
|
+
},
|
2431
|
+
sthis,
|
2432
|
+
ensureLogger(sthis, "MetaStoreImpl")
|
2433
|
+
);
|
2434
|
+
this.storeType = "meta";
|
2435
|
+
this.subscribers = /* @__PURE__ */ new Map();
|
2436
|
+
}
|
2437
|
+
onLoad(branch, loadHandler) {
|
2438
|
+
const subscribers = this.subscribers.get(branch) || [];
|
2439
|
+
subscribers.push(loadHandler);
|
2440
|
+
this.subscribers.set(branch, subscribers);
|
2441
|
+
return () => {
|
2442
|
+
const subscribers2 = this.subscribers.get(branch) || [];
|
2443
|
+
const idx = subscribers2.indexOf(loadHandler);
|
2444
|
+
if (idx > -1) subscribers2.splice(idx, 1);
|
2445
|
+
};
|
2446
|
+
}
|
2447
|
+
makeHeader({ cars }) {
|
2261
2448
|
const toEncode = { cars };
|
2262
|
-
if (key) toEncode.key = key;
|
2263
2449
|
return (0, import_dag_json.format)(toEncode);
|
2264
2450
|
}
|
2265
2451
|
parseHeader(headerData) {
|
2266
2452
|
const got = (0, import_dag_json.parse)(headerData);
|
2267
2453
|
return got;
|
2268
2454
|
}
|
2269
|
-
async
|
2270
|
-
|
2271
|
-
|
2272
|
-
|
2273
|
-
|
2455
|
+
async handleSubscribers(dbMetas, branch) {
|
2456
|
+
try {
|
2457
|
+
const subscribers = this.subscribers.get(branch) || [];
|
2458
|
+
await Promise.all(subscribers.map((subscriber) => subscriber(dbMetas)));
|
2459
|
+
} catch (e) {
|
2460
|
+
this.logger.Error().Err(e).Msg("handleSubscribers").AsError();
|
2274
2461
|
}
|
2275
|
-
|
2276
|
-
|
2462
|
+
}
|
2463
|
+
async handleByteHeads(byteHeads, branch = "main") {
|
2464
|
+
let dbMetas;
|
2465
|
+
try {
|
2466
|
+
dbMetas = this.dbMetasForByteHeads(byteHeads);
|
2467
|
+
} catch (e) {
|
2468
|
+
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
2469
|
+
}
|
2470
|
+
await this.handleSubscribers(dbMetas, branch);
|
2471
|
+
return dbMetas;
|
2472
|
+
}
|
2473
|
+
dbMetasForByteHeads(byteHeads) {
|
2474
|
+
return byteHeads.map((bytes) => {
|
2475
|
+
const txt = this.sthis.txt.decode(bytes);
|
2476
|
+
return this.parseHeader(txt);
|
2477
|
+
});
|
2277
2478
|
}
|
2278
2479
|
async load(branch) {
|
2279
|
-
|
2280
|
-
|
2281
|
-
|
2282
|
-
|
2480
|
+
branch = branch || "main";
|
2481
|
+
this.logger.Debug().Str("branch", branch).Msg("loading");
|
2482
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2483
|
+
if (url.isErr()) {
|
2484
|
+
throw this.logger.Error().Result("buidUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2283
2485
|
}
|
2284
|
-
const bytes = await this.gateway.get(
|
2486
|
+
const bytes = await this.gateway.get(url.Ok());
|
2285
2487
|
if (bytes.isErr()) {
|
2286
2488
|
if (isNotFoundError(bytes)) {
|
2287
2489
|
return void 0;
|
2288
2490
|
}
|
2289
|
-
throw this.logger.Error().Url(
|
2290
|
-
}
|
2291
|
-
try {
|
2292
|
-
return [this.parseHeader(textDecoder.decode(bytes.Ok()))];
|
2293
|
-
} catch (e) {
|
2294
|
-
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
2491
|
+
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
2295
2492
|
}
|
2493
|
+
return this.handleByteHeads([bytes.Ok()], branch);
|
2296
2494
|
}
|
2297
|
-
async save(meta, branch
|
2495
|
+
async save(meta, branch) {
|
2496
|
+
branch = branch || "main";
|
2298
2497
|
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
2299
2498
|
const bytes = this.makeHeader(meta);
|
2300
|
-
const
|
2301
|
-
if (
|
2302
|
-
throw this.logger.Error().Err(
|
2499
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2500
|
+
if (url.isErr()) {
|
2501
|
+
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2303
2502
|
}
|
2304
|
-
const res = await this.gateway.put(
|
2503
|
+
const res = await this.gateway.put(url.Ok(), this.sthis.txt.encode(bytes));
|
2305
2504
|
if (res.isErr()) {
|
2306
2505
|
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2307
2506
|
}
|
2308
|
-
|
2507
|
+
await this.handleSubscribers([meta], branch);
|
2508
|
+
return res;
|
2309
2509
|
}
|
2310
2510
|
async close() {
|
2311
|
-
await this.gateway.close(this.url);
|
2511
|
+
await this.gateway.close(this.url());
|
2312
2512
|
this._onClosed.forEach((fn) => fn());
|
2313
|
-
return
|
2513
|
+
return import_cement7.Result.Ok(void 0);
|
2314
2514
|
}
|
2315
2515
|
async destroy() {
|
2316
|
-
return this.gateway.destroy(this.url);
|
2516
|
+
return this.gateway.destroy(this.url());
|
2317
2517
|
}
|
2318
2518
|
};
|
2319
|
-
var
|
2320
|
-
|
2519
|
+
var DataStoreImpl = class extends BaseStoreImpl {
|
2520
|
+
// readonly tag: string = "car-base";
|
2521
|
+
constructor(sthis, name, url, opts) {
|
2321
2522
|
super(
|
2322
2523
|
name,
|
2323
|
-
|
2324
|
-
|
2325
|
-
|
2326
|
-
}
|
2524
|
+
url,
|
2525
|
+
{
|
2526
|
+
...opts
|
2527
|
+
},
|
2528
|
+
sthis,
|
2529
|
+
ensureLogger(sthis, "DataStoreImpl")
|
2327
2530
|
);
|
2328
|
-
this.
|
2329
|
-
this.gateway = gateway;
|
2330
|
-
}
|
2331
|
-
async start() {
|
2332
|
-
this.logger.Debug().Msg("starting-gateway");
|
2333
|
-
const res = await this.gateway.start(this.url);
|
2334
|
-
if (res.isErr()) {
|
2335
|
-
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
2336
|
-
return res;
|
2337
|
-
}
|
2338
|
-
this._onStarted.forEach((fn) => fn());
|
2339
|
-
const version = guardVersion(this.url);
|
2340
|
-
if (version.isErr()) {
|
2341
|
-
this.logger.Error().Result("version", version).Msg("guardVersion");
|
2342
|
-
await this.close();
|
2343
|
-
return version;
|
2344
|
-
}
|
2345
|
-
this.logger.Debug().Msg("started");
|
2346
|
-
return version;
|
2531
|
+
this.storeType = "data";
|
2347
2532
|
}
|
2348
2533
|
async load(cid) {
|
2349
2534
|
this.logger.Debug().Any("cid", cid).Msg("loading");
|
2350
|
-
const
|
2351
|
-
if (
|
2352
|
-
throw this.logger.Error().Err(
|
2535
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
2536
|
+
if (url.isErr()) {
|
2537
|
+
throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
|
2353
2538
|
}
|
2354
|
-
const res = await this.gateway.get(
|
2539
|
+
const res = await this.gateway.get(url.Ok());
|
2355
2540
|
if (res.isErr()) {
|
2356
2541
|
throw res.Err();
|
2357
2542
|
}
|
@@ -2360,57 +2545,64 @@ var DataStore = class extends VersionedStore {
|
|
2360
2545
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
2361
2546
|
async save(car, opts) {
|
2362
2547
|
this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
|
2363
|
-
const
|
2364
|
-
if (
|
2365
|
-
throw this.logger.Error().Err(
|
2548
|
+
const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
|
2549
|
+
if (url.isErr()) {
|
2550
|
+
throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
|
2366
2551
|
}
|
2367
|
-
const res = await this.gateway.put(
|
2552
|
+
const res = await this.gateway.put(url.Ok(), car.bytes);
|
2368
2553
|
if (res.isErr()) {
|
2369
2554
|
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2370
2555
|
}
|
2371
2556
|
return res.Ok();
|
2372
2557
|
}
|
2373
2558
|
async remove(cid) {
|
2374
|
-
const
|
2375
|
-
if (
|
2376
|
-
return
|
2559
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
2560
|
+
if (url.isErr()) {
|
2561
|
+
return url;
|
2377
2562
|
}
|
2378
|
-
return this.gateway.delete(
|
2563
|
+
return this.gateway.delete(url.Ok());
|
2379
2564
|
}
|
2380
2565
|
async close() {
|
2381
|
-
await this.gateway.close(this.url);
|
2566
|
+
await this.gateway.close(this.url());
|
2382
2567
|
this._onClosed.forEach((fn) => fn());
|
2383
|
-
return
|
2568
|
+
return import_cement7.Result.Ok(void 0);
|
2384
2569
|
}
|
2385
2570
|
destroy() {
|
2386
|
-
return this.gateway.destroy(this.url);
|
2571
|
+
return this.gateway.destroy(this.url());
|
2387
2572
|
}
|
2388
2573
|
};
|
2389
|
-
var
|
2390
|
-
constructor(loader,
|
2391
|
-
super(
|
2392
|
-
|
2393
|
-
|
2574
|
+
var WALStoreImpl = class extends BaseStoreImpl {
|
2575
|
+
constructor(loader, url, opts) {
|
2576
|
+
super(
|
2577
|
+
loader.name,
|
2578
|
+
url,
|
2579
|
+
{
|
2580
|
+
...opts
|
2581
|
+
},
|
2582
|
+
loader.sthis,
|
2583
|
+
ensureLogger(loader.sthis, "WALStoreImpl")
|
2584
|
+
);
|
2585
|
+
this.storeType = "wal";
|
2586
|
+
this._ready = new import_cement7.ResolveOnce();
|
2394
2587
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
2395
2588
|
this.processing = void 0;
|
2396
2589
|
this.processQueue = new CommitQueue();
|
2397
|
-
this.
|
2398
|
-
|
2399
|
-
|
2400
|
-
|
2401
|
-
|
2402
|
-
|
2403
|
-
|
2404
|
-
|
2590
|
+
this.ready = async () => {
|
2591
|
+
return this._ready.once(async () => {
|
2592
|
+
const walState = await this.load().catch((e) => {
|
2593
|
+
this.logger.Error().Any("error", e).Msg("error loading wal");
|
2594
|
+
return void 0;
|
2595
|
+
});
|
2596
|
+
if (!walState) {
|
2597
|
+
this.walState.operations = [];
|
2598
|
+
this.walState.fileOperations = [];
|
2599
|
+
} else {
|
2600
|
+
this.walState.operations = walState.operations || [];
|
2601
|
+
this.walState.fileOperations = walState.fileOperations || [];
|
2602
|
+
}
|
2405
2603
|
});
|
2406
|
-
|
2407
|
-
|
2408
|
-
this.walState.fileOperations = [];
|
2409
|
-
} else {
|
2410
|
-
this.walState.operations = walState.operations || [];
|
2411
|
-
this.walState.fileOperations = walState.fileOperations || [];
|
2412
|
-
}
|
2413
|
-
});
|
2604
|
+
};
|
2605
|
+
this.loader = loader;
|
2414
2606
|
}
|
2415
2607
|
async enqueue(dbMeta, opts) {
|
2416
2608
|
await this.ready();
|
@@ -2420,19 +2612,23 @@ var RemoteWAL = class extends VersionedStore {
|
|
2420
2612
|
this.walState.operations.push(dbMeta);
|
2421
2613
|
}
|
2422
2614
|
await this.save(this.walState);
|
2423
|
-
void this.
|
2615
|
+
void this.process();
|
2424
2616
|
}
|
2425
2617
|
async enqueueFile(fileCid, publicFile = false) {
|
2426
2618
|
await this.ready();
|
2427
2619
|
this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
|
2428
2620
|
}
|
2429
|
-
async
|
2621
|
+
async process() {
|
2430
2622
|
await this.ready();
|
2431
2623
|
if (!this.loader.remoteCarStore) return;
|
2432
2624
|
await this.processQueue.enqueue(async () => {
|
2433
|
-
|
2625
|
+
try {
|
2626
|
+
await this._doProcess();
|
2627
|
+
} catch (e) {
|
2628
|
+
this.logger.Error().Any("error", e).Msg("error processing wal");
|
2629
|
+
}
|
2434
2630
|
if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
|
2435
|
-
setTimeout(() => void this.
|
2631
|
+
setTimeout(() => void this.process(), 0);
|
2436
2632
|
}
|
2437
2633
|
});
|
2438
2634
|
}
|
@@ -2450,7 +2646,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2450
2646
|
for (const cid of dbMeta.cars) {
|
2451
2647
|
const car = await (await this.loader.carStore()).load(cid);
|
2452
2648
|
if (!car) {
|
2453
|
-
if (
|
2649
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
2454
2650
|
throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
|
2455
2651
|
} else {
|
2456
2652
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -2465,7 +2661,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2465
2661
|
for (const cid of dbMeta.cars) {
|
2466
2662
|
const car = await (await this.loader.carStore()).load(cid).catch(() => null);
|
2467
2663
|
if (!car) {
|
2468
|
-
if (
|
2664
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
2469
2665
|
throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
|
2470
2666
|
} else {
|
2471
2667
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -2490,11 +2686,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
2490
2686
|
const res = await Promise.allSettled(uploads);
|
2491
2687
|
const errors = res.filter((r) => r.status === "rejected");
|
2492
2688
|
if (errors.length) {
|
2493
|
-
throw this.logger.Error().Any(
|
2494
|
-
"errors",
|
2495
|
-
errors.map((e) => e.reason)
|
2496
|
-
).Msg("error uploading").AsError();
|
2497
|
-
errors[0].reason;
|
2689
|
+
throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
|
2498
2690
|
}
|
2499
2691
|
if (operations.length) {
|
2500
2692
|
const lastOp = operations[operations.length - 1];
|
@@ -2509,29 +2701,11 @@ var RemoteWAL = class extends VersionedStore {
|
|
2509
2701
|
})();
|
2510
2702
|
await rmlp;
|
2511
2703
|
}
|
2512
|
-
async start() {
|
2513
|
-
const res = await this.gateway.start(this.url);
|
2514
|
-
if (res.isErr()) {
|
2515
|
-
return res;
|
2516
|
-
}
|
2517
|
-
const ver = guardVersion(this.url);
|
2518
|
-
if (ver.isErr()) {
|
2519
|
-
await this.close();
|
2520
|
-
return ver;
|
2521
|
-
}
|
2522
|
-
const ready = await exception2Result(() => this.ready());
|
2523
|
-
this._onStarted.forEach((fn) => fn());
|
2524
|
-
if (ready.isErr()) {
|
2525
|
-
await this.close();
|
2526
|
-
return ready;
|
2527
|
-
}
|
2528
|
-
return ready;
|
2529
|
-
}
|
2530
2704
|
async load() {
|
2531
2705
|
this.logger.Debug().Msg("loading");
|
2532
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
2706
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
2533
2707
|
if (filepath.isErr()) {
|
2534
|
-
throw this.logger.Error().Err(filepath.Err()).
|
2708
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
2535
2709
|
}
|
2536
2710
|
const bytes = await this.gateway.get(filepath.Ok());
|
2537
2711
|
if (bytes.isErr()) {
|
@@ -2541,15 +2715,15 @@ var RemoteWAL = class extends VersionedStore {
|
|
2541
2715
|
throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
|
2542
2716
|
}
|
2543
2717
|
try {
|
2544
|
-
return bytes && (0, import_dag_json.parse)(
|
2718
|
+
return bytes && (0, import_dag_json.parse)(this.sthis.txt.decode(bytes.Ok()));
|
2545
2719
|
} catch (e) {
|
2546
2720
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
2547
2721
|
}
|
2548
2722
|
}
|
2549
2723
|
async save(state) {
|
2550
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
2724
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
2551
2725
|
if (filepath.isErr()) {
|
2552
|
-
throw this.logger.Error().Err(filepath.Err()).
|
2726
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
2553
2727
|
}
|
2554
2728
|
let encoded;
|
2555
2729
|
try {
|
@@ -2557,56 +2731,68 @@ var RemoteWAL = class extends VersionedStore {
|
|
2557
2731
|
} catch (e) {
|
2558
2732
|
throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
|
2559
2733
|
}
|
2560
|
-
const res = await this.gateway.put(filepath.Ok(),
|
2734
|
+
const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
|
2561
2735
|
if (res.isErr()) {
|
2562
2736
|
throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
|
2563
2737
|
}
|
2564
2738
|
}
|
2565
2739
|
async close() {
|
2566
|
-
await this.gateway.close(this.url);
|
2740
|
+
await this.gateway.close(this.url());
|
2567
2741
|
this._onClosed.forEach((fn) => fn());
|
2568
|
-
return
|
2742
|
+
return import_cement7.Result.Ok(void 0);
|
2569
2743
|
}
|
2570
2744
|
destroy() {
|
2571
|
-
return this.gateway.destroy(this.url);
|
2745
|
+
return this.gateway.destroy(this.url());
|
2572
2746
|
}
|
2573
2747
|
};
|
2574
2748
|
|
2575
2749
|
// src/blockstore/store-factory.ts
|
2576
2750
|
init_utils();
|
2577
|
-
function ensureIsIndex(
|
2751
|
+
function ensureIsIndex(url, isIndex) {
|
2578
2752
|
if (isIndex) {
|
2579
|
-
|
2580
|
-
return url2;
|
2581
|
-
} else {
|
2582
|
-
url2.searchParams.delete("index");
|
2583
|
-
return url2;
|
2753
|
+
return url.build().setParam("index", isIndex).URI();
|
2584
2754
|
}
|
2755
|
+
return url.build().delParam("index").URI();
|
2585
2756
|
}
|
2586
|
-
function
|
2587
|
-
if (
|
2588
|
-
|
2589
|
-
const url2 = new URL(pathOrUrl);
|
2590
|
-
return ensureIsIndex(url2, isIndex);
|
2591
|
-
} catch (e) {
|
2592
|
-
const url2 = new URL(`file://${pathOrUrl}`);
|
2593
|
-
return ensureIsIndex(url2, isIndex);
|
2757
|
+
function ensureName(name, url) {
|
2758
|
+
if (!url.hasParam("name")) {
|
2759
|
+
return url.build().setParam("name", name).URI();
|
2594
2760
|
}
|
2761
|
+
return url;
|
2595
2762
|
}
|
2596
2763
|
var storeFactory = /* @__PURE__ */ new Map();
|
2597
|
-
function ensureName(name, url2) {
|
2598
|
-
if (!url2.searchParams.has("name")) {
|
2599
|
-
url2.searchParams.set("name", name);
|
2600
|
-
}
|
2601
|
-
}
|
2602
2764
|
function buildURL(optURL, loader) {
|
2603
2765
|
const storeOpts = loader.ebOpts.store;
|
2604
2766
|
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
2605
2767
|
let obuUrl;
|
2606
2768
|
if (obuItem && obuItem.overrideBaseURL) {
|
2607
|
-
obuUrl =
|
2769
|
+
obuUrl = import_cement10.URI.from(obuItem.overrideBaseURL);
|
2608
2770
|
}
|
2609
|
-
|
2771
|
+
const ret = ensureIsIndex(
|
2772
|
+
import_cement10.URI.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2773
|
+
storeOpts.isIndex
|
2774
|
+
);
|
2775
|
+
return ret;
|
2776
|
+
}
|
2777
|
+
var onceGateway = new import_cement10.KeyedResolvOnce();
|
2778
|
+
async function getGatewayFromURL(url, sthis) {
|
2779
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2780
|
+
const item = storeFactory.get(url.protocol);
|
2781
|
+
if (item) {
|
2782
|
+
const ret = {
|
2783
|
+
gateway: await item.gateway(sthis),
|
2784
|
+
test: await item.test(sthis)
|
2785
|
+
};
|
2786
|
+
const res = await ret.gateway.start(url);
|
2787
|
+
if (res.isErr()) {
|
2788
|
+
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2789
|
+
return void 0;
|
2790
|
+
}
|
2791
|
+
return ret;
|
2792
|
+
}
|
2793
|
+
sthis.logger.Warn().Url(url).Msg("unsupported protocol");
|
2794
|
+
return void 0;
|
2795
|
+
});
|
2610
2796
|
}
|
2611
2797
|
function registerStoreProtocol(item) {
|
2612
2798
|
let protocol = item.protocol;
|
@@ -2615,8 +2801,7 @@ function registerStoreProtocol(item) {
|
|
2615
2801
|
}
|
2616
2802
|
if (storeFactory.has(protocol)) {
|
2617
2803
|
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
2618
|
-
|
2619
|
-
logger.Warn().Msg(`protocol ${protocol} already registered`);
|
2804
|
+
throw new Error(`we need a logger here`);
|
2620
2805
|
return () => {
|
2621
2806
|
};
|
2622
2807
|
}
|
@@ -2631,106 +2816,83 @@ function registerStoreProtocol(item) {
|
|
2631
2816
|
storeFactory.delete(protocol);
|
2632
2817
|
};
|
2633
2818
|
}
|
2634
|
-
function runStoreFactory(url2, logger, run) {
|
2635
|
-
const item = storeFactory.get(url2.protocol);
|
2636
|
-
if (!item) {
|
2637
|
-
throw logger.Error().Url(url2).Str("protocol", url2.protocol).Any("keys", Array(storeFactory.keys())).Msg(`unsupported protocol`).AsError();
|
2638
|
-
}
|
2639
|
-
logger.Debug().Str("protocol", url2.protocol).Msg("run");
|
2640
|
-
return run(item);
|
2641
|
-
}
|
2642
|
-
var onceLoadDataGateway = new import_cement8.KeyedResolvOnce();
|
2643
|
-
function loadDataGateway(url2, logger) {
|
2644
|
-
return onceLoadDataGateway.get(url2.protocol).once(async () => {
|
2645
|
-
return runStoreFactory(url2, logger, async (item) => item.data(logger));
|
2646
|
-
});
|
2647
|
-
}
|
2648
|
-
var onceDataStoreFactory = new import_cement8.KeyedResolvOnce();
|
2649
2819
|
async function dataStoreFactory(loader) {
|
2650
|
-
const
|
2651
|
-
|
2652
|
-
const
|
2653
|
-
|
2654
|
-
|
2655
|
-
|
2656
|
-
|
2657
|
-
|
2658
|
-
|
2659
|
-
|
2660
|
-
}
|
2661
|
-
logger.Debug().Str("prepared", store.url.toString()).Msg("produced");
|
2662
|
-
return store;
|
2663
|
-
});
|
2664
|
-
}
|
2665
|
-
var onceLoadMetaGateway = new import_cement8.KeyedResolvOnce();
|
2666
|
-
function loadMetaGateway(url2, logger) {
|
2667
|
-
return onceLoadMetaGateway.get(url2.protocol).once(async () => {
|
2668
|
-
return runStoreFactory(url2, logger, async (item) => item.meta(logger));
|
2820
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
2821
|
+
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
2822
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2823
|
+
if (!gateway) {
|
2824
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2825
|
+
}
|
2826
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2827
|
+
gateway: gateway.gateway,
|
2828
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2829
|
+
...loader.ebOpts.keyBag
|
2830
|
+
})
|
2669
2831
|
});
|
2832
|
+
return store;
|
2670
2833
|
}
|
2671
|
-
var onceMetaStoreFactory = new import_cement8.KeyedResolvOnce();
|
2672
2834
|
async function metaStoreFactory(loader) {
|
2673
|
-
const
|
2674
|
-
|
2675
|
-
|
2676
|
-
|
2677
|
-
|
2678
|
-
logger.
|
2679
|
-
|
2680
|
-
|
2681
|
-
|
2682
|
-
|
2683
|
-
|
2684
|
-
}
|
2685
|
-
return store;
|
2686
|
-
});
|
2687
|
-
}
|
2688
|
-
var onceWalGateway = new import_cement8.KeyedResolvOnce();
|
2689
|
-
function loadWalGateway(url2, logger) {
|
2690
|
-
return onceWalGateway.get(url2.protocol).once(async () => {
|
2691
|
-
return runStoreFactory(url2, logger, async (item) => item.wal(logger));
|
2835
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
2836
|
+
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
2837
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2838
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2839
|
+
if (!gateway) {
|
2840
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2841
|
+
}
|
2842
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2843
|
+
gateway: gateway.gateway,
|
2844
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2845
|
+
...loader.ebOpts.keyBag
|
2846
|
+
})
|
2692
2847
|
});
|
2848
|
+
return store;
|
2693
2849
|
}
|
2694
|
-
var onceRemoteWalFactory = new import_cement8.KeyedResolvOnce();
|
2695
2850
|
async function remoteWalFactory(loader) {
|
2696
|
-
const
|
2697
|
-
|
2698
|
-
const
|
2699
|
-
|
2700
|
-
|
2701
|
-
|
2702
|
-
|
2703
|
-
|
2704
|
-
|
2705
|
-
|
2706
|
-
|
2707
|
-
}
|
2708
|
-
return store;
|
2851
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
2852
|
+
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
2853
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2854
|
+
if (!gateway) {
|
2855
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2856
|
+
}
|
2857
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2858
|
+
const store = new WALStoreImpl(loader, url, {
|
2859
|
+
gateway: gateway.gateway,
|
2860
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2861
|
+
...loader.ebOpts.keyBag
|
2862
|
+
})
|
2709
2863
|
});
|
2864
|
+
return store;
|
2710
2865
|
}
|
2711
|
-
async function testStoreFactory(
|
2712
|
-
|
2713
|
-
|
2714
|
-
|
2715
|
-
|
2716
|
-
|
2717
|
-
|
2718
|
-
|
2866
|
+
async function testStoreFactory(url, sthis) {
|
2867
|
+
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
2868
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2869
|
+
if (!gateway) {
|
2870
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2871
|
+
}
|
2872
|
+
return gateway.test;
|
2873
|
+
}
|
2874
|
+
async function ensureStart(store, logger) {
|
2875
|
+
const ret = await store.start();
|
2876
|
+
if (ret.isErr()) {
|
2877
|
+
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2878
|
+
}
|
2879
|
+
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2880
|
+
return store;
|
2719
2881
|
}
|
2720
|
-
function toStoreRuntime(opts,
|
2721
|
-
const logger = ensureLogger(
|
2882
|
+
function toStoreRuntime(opts, sthis) {
|
2883
|
+
const logger = ensureLogger(sthis, "toStoreRuntime", {});
|
2722
2884
|
return {
|
2723
|
-
makeMetaStore: (loader) => {
|
2885
|
+
makeMetaStore: async (loader) => {
|
2724
2886
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
|
2725
|
-
return (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader);
|
2887
|
+
return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
|
2726
2888
|
},
|
2727
|
-
makeDataStore: (loader) => {
|
2889
|
+
makeDataStore: async (loader) => {
|
2728
2890
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
|
2729
|
-
return (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader);
|
2891
|
+
return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
|
2730
2892
|
},
|
2731
|
-
|
2732
|
-
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.
|
2733
|
-
return (loader.ebOpts.store.
|
2893
|
+
makeWALStore: async (loader) => {
|
2894
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
|
2895
|
+
return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
|
2734
2896
|
},
|
2735
2897
|
encodeFile: opts.encodeFile || encodeFile,
|
2736
2898
|
decodeFile: opts.decodeFile || decodeFile
|
@@ -2738,48 +2900,202 @@ function toStoreRuntime(opts, ilogger) {
|
|
2738
2900
|
}
|
2739
2901
|
registerStoreProtocol({
|
2740
2902
|
protocol: "file:",
|
2741
|
-
|
2742
|
-
const {
|
2743
|
-
return new
|
2744
|
-
},
|
2745
|
-
meta: async (logger) => {
|
2746
|
-
const { FileMetaGateway: FileMetaGateway2 } = await Promise.resolve().then(() => (init_store_file(), store_file_exports));
|
2747
|
-
return new FileMetaGateway2(logger);
|
2748
|
-
},
|
2749
|
-
wal: async (logger) => {
|
2750
|
-
const { FileWALGateway: FileWALGateway2 } = await Promise.resolve().then(() => (init_store_file(), store_file_exports));
|
2751
|
-
return new FileWALGateway2(logger);
|
2903
|
+
gateway: async (logger) => {
|
2904
|
+
const { FileGateway: FileGateway2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
2905
|
+
return new FileGateway2(logger);
|
2752
2906
|
},
|
2753
2907
|
test: async (logger) => {
|
2754
|
-
const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (
|
2908
|
+
const { FileTestStore: FileTestStore2 } = await Promise.resolve().then(() => (init_gateway(), gateway_exports));
|
2755
2909
|
return new FileTestStore2(logger);
|
2756
2910
|
}
|
2757
2911
|
});
|
2758
2912
|
registerStoreProtocol({
|
2759
2913
|
protocol: "indexdb:",
|
2760
|
-
|
2761
|
-
const {
|
2762
|
-
return new
|
2763
|
-
},
|
2764
|
-
meta: async (logger) => {
|
2765
|
-
const { IndexDBMetaGateway: IndexDBMetaGateway2 } = await Promise.resolve().then(() => (init_store_indexdb(), store_indexdb_exports));
|
2766
|
-
return new IndexDBMetaGateway2(logger);
|
2767
|
-
},
|
2768
|
-
wal: async (logger) => {
|
2769
|
-
const { IndexDBMetaGateway: IndexDBMetaGateway2 } = await Promise.resolve().then(() => (init_store_indexdb(), store_indexdb_exports));
|
2770
|
-
return new IndexDBMetaGateway2(logger);
|
2914
|
+
gateway: async (logger) => {
|
2915
|
+
const { IndexDBGateway: IndexDBGateway2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
2916
|
+
return new IndexDBGateway2(logger);
|
2771
2917
|
},
|
2772
2918
|
test: async (logger) => {
|
2773
|
-
const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (
|
2919
|
+
const { IndexDBTestStore: IndexDBTestStore2 } = await Promise.resolve().then(() => (init_gateway2(), gateway_exports2));
|
2774
2920
|
return new IndexDBTestStore2(logger);
|
2775
2921
|
}
|
2776
2922
|
});
|
2777
2923
|
|
2778
|
-
// src/blockstore/
|
2779
|
-
|
2924
|
+
// src/blockstore/connection-base.ts
|
2925
|
+
var import_clock = require("@web3-storage/pail/clock");
|
2926
|
+
var import_block6 = require("@web3-storage/pail/block");
|
2927
|
+
|
2928
|
+
// src/blockstore/task-manager.ts
|
2929
|
+
init_utils();
|
2930
|
+
var TaskManager = class {
|
2931
|
+
constructor(loader) {
|
2932
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
2933
|
+
this.queue = [];
|
2934
|
+
this.isProcessing = false;
|
2935
|
+
this.loader = loader;
|
2936
|
+
this.logger = ensureLogger(loader.sthis, "TaskManager");
|
2937
|
+
}
|
2938
|
+
async handleEvent(eventBlock) {
|
2939
|
+
const cid = eventBlock.cid.toString();
|
2940
|
+
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
2941
|
+
for (const parent of parents) {
|
2942
|
+
this.eventsWeHandled.add(parent);
|
2943
|
+
}
|
2944
|
+
this.queue.push({ cid, eventBlock, retries: 0 });
|
2945
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
2946
|
+
void this.processQueue();
|
2947
|
+
}
|
2948
|
+
async processQueue() {
|
2949
|
+
if (this.isProcessing) return;
|
2950
|
+
this.isProcessing = true;
|
2951
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2952
|
+
const first = filteredQueue[0];
|
2953
|
+
if (!first) {
|
2954
|
+
return;
|
2955
|
+
}
|
2956
|
+
try {
|
2957
|
+
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
2958
|
+
this.eventsWeHandled.add(first.cid);
|
2959
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
2960
|
+
} catch (err) {
|
2961
|
+
if (first.retries++ > 3) {
|
2962
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
2963
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
2964
|
+
}
|
2965
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
2966
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
2967
|
+
} finally {
|
2968
|
+
this.isProcessing = false;
|
2969
|
+
if (this.queue.length > 0) {
|
2970
|
+
void this.processQueue();
|
2971
|
+
}
|
2972
|
+
}
|
2973
|
+
}
|
2974
|
+
};
|
2975
|
+
|
2976
|
+
// src/blockstore/store-remote.ts
|
2977
|
+
async function RemoteDataStore(sthis, name, url, opts) {
|
2978
|
+
const ds = new DataStoreImpl(sthis, name, url, opts);
|
2979
|
+
await ds.start();
|
2980
|
+
return ds;
|
2981
|
+
}
|
2982
|
+
async function RemoteMetaStore(sthis, name, url, opts) {
|
2983
|
+
const ms = new MetaStoreImpl(sthis, name, url, opts);
|
2984
|
+
await ms.start();
|
2985
|
+
return ms;
|
2986
|
+
}
|
2987
|
+
|
2988
|
+
// src/blockstore/connection-base.ts
|
2989
|
+
var ConnectionBase = class {
|
2990
|
+
constructor(url, logger) {
|
2991
|
+
// readonly ready: Promise<unknown>;
|
2992
|
+
// todo move to LRU blockstore https://github.com/web3-storage/w3clock/blob/main/src/worker/block.js
|
2993
|
+
this.eventBlocks = new import_block6.MemoryBlockstore();
|
2994
|
+
this.parents = [];
|
2995
|
+
this.loaded = Promise.resolve();
|
2996
|
+
this.logger = logger;
|
2997
|
+
this.url = url;
|
2998
|
+
}
|
2999
|
+
async refresh() {
|
3000
|
+
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load("main");
|
3001
|
+
await (await throwFalsy(this.loader).WALStore()).process();
|
3002
|
+
}
|
3003
|
+
async connect_X({ loader }) {
|
3004
|
+
if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
|
3005
|
+
await this.connectMeta_X({ loader });
|
3006
|
+
await this.connectStorage_X({ loader });
|
3007
|
+
}
|
3008
|
+
async connectMeta_X({ loader }) {
|
3009
|
+
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
3010
|
+
this.loader = loader;
|
3011
|
+
this.taskManager = new TaskManager(loader);
|
3012
|
+
await this.onConnect();
|
3013
|
+
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
3014
|
+
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
3015
|
+
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
3016
|
+
const name = metaUrl.toString();
|
3017
|
+
const remote = await RemoteMetaStore(loader.sthis, name, metaUrl, {
|
3018
|
+
gateway: gateway.gateway,
|
3019
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag)
|
3020
|
+
});
|
3021
|
+
remote.onLoad("main", async (metas) => {
|
3022
|
+
if (metas) {
|
3023
|
+
this.logger.Debug().Any("metas", metas).Bool("loader", this.loader).Msg("connectMeta_X: handleDbMetasFromStore pre");
|
3024
|
+
await throwFalsy(this.loader).handleDbMetasFromStore(metas);
|
3025
|
+
this.logger.Debug().Any("metas", metas).Msg("connectMeta_X: handleDbMetasFromStore post");
|
3026
|
+
}
|
3027
|
+
});
|
3028
|
+
this.loader.remoteMetaStore = remote;
|
3029
|
+
this.loaded = this.loader.ready().then(async () => {
|
3030
|
+
remote.load("main").then(async () => {
|
3031
|
+
(await throwFalsy(this.loader).WALStore()).process();
|
3032
|
+
});
|
3033
|
+
});
|
3034
|
+
}
|
3035
|
+
async connectStorage_X({ loader }) {
|
3036
|
+
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
3037
|
+
this.loader = loader;
|
3038
|
+
const dataUrl = this.url.build().defParam("store", "data").URI();
|
3039
|
+
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
3040
|
+
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
3041
|
+
const name = dataUrl.toString();
|
3042
|
+
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
3043
|
+
gateway: gateway.gateway,
|
3044
|
+
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
3045
|
+
});
|
3046
|
+
loader.remoteFileStore = loader.remoteCarStore;
|
3047
|
+
}
|
3048
|
+
async createEventBlock(bytes) {
|
3049
|
+
const data = {
|
3050
|
+
dbMeta: bytes
|
3051
|
+
};
|
3052
|
+
const event = await import_clock.EventBlock.create(
|
3053
|
+
data,
|
3054
|
+
this.parents
|
3055
|
+
);
|
3056
|
+
await this.eventBlocks.put(event.cid, event.bytes);
|
3057
|
+
return event;
|
3058
|
+
}
|
3059
|
+
async decodeEventBlock(bytes) {
|
3060
|
+
const event = await (0, import_clock.decodeEventBlock)(bytes);
|
3061
|
+
return event;
|
3062
|
+
}
|
3063
|
+
// move this stuff to connect
|
3064
|
+
// async getDashboardURL(compact = true) {
|
3065
|
+
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
3066
|
+
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
3067
|
+
// // if (compact) {
|
3068
|
+
// // await this.compact()
|
3069
|
+
// // }
|
3070
|
+
// const currents = await this.loader?.metaStore?.load()
|
3071
|
+
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
3072
|
+
// if (currents.length > 1)
|
3073
|
+
// throw new Error("Can't sync database with split heads: make an update first")
|
3074
|
+
// const current = currents[0]
|
3075
|
+
// const params = {
|
3076
|
+
// car: current.car.toString()
|
3077
|
+
// }
|
3078
|
+
// if (current.key) {
|
3079
|
+
// // @ts-ignore
|
3080
|
+
// params.key = current.key.toString()
|
3081
|
+
// }
|
3082
|
+
// // @ts-ignore
|
3083
|
+
// if (this.name) {
|
3084
|
+
// // @ts-ignore
|
3085
|
+
// params.name = this.name
|
3086
|
+
// }
|
3087
|
+
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
3088
|
+
// console.log('Import to dashboard: ' + url.toString())
|
3089
|
+
// return url
|
3090
|
+
// }
|
3091
|
+
// openDashboard() {
|
3092
|
+
// void this.getDashboardURL().then(url => {
|
3093
|
+
// if (url) window.open(url.toString(), '_blank')
|
3094
|
+
// })
|
3095
|
+
// }
|
3096
|
+
};
|
2780
3097
|
|
2781
3098
|
// src/crdt-helpers.ts
|
2782
|
-
init_types();
|
2783
3099
|
function time(tag) {
|
2784
3100
|
}
|
2785
3101
|
function timeEnd(tag) {
|
@@ -2827,7 +3143,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
2827
3143
|
await processFiles(store, blocks, update.value, logger);
|
2828
3144
|
value = { doc: update.value };
|
2829
3145
|
}
|
2830
|
-
const block = await
|
3146
|
+
const block = await encode({ value, hasher: import_sha25.sha256, codec });
|
2831
3147
|
blocks.putSync(block.cid, block.bytes);
|
2832
3148
|
return block.cid;
|
2833
3149
|
}
|
@@ -2836,10 +3152,16 @@ async function processFiles(store, blocks, doc, logger) {
|
|
2836
3152
|
await processFileset(logger, store, blocks, doc._files);
|
2837
3153
|
}
|
2838
3154
|
if (doc._publicFiles) {
|
2839
|
-
await processFileset(
|
3155
|
+
await processFileset(
|
3156
|
+
logger,
|
3157
|
+
store,
|
3158
|
+
blocks,
|
3159
|
+
doc._publicFiles
|
3160
|
+
/*, true*/
|
3161
|
+
);
|
2840
3162
|
}
|
2841
3163
|
}
|
2842
|
-
async function processFileset(logger, store, blocks, files
|
3164
|
+
async function processFileset(logger, store, blocks, files) {
|
2843
3165
|
const dbBlockstore = blocks.parent;
|
2844
3166
|
if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
|
2845
3167
|
const t = new CarTransaction(dbBlockstore);
|
@@ -2861,9 +3183,10 @@ async function processFileset(logger, store, blocks, files, publicFiles = false)
|
|
2861
3183
|
}
|
2862
3184
|
}
|
2863
3185
|
if (didPut.length) {
|
2864
|
-
const car = await dbBlockstore.loader.commitFiles(
|
2865
|
-
|
2866
|
-
|
3186
|
+
const car = await dbBlockstore.loader.commitFiles(
|
3187
|
+
t,
|
3188
|
+
{ files }
|
3189
|
+
);
|
2867
3190
|
if (car) {
|
2868
3191
|
for (const name of didPut) {
|
2869
3192
|
files[name] = { car, ...files[name] };
|
@@ -2897,7 +3220,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2897
3220
|
fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
|
2898
3221
|
{
|
2899
3222
|
get: async (cid) => {
|
2900
|
-
return await blocks.getFile(throwFalsy(fileMeta.car), cid
|
3223
|
+
return await blocks.getFile(throwFalsy(fileMeta.car), cid);
|
2901
3224
|
}
|
2902
3225
|
},
|
2903
3226
|
fileMeta.cid,
|
@@ -2911,7 +3234,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
2911
3234
|
async function getValueFromLink(blocks, link, logger) {
|
2912
3235
|
const block = await blocks.get(link);
|
2913
3236
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
2914
|
-
const { value } = await
|
3237
|
+
const { value } = await decode({ bytes: block.bytes, hasher: import_sha25.sha256, codec });
|
2915
3238
|
const cvalue = {
|
2916
3239
|
...value,
|
2917
3240
|
cid: link
|
@@ -2920,17 +3243,21 @@ async function getValueFromLink(blocks, link, logger) {
|
|
2920
3243
|
return cvalue;
|
2921
3244
|
}
|
2922
3245
|
var DirtyEventFetcher = class extends import_clock2.EventFetcher {
|
3246
|
+
constructor(logger, blocks) {
|
3247
|
+
super(blocks);
|
3248
|
+
this.logger = logger;
|
3249
|
+
}
|
2923
3250
|
async get(link) {
|
2924
3251
|
try {
|
2925
3252
|
return super.get(link);
|
2926
3253
|
} catch (e) {
|
2927
|
-
|
3254
|
+
this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
|
2928
3255
|
return { value: void 0 };
|
2929
3256
|
}
|
2930
3257
|
}
|
2931
3258
|
};
|
2932
3259
|
async function clockChangesSince(blocks, head, since, opts, logger) {
|
2933
|
-
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new import_clock2.EventFetcher(blocks);
|
3260
|
+
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new import_clock2.EventFetcher(blocks);
|
2934
3261
|
const keys = /* @__PURE__ */ new Set();
|
2935
3262
|
const updates = await gatherUpdates(
|
2936
3263
|
blocks,
|
@@ -3028,38 +3355,34 @@ async function doCompact(blockLog, head, logger) {
|
|
3028
3355
|
async function getBlock(blocks, cidString) {
|
3029
3356
|
const block = await blocks.get((0, import_link.parse)(cidString));
|
3030
3357
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
3031
|
-
const { cid, value } = await
|
3032
|
-
return new
|
3358
|
+
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: import_sha25.sha256 });
|
3359
|
+
return new Block({ cid, value, bytes: block.bytes });
|
3033
3360
|
}
|
3034
3361
|
|
3035
|
-
// src/indexer.ts
|
3036
|
-
init_types();
|
3037
|
-
|
3038
3362
|
// src/indexer-helpers.ts
|
3039
|
-
var
|
3040
|
-
var
|
3041
|
-
var codec3 = __toESM(require("@ipld/dag-cbor"), 1);
|
3363
|
+
var import_sha26 = require("multiformats/hashes/sha2");
|
3364
|
+
var codec2 = __toESM(require("@ipld/dag-cbor"), 1);
|
3042
3365
|
var import_charwise = __toESM(require("charwise"), 1);
|
3043
3366
|
var DbIndex = __toESM(require("prolly-trees/db-index"), 1);
|
3044
|
-
var
|
3045
|
-
var
|
3367
|
+
var import_utils15 = require("prolly-trees/utils");
|
3368
|
+
var import_cache = require("prolly-trees/cache");
|
3046
3369
|
var IndexTree = class {
|
3047
3370
|
};
|
3048
3371
|
function refCompare(aRef, bRef) {
|
3049
3372
|
if (Number.isNaN(aRef)) return -1;
|
3050
3373
|
if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
|
3051
3374
|
if (aRef === Infinity) return 1;
|
3052
|
-
return (0,
|
3375
|
+
return (0, import_utils15.simpleCompare)(aRef, bRef);
|
3053
3376
|
}
|
3054
3377
|
function compare(a, b) {
|
3055
3378
|
const [aKey, aRef] = a;
|
3056
3379
|
const [bKey, bRef] = b;
|
3057
|
-
const comp = (0,
|
3380
|
+
const comp = (0, import_utils15.simpleCompare)(aKey, bKey);
|
3058
3381
|
if (comp !== 0) return comp;
|
3059
3382
|
return refCompare(aRef, bRef);
|
3060
3383
|
}
|
3061
|
-
var byKeyOpts = { cache:
|
3062
|
-
var byIdOpts = { cache:
|
3384
|
+
var byKeyOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare };
|
3385
|
+
var byIdOpts = { cache: import_cache.nocache, chunker: (0, import_utils15.bf)(30), codec: codec2, hasher: import_sha26.sha256, compare: import_utils15.simpleCompare };
|
3063
3386
|
function indexEntriesForChanges(changes, mapFn) {
|
3064
3387
|
const indexEntries = [];
|
3065
3388
|
changes.forEach(({ id: key, value, del }) => {
|
@@ -3087,7 +3410,7 @@ function makeProllyGetBlock(blocks) {
|
|
3087
3410
|
const block = await blocks.get(address);
|
3088
3411
|
if (!block) throw new Error(`Missing block ${address.toString()}`);
|
3089
3412
|
const { cid, bytes } = block;
|
3090
|
-
return
|
3413
|
+
return create({ cid, bytes, hasher: import_sha26.sha256, codec: codec2 });
|
3091
3414
|
};
|
3092
3415
|
}
|
3093
3416
|
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
@@ -3159,25 +3482,25 @@ function encodeKey(key) {
|
|
3159
3482
|
|
3160
3483
|
// src/indexer.ts
|
3161
3484
|
init_utils();
|
3162
|
-
function index({ _crdt }, name, mapFn, meta) {
|
3485
|
+
function index(sthis, { _crdt }, name, mapFn, meta) {
|
3163
3486
|
if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
3164
3487
|
if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
3165
3488
|
if (_crdt.indexers.has(name)) {
|
3166
3489
|
const idx = _crdt.indexers.get(name);
|
3167
3490
|
idx.applyMapFn(name, mapFn, meta);
|
3168
3491
|
} else {
|
3169
|
-
const idx = new Index(_crdt, name, mapFn, meta);
|
3492
|
+
const idx = new Index(sthis, _crdt, name, mapFn, meta);
|
3170
3493
|
_crdt.indexers.set(name, idx);
|
3171
3494
|
}
|
3172
3495
|
return _crdt.indexers.get(name);
|
3173
3496
|
}
|
3174
3497
|
var Index = class {
|
3175
|
-
constructor(crdt, name, mapFn, meta) {
|
3498
|
+
constructor(sthis, crdt, name, mapFn, meta) {
|
3176
3499
|
this.mapFnString = "";
|
3177
3500
|
this.byKey = new IndexTree();
|
3178
3501
|
this.byId = new IndexTree();
|
3179
3502
|
this.includeDocsDefault = false;
|
3180
|
-
this.logger = ensureLogger(
|
3503
|
+
this.logger = ensureLogger(sthis, "Index");
|
3181
3504
|
this.blockstore = crdt.indexBlockstore;
|
3182
3505
|
this.crdt = crdt;
|
3183
3506
|
this.applyMapFn(name, mapFn, meta);
|
@@ -3232,7 +3555,7 @@ var Index = class {
|
|
3232
3555
|
}
|
3233
3556
|
if (this.mapFnString) {
|
3234
3557
|
if (this.mapFnString !== mapFn.toString()) {
|
3235
|
-
throw this.logger.Error().Msg("cannot apply different mapFn app").AsError();
|
3558
|
+
throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
|
3236
3559
|
}
|
3237
3560
|
} else {
|
3238
3561
|
this.mapFnString = mapFn.toString();
|
@@ -3367,8 +3690,7 @@ var Index = class {
|
|
3367
3690
|
// src/crdt-clock.ts
|
3368
3691
|
var import_clock3 = require("@web3-storage/pail/clock");
|
3369
3692
|
var import_crdt2 = require("@web3-storage/pail/crdt");
|
3370
|
-
var
|
3371
|
-
init_types();
|
3693
|
+
var import_cement11 = require("@adviser/cement");
|
3372
3694
|
|
3373
3695
|
// src/apply-head-queue.ts
|
3374
3696
|
function applyHeadQueue(worker, logger) {
|
@@ -3426,9 +3748,9 @@ var CRDTClock = class {
|
|
3426
3748
|
this.zoomers = /* @__PURE__ */ new Set();
|
3427
3749
|
this.watchers = /* @__PURE__ */ new Set();
|
3428
3750
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
3429
|
-
this._ready = new
|
3751
|
+
this._ready = new import_cement11.ResolveOnce();
|
3430
3752
|
this.blockstore = blockstore;
|
3431
|
-
this.logger = ensureLogger(blockstore.
|
3753
|
+
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
3432
3754
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
3433
3755
|
}
|
3434
3756
|
async ready() {
|
@@ -3501,7 +3823,7 @@ var CRDTClock = class {
|
|
3501
3823
|
}
|
3502
3824
|
return { head: advancedHead };
|
3503
3825
|
},
|
3504
|
-
{ noLoader }
|
3826
|
+
{ noLoader, add: false }
|
3505
3827
|
);
|
3506
3828
|
this.setHead(meta.head);
|
3507
3829
|
}
|
@@ -3536,13 +3858,14 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
3536
3858
|
// src/crdt.ts
|
3537
3859
|
init_utils();
|
3538
3860
|
var CRDT = class {
|
3539
|
-
constructor(name, opts = {}) {
|
3540
|
-
this.onceReady = new import_cement10.ResolveOnce();
|
3861
|
+
constructor(sthis, name, opts = {}) {
|
3541
3862
|
this.indexers = /* @__PURE__ */ new Map();
|
3863
|
+
this.onceReady = new import_cement12.ResolveOnce();
|
3864
|
+
this.sthis = sthis;
|
3542
3865
|
this.name = name;
|
3543
|
-
this.logger = ensureLogger(
|
3866
|
+
this.logger = ensureLogger(sthis, "CRDT");
|
3544
3867
|
this.opts = opts;
|
3545
|
-
this.blockstore = blockstoreFactory({
|
3868
|
+
this.blockstore = blockstoreFactory(sthis, {
|
3546
3869
|
name,
|
3547
3870
|
applyMeta: async (meta) => {
|
3548
3871
|
const crdtMeta = meta;
|
@@ -3554,22 +3877,20 @@ var CRDT = class {
|
|
3554
3877
|
return { head: this.clock.head };
|
3555
3878
|
},
|
3556
3879
|
autoCompact: this.opts.autoCompact || 100,
|
3557
|
-
crypto: this.opts.crypto,
|
3558
3880
|
store: { ...this.opts.store, isIndex: void 0 },
|
3559
3881
|
public: this.opts.public,
|
3560
3882
|
meta: this.opts.meta,
|
3561
3883
|
threshold: this.opts.threshold
|
3562
3884
|
});
|
3563
|
-
this.indexBlockstore = blockstoreFactory({
|
3885
|
+
this.indexBlockstore = blockstoreFactory(sthis, {
|
3564
3886
|
name,
|
3565
3887
|
applyMeta: async (meta) => {
|
3566
3888
|
const idxCarMeta = meta;
|
3567
3889
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
3568
3890
|
for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
|
3569
|
-
index({ _crdt: this }, name2, void 0, idx);
|
3891
|
+
index(this.sthis, { _crdt: this }, name2, void 0, idx);
|
3570
3892
|
}
|
3571
3893
|
},
|
3572
|
-
crypto: this.opts.crypto,
|
3573
3894
|
store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
|
3574
3895
|
public: this.opts.public
|
3575
3896
|
});
|
@@ -3580,17 +3901,6 @@ var CRDT = class {
|
|
3580
3901
|
}
|
3581
3902
|
});
|
3582
3903
|
}
|
3583
|
-
async ready() {
|
3584
|
-
return this.onceReady.once(async () => {
|
3585
|
-
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3586
|
-
});
|
3587
|
-
}
|
3588
|
-
async close() {
|
3589
|
-
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
3590
|
-
}
|
3591
|
-
async destroy() {
|
3592
|
-
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
3593
|
-
}
|
3594
3904
|
async bulk(updates) {
|
3595
3905
|
await this.ready();
|
3596
3906
|
const prevHead = [...this.clock.head];
|
@@ -3611,6 +3921,21 @@ var CRDT = class {
|
|
3611
3921
|
await this.clock.applyHead(done.meta.head, prevHead, updates);
|
3612
3922
|
return done.meta;
|
3613
3923
|
}
|
3924
|
+
async ready() {
|
3925
|
+
return this.onceReady.once(async () => {
|
3926
|
+
try {
|
3927
|
+
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3928
|
+
} catch (e) {
|
3929
|
+
throw this.logger.Error().Err(e).Msg("CRDT not ready").AsError();
|
3930
|
+
}
|
3931
|
+
});
|
3932
|
+
}
|
3933
|
+
async close() {
|
3934
|
+
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
3935
|
+
}
|
3936
|
+
async destroy() {
|
3937
|
+
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
3938
|
+
}
|
3614
3939
|
// if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
|
3615
3940
|
async allDocs() {
|
3616
3941
|
await this.ready();
|
@@ -3649,20 +3974,19 @@ var CRDT = class {
|
|
3649
3974
|
};
|
3650
3975
|
|
3651
3976
|
// src/database.ts
|
3652
|
-
init_sys_container();
|
3653
3977
|
init_utils();
|
3654
|
-
init_gateway();
|
3655
3978
|
var Database = class {
|
3656
3979
|
constructor(name, opts) {
|
3657
3980
|
this.opts = {};
|
3658
3981
|
this._listening = false;
|
3659
3982
|
this._listeners = /* @__PURE__ */ new Set();
|
3660
3983
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
3661
|
-
this._ready = new
|
3984
|
+
this._ready = new import_cement13.ResolveOnce();
|
3662
3985
|
this.name = name;
|
3663
3986
|
this.opts = opts || this.opts;
|
3664
|
-
this.
|
3665
|
-
this.
|
3987
|
+
this.sthis = ensureSuperThis(this.opts);
|
3988
|
+
this.logger = ensureLogger(this.sthis, "Database");
|
3989
|
+
this._crdt = new CRDT(this.sthis, name, this.opts);
|
3666
3990
|
this.blockstore = this._crdt.blockstore;
|
3667
3991
|
this._writeQueue = writeQueue(async (updates) => {
|
3668
3992
|
return await this._crdt.bulk(updates);
|
@@ -3686,7 +4010,7 @@ var Database = class {
|
|
3686
4010
|
}
|
3687
4011
|
async ready() {
|
3688
4012
|
return this._ready.once(async () => {
|
3689
|
-
await
|
4013
|
+
await this.sthis.start();
|
3690
4014
|
await this._crdt.ready();
|
3691
4015
|
await this.blockstore.ready();
|
3692
4016
|
});
|
@@ -3706,7 +4030,7 @@ var Database = class {
|
|
3706
4030
|
await this.ready();
|
3707
4031
|
this.logger.Debug().Str("id", doc._id).Msg("put");
|
3708
4032
|
const { _id, ...value } = doc;
|
3709
|
-
const docId = _id ||
|
4033
|
+
const docId = _id || this.sthis.nextId().str;
|
3710
4034
|
const result = await this._writeQueue.push({
|
3711
4035
|
id: docId,
|
3712
4036
|
value: {
|
@@ -3771,7 +4095,7 @@ var Database = class {
|
|
3771
4095
|
await this.ready();
|
3772
4096
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
3773
4097
|
const _crdt = this._crdt;
|
3774
|
-
const idx = typeof field === "string" ? index({ _crdt }, field) : index({ _crdt }, makeName(field.toString()), field);
|
4098
|
+
const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
|
3775
4099
|
return await idx.query(opts);
|
3776
4100
|
}
|
3777
4101
|
async compact() {
|
@@ -3808,12 +4132,7 @@ function fireproof(name, opts) {
|
|
3808
4132
|
const key = JSON.stringify(
|
3809
4133
|
toSortedArray({
|
3810
4134
|
name,
|
3811
|
-
stores: toSortedArray(opts?.store?.stores)
|
3812
|
-
makeMetaStore: !!opts?.store?.makeMetaStore,
|
3813
|
-
makeDataStore: !!opts?.store?.makeDataStore,
|
3814
|
-
makeRemoteWAL: !!opts?.store?.makeRemoteWAL,
|
3815
|
-
encodeFile: !!opts?.store?.encodeFile,
|
3816
|
-
decodeFile: !!opts?.store?.decodeFile
|
4135
|
+
stores: toSortedArray(opts?.store?.stores)
|
3817
4136
|
})
|
3818
4137
|
);
|
3819
4138
|
let db = Database.databases.get(key);
|
@@ -3828,7 +4147,10 @@ function makeName(fnString) {
|
|
3828
4147
|
let found = null;
|
3829
4148
|
const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
|
3830
4149
|
if (matches.length === 0) {
|
3831
|
-
found = /=>\s*(
|
4150
|
+
found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
|
4151
|
+
if (found && found[1].includes("return")) {
|
4152
|
+
found = null;
|
4153
|
+
}
|
3832
4154
|
}
|
3833
4155
|
if (!found) {
|
3834
4156
|
return fnString;
|
@@ -3837,14 +4159,41 @@ function makeName(fnString) {
|
|
3837
4159
|
}
|
3838
4160
|
}
|
3839
4161
|
|
4162
|
+
// src/runtime/index.ts
|
4163
|
+
var runtime_exports = {};
|
4164
|
+
__export(runtime_exports, {
|
4165
|
+
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
4166
|
+
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
4167
|
+
files: () => files_exports,
|
4168
|
+
getFileName: () => getFileName,
|
4169
|
+
getFileSystem: () => getFileSystem,
|
4170
|
+
getPath: () => getPath,
|
4171
|
+
kb: () => key_bag_exports,
|
4172
|
+
kc: () => keyed_crypto_exports,
|
4173
|
+
mf: () => wait_pr_multiformats_exports,
|
4174
|
+
toArrayBuffer: () => toArrayBuffer
|
4175
|
+
});
|
4176
|
+
init_utils2();
|
4177
|
+
|
4178
|
+
// src/runtime/wait-pr-multiformats/index.ts
|
4179
|
+
var wait_pr_multiformats_exports = {};
|
4180
|
+
__export(wait_pr_multiformats_exports, {
|
4181
|
+
block: () => block_exports,
|
4182
|
+
codec: () => codec_interface_exports
|
4183
|
+
});
|
4184
|
+
|
4185
|
+
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
4186
|
+
var codec_interface_exports = {};
|
4187
|
+
|
4188
|
+
// src/runtime/index.ts
|
4189
|
+
init_version();
|
4190
|
+
init_version2();
|
4191
|
+
|
3840
4192
|
// src/index.ts
|
3841
|
-
init_types();
|
3842
|
-
init_runtime();
|
3843
|
-
init_runtime();
|
3844
4193
|
init_utils();
|
3845
4194
|
|
3846
4195
|
// src/version.ts
|
3847
4196
|
var PACKAGE_VERSION = Object.keys({
|
3848
|
-
"0.19.
|
4197
|
+
"0.19.9-dev-frag": "xxxx"
|
3849
4198
|
})[0];
|
3850
4199
|
//# sourceMappingURL=index.cjs.map
|