@fireproof/core 0.19.8-dev-global → 0.19.11-dev-dryrun
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/chunk-7EWIAXTM.js +7 -0
- package/chunk-7EWIAXTM.js.map +1 -0
- package/chunk-MAK4D54P.js +280 -0
- package/chunk-MAK4D54P.js.map +1 -0
- package/chunk-PB4BKL4O.js +7 -0
- package/chunk-PB4BKL4O.js.map +1 -0
- package/chunk-XINRLWR3.js +75 -0
- package/chunk-XINRLWR3.js.map +1 -0
- package/deno.json +20 -0
- package/{store-indexdb-WLRSICCB.js → gateway-7OM6OSYK.js} +49 -82
- package/gateway-7OM6OSYK.js.map +1 -0
- package/gateway-VWWKLHUI.js +144 -0
- package/gateway-VWWKLHUI.js.map +1 -0
- package/index.cjs +2351 -1825
- package/index.cjs.map +1 -1
- package/index.d.cts +659 -535
- package/index.d.ts +659 -535
- package/index.global.js +26315 -20709
- package/index.global.js.map +1 -1
- package/index.js +1700 -1058
- package/index.js.map +1 -1
- package/key-bag-file-DFMW6ZM6.js +54 -0
- package/key-bag-file-DFMW6ZM6.js.map +1 -0
- package/key-bag-indexdb-R2RWGSQ4.js +50 -0
- package/key-bag-indexdb-R2RWGSQ4.js.map +1 -0
- package/mem-filesystem-BZQZLUR6.js +41 -0
- package/mem-filesystem-BZQZLUR6.js.map +1 -0
- package/metafile-cjs.json +1 -1
- package/metafile-esm.json +1 -1
- package/metafile-iife.json +1 -1
- package/node-filesystem-7YZR3POJ.js +45 -0
- package/node-filesystem-7YZR3POJ.js.map +1 -0
- package/package.json +12 -8
- package/tests/blockstore/fragment-gateway.test.ts +107 -0
- package/tests/blockstore/keyed-crypto.test.ts +332 -0
- package/tests/blockstore/loader.test.ts +24 -19
- package/tests/blockstore/store.test.ts +51 -40
- package/tests/blockstore/transaction.test.ts +19 -15
- package/tests/fireproof/all-gateway.test.ts +395 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.car +0 -0
- package/tests/fireproof/cars/bafkreidxwt2nhvbl4fnqfw3ctlt6zbrir4kqwmjo5im6rf4q5si27kgo2i.js +316 -0
- package/tests/fireproof/config.test.ts +94 -78
- package/tests/fireproof/convert_uint8.py +27 -0
- package/tests/fireproof/crdt.test.ts +34 -28
- package/tests/fireproof/database.test.ts +22 -14
- package/tests/fireproof/fireproof.test.fixture.ts +133 -0
- package/tests/fireproof/fireproof.test.ts +331 -219
- package/tests/fireproof/hello.test.ts +34 -18
- package/tests/fireproof/indexer.test.ts +34 -27
- package/tests/fireproof/utils.test.ts +65 -0
- package/tests/helpers.ts +28 -57
- package/utils-AISQB3PB.js +14 -0
- package/utils-AISQB3PB.js.map +1 -0
- package/chunk-BNL4PVBF.js +0 -314
- package/chunk-BNL4PVBF.js.map +0 -1
- package/chunk-JW2QT6BF.js +0 -184
- package/chunk-JW2QT6BF.js.map +0 -1
- package/node-sys-container-MIEX6ELJ.js +0 -29
- package/node-sys-container-MIEX6ELJ.js.map +0 -1
- package/store-file-VJ6BI4II.js +0 -191
- package/store-file-VJ6BI4II.js.map +0 -1
- package/store-indexdb-WLRSICCB.js.map +0 -1
package/index.js
CHANGED
@@ -1,30 +1,33 @@
|
|
1
|
+
import {
|
2
|
+
FILESTORE_VERSION
|
3
|
+
} from "./chunk-7EWIAXTM.js";
|
4
|
+
import {
|
5
|
+
getFileName,
|
6
|
+
getFileSystem,
|
7
|
+
getPath,
|
8
|
+
toArrayBuffer
|
9
|
+
} from "./chunk-XINRLWR3.js";
|
10
|
+
import {
|
11
|
+
INDEXDB_VERSION
|
12
|
+
} from "./chunk-PB4BKL4O.js";
|
1
13
|
import {
|
2
14
|
NotFoundError,
|
3
15
|
Result,
|
16
|
+
UInt8ArrayEqual,
|
17
|
+
__export,
|
4
18
|
dataDir,
|
5
|
-
decodeFile,
|
6
|
-
encodeFile,
|
7
19
|
ensureLogger,
|
8
|
-
|
20
|
+
ensureSuperLog,
|
21
|
+
ensureSuperThis,
|
9
22
|
exceptionWrapper,
|
10
23
|
getKey,
|
11
24
|
getName,
|
12
25
|
getStore,
|
13
|
-
isNotFoundError
|
14
|
-
|
15
|
-
toCryptoOpts
|
16
|
-
} from "./chunk-BNL4PVBF.js";
|
17
|
-
import {
|
18
|
-
SysContainer,
|
19
|
-
__export,
|
20
|
-
falsyToUndef,
|
21
|
-
isFalsy,
|
22
|
-
throwFalsy
|
23
|
-
} from "./chunk-JW2QT6BF.js";
|
26
|
+
isNotFoundError
|
27
|
+
} from "./chunk-MAK4D54P.js";
|
24
28
|
|
25
29
|
// src/database.ts
|
26
|
-
import {
|
27
|
-
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
30
|
+
import { ResolveOnce as ResolveOnce6 } from "@adviser/cement";
|
28
31
|
|
29
32
|
// src/write-queue.ts
|
30
33
|
function writeQueue(worker, payload = Infinity, unbounded = false) {
|
@@ -67,13 +70,83 @@ function writeQueue(worker, payload = Infinity, unbounded = false) {
|
|
67
70
|
}
|
68
71
|
|
69
72
|
// src/crdt.ts
|
70
|
-
import { ResolveOnce as
|
73
|
+
import { ResolveOnce as ResolveOnce5 } from "@adviser/cement";
|
74
|
+
|
75
|
+
// src/runtime/wait-pr-multiformats/block.ts
|
76
|
+
var block_exports = {};
|
77
|
+
__export(block_exports, {
|
78
|
+
Block: () => Block,
|
79
|
+
create: () => create,
|
80
|
+
createUnsafe: () => createUnsafe,
|
81
|
+
decode: () => decode,
|
82
|
+
encode: () => encode
|
83
|
+
});
|
84
|
+
import { bytes as binary, CID } from "multiformats";
|
85
|
+
import { Block as mfBlock } from "multiformats/block";
|
86
|
+
var Block = mfBlock;
|
87
|
+
async function decode({
|
88
|
+
bytes,
|
89
|
+
codec: codec3,
|
90
|
+
hasher: hasher7
|
91
|
+
}) {
|
92
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
93
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
94
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
95
|
+
const hash = await hasher7.digest(bytes);
|
96
|
+
const cid = CID.create(1, codec3.code, hash);
|
97
|
+
return new mfBlock({ value, bytes, cid });
|
98
|
+
}
|
99
|
+
async function encode({
|
100
|
+
value,
|
101
|
+
codec: codec3,
|
102
|
+
hasher: hasher7
|
103
|
+
}) {
|
104
|
+
if (typeof value === "undefined") throw new Error('Missing required argument "value"');
|
105
|
+
if (codec3 == null || hasher7 == null) throw new Error("Missing required argument: codec or hasher");
|
106
|
+
const bytes = await Promise.resolve(codec3.encode(value));
|
107
|
+
const hash = await hasher7.digest(bytes);
|
108
|
+
const cid = CID.create(1, codec3.code, hash);
|
109
|
+
return new mfBlock({ value, bytes, cid });
|
110
|
+
}
|
111
|
+
async function create({
|
112
|
+
bytes,
|
113
|
+
cid,
|
114
|
+
hasher: hasher7,
|
115
|
+
codec: codec3
|
116
|
+
}) {
|
117
|
+
if (bytes == null) throw new Error('Missing required argument "bytes"');
|
118
|
+
if (hasher7 == null) throw new Error('Missing required argument "hasher"');
|
119
|
+
const value = await Promise.resolve(codec3.decode(bytes));
|
120
|
+
const hash = await hasher7.digest(bytes);
|
121
|
+
if (!binary.equals(cid.multihash.bytes, hash.bytes)) {
|
122
|
+
throw new Error("CID hash does not match bytes");
|
123
|
+
}
|
124
|
+
return createUnsafe({
|
125
|
+
bytes,
|
126
|
+
cid,
|
127
|
+
value,
|
128
|
+
codec: codec3
|
129
|
+
});
|
130
|
+
}
|
131
|
+
async function createUnsafe({
|
132
|
+
bytes,
|
133
|
+
cid,
|
134
|
+
value: maybeValue,
|
135
|
+
codec: codec3
|
136
|
+
}) {
|
137
|
+
const value = await Promise.resolve(maybeValue !== void 0 ? maybeValue : codec3?.decode(bytes));
|
138
|
+
if (value === void 0) throw new Error('Missing required argument, must either provide "value" or "codec"');
|
139
|
+
return new Block({
|
140
|
+
cid,
|
141
|
+
bytes,
|
142
|
+
value
|
143
|
+
});
|
144
|
+
}
|
71
145
|
|
72
146
|
// src/crdt-helpers.ts
|
73
|
-
import {
|
74
|
-
import {
|
75
|
-
import
|
76
|
-
import * as codec2 from "@ipld/dag-cbor";
|
147
|
+
import { parse as parse3 } from "multiformats/link";
|
148
|
+
import { sha256 as hasher5 } from "multiformats/hashes/sha2";
|
149
|
+
import * as codec from "@ipld/dag-cbor";
|
77
150
|
import { put, get, entries, root } from "@web3-storage/pail/crdt";
|
78
151
|
import { EventFetcher, vis } from "@web3-storage/pail/clock";
|
79
152
|
import * as Batch from "@web3-storage/pail/crdt/batch";
|
@@ -84,512 +157,147 @@ __export(blockstore_exports, {
|
|
84
157
|
BaseBlockstore: () => BaseBlockstore,
|
85
158
|
CarTransaction: () => CarTransaction,
|
86
159
|
CompactionFetcher: () => CompactionFetcher,
|
87
|
-
ConnectREST: () => ConnectREST,
|
88
160
|
ConnectionBase: () => ConnectionBase,
|
89
|
-
DataStore: () => DataStore,
|
90
161
|
EncryptedBlockstore: () => EncryptedBlockstore,
|
91
|
-
|
162
|
+
FragmentGateway: () => FragmentGateway,
|
92
163
|
Loader: () => Loader,
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
isNotFoundError: () => isNotFoundError,
|
164
|
+
addCryptoKeyToGatewayMetaPayload: () => addCryptoKeyToGatewayMetaPayload,
|
165
|
+
ensureStart: () => ensureStart,
|
166
|
+
getGatewayFromURL: () => getGatewayFromURL,
|
97
167
|
parseCarFile: () => parseCarFile,
|
98
168
|
registerStoreProtocol: () => registerStoreProtocol,
|
169
|
+
setCryptoKeyFromGatewayMetaPayload: () => setCryptoKeyFromGatewayMetaPayload,
|
99
170
|
testStoreFactory: () => testStoreFactory,
|
100
|
-
|
101
|
-
|
171
|
+
toCIDBlock: () => toCIDBlock,
|
172
|
+
toStoreRuntime: () => toStoreRuntime
|
102
173
|
});
|
103
174
|
|
104
|
-
// src/blockstore/
|
105
|
-
|
106
|
-
|
175
|
+
// src/blockstore/types.ts
|
176
|
+
function toCIDBlock(block) {
|
177
|
+
return block;
|
178
|
+
}
|
107
179
|
|
108
|
-
// src/blockstore/
|
109
|
-
|
110
|
-
constructor(loader) {
|
111
|
-
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
112
|
-
this.queue = [];
|
113
|
-
this.isProcessing = false;
|
114
|
-
this.loader = loader;
|
115
|
-
this.logger = ensureLogger(loader.logger, "TaskManager");
|
116
|
-
}
|
117
|
-
async handleEvent(eventBlock) {
|
118
|
-
const cid = eventBlock.cid.toString();
|
119
|
-
const parents = eventBlock.value.parents.map((cid2) => cid2.toString());
|
120
|
-
for (const parent of parents) {
|
121
|
-
this.eventsWeHandled.add(parent);
|
122
|
-
}
|
123
|
-
this.queue.push({ cid, eventBlock, retries: 0 });
|
124
|
-
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
125
|
-
void this.processQueue();
|
126
|
-
}
|
127
|
-
async processQueue() {
|
128
|
-
if (this.isProcessing) return;
|
129
|
-
this.isProcessing = true;
|
130
|
-
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
131
|
-
const first = filteredQueue[0];
|
132
|
-
if (!first) {
|
133
|
-
return;
|
134
|
-
}
|
135
|
-
try {
|
136
|
-
this.loader?.remoteMetaStore?.handleByteHeads([first.eventBlock.value.data.dbMeta]);
|
137
|
-
this.eventsWeHandled.add(first.cid);
|
138
|
-
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
139
|
-
} catch (err) {
|
140
|
-
if (first.retries++ > 3) {
|
141
|
-
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
142
|
-
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
143
|
-
}
|
144
|
-
await new Promise((resolve) => setTimeout(resolve, 50));
|
145
|
-
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
146
|
-
} finally {
|
147
|
-
this.isProcessing = false;
|
148
|
-
if (this.queue.length > 0) {
|
149
|
-
void this.processQueue();
|
150
|
-
}
|
151
|
-
}
|
152
|
-
}
|
153
|
-
};
|
180
|
+
// src/blockstore/store-factory.ts
|
181
|
+
import { KeyedResolvOnce as KeyedResolvOnce2, URI as URI5 } from "@adviser/cement";
|
154
182
|
|
155
|
-
// src/
|
156
|
-
var
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
183
|
+
// src/runtime/files.ts
|
184
|
+
var files_exports = {};
|
185
|
+
__export(files_exports, {
|
186
|
+
decodeFile: () => decodeFile,
|
187
|
+
encodeFile: () => encodeFile
|
188
|
+
});
|
189
|
+
import * as UnixFS from "@ipld/unixfs";
|
190
|
+
import * as raw from "multiformats/codecs/raw";
|
191
|
+
import { withMaxChunkSize } from "@ipld/unixfs/file/chunker/fixed";
|
192
|
+
import { withWidth } from "@ipld/unixfs/file/layout/balanced";
|
193
|
+
import { exporter } from "ipfs-unixfs-exporter";
|
194
|
+
var queuingStrategy = UnixFS.withCapacity();
|
195
|
+
var settings = UnixFS.configure({
|
196
|
+
fileChunkEncoder: raw,
|
197
|
+
smallFileEncoder: raw,
|
198
|
+
chunker: withMaxChunkSize(1024 * 1024),
|
199
|
+
fileLayout: withWidth(1024)
|
200
|
+
});
|
201
|
+
async function collect(collectable) {
|
202
|
+
const chunks = [];
|
203
|
+
await collectable.pipeTo(
|
204
|
+
new WritableStream({
|
205
|
+
write(chunk) {
|
206
|
+
chunks.push(chunk);
|
207
|
+
}
|
208
|
+
})
|
209
|
+
);
|
210
|
+
return chunks;
|
211
|
+
}
|
212
|
+
async function encodeFile(blob) {
|
213
|
+
const readable = createFileEncoderStream(blob);
|
214
|
+
const blocks = await collect(readable);
|
215
|
+
return { cid: blocks.at(-1).cid, blocks };
|
216
|
+
}
|
217
|
+
async function decodeFile(blocks, cid, meta) {
|
218
|
+
const entry = await exporter(cid.toString(), blocks, { length: meta.size });
|
219
|
+
const chunks = [];
|
220
|
+
for await (const chunk of entry.content()) {
|
221
|
+
chunks.push(chunk);
|
222
|
+
}
|
223
|
+
return new File(chunks, entry.name, { type: meta.type, lastModified: 0 });
|
224
|
+
}
|
225
|
+
function createFileEncoderStream(blob) {
|
226
|
+
const { readable, writable } = new TransformStream({}, queuingStrategy);
|
227
|
+
const unixfsWriter = UnixFS.createWriter({ writable, settings });
|
228
|
+
const fileBuilder = new UnixFSFileBuilder("", blob);
|
229
|
+
void (async () => {
|
230
|
+
await fileBuilder.finalize(unixfsWriter);
|
231
|
+
await unixfsWriter.close();
|
232
|
+
})();
|
233
|
+
return readable;
|
234
|
+
}
|
235
|
+
var UnixFSFileBuilder = class {
|
236
|
+
#file;
|
237
|
+
constructor(name, file) {
|
238
|
+
this.name = name;
|
239
|
+
this.#file = file;
|
240
|
+
}
|
241
|
+
async finalize(writer) {
|
242
|
+
const unixfsFileWriter = UnixFS.createFileWriter(writer);
|
243
|
+
await this.#file.stream().pipeTo(
|
244
|
+
new WritableStream({
|
245
|
+
async write(chunk) {
|
246
|
+
await unixfsFileWriter.write(chunk);
|
247
|
+
}
|
248
|
+
})
|
196
249
|
);
|
197
|
-
await
|
198
|
-
return event;
|
199
|
-
}
|
200
|
-
async decodeEventBlock(bytes) {
|
201
|
-
const event = await decodeEventBlock(bytes);
|
202
|
-
return event;
|
203
|
-
}
|
204
|
-
// move this stuff to connect
|
205
|
-
// async getDashboardURL(compact = true) {
|
206
|
-
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
207
|
-
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
208
|
-
// // if (compact) {
|
209
|
-
// // await this.compact()
|
210
|
-
// // }
|
211
|
-
// const currents = await this.loader?.metaStore?.load()
|
212
|
-
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
213
|
-
// if (currents.length > 1)
|
214
|
-
// throw new Error("Can't sync database with split heads: make an update first")
|
215
|
-
// const current = currents[0]
|
216
|
-
// const params = {
|
217
|
-
// car: current.car.toString()
|
218
|
-
// }
|
219
|
-
// if (current.key) {
|
220
|
-
// // @ts-ignore
|
221
|
-
// params.key = current.key.toString()
|
222
|
-
// }
|
223
|
-
// // @ts-ignore
|
224
|
-
// if (this.name) {
|
225
|
-
// // @ts-ignore
|
226
|
-
// params.name = this.name
|
227
|
-
// }
|
228
|
-
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
229
|
-
// console.log('Import to dashboard: ' + url.toString())
|
230
|
-
// return url
|
231
|
-
// }
|
232
|
-
// openDashboard() {
|
233
|
-
// void this.getDashboardURL().then(url => {
|
234
|
-
// if (url) window.open(url.toString(), '_blank')
|
235
|
-
// })
|
236
|
-
// }
|
237
|
-
};
|
238
|
-
|
239
|
-
// src/blockstore/connect-rest.ts
|
240
|
-
var ConnectREST = class extends ConnectionBase {
|
241
|
-
constructor(base, logger) {
|
242
|
-
super(ensureLogger(logger, "ConnectREST"));
|
243
|
-
this.baseUrl = new URL(base);
|
244
|
-
}
|
245
|
-
async dataUpload(bytes, params) {
|
246
|
-
const carCid = params.car.toString();
|
247
|
-
const uploadURL = new URL(`/cars/${carCid}.car`, this.baseUrl);
|
248
|
-
const done = await fetch(uploadURL, { method: "PUT", body: bytes });
|
249
|
-
if (!done.ok) {
|
250
|
-
throw this.logger.Error().Msg("failed to upload data " + done.statusText);
|
251
|
-
}
|
252
|
-
}
|
253
|
-
async dataDownload(params) {
|
254
|
-
const { car } = params;
|
255
|
-
const fetchFromUrl = new URL(`/cars/${car.toString()}.car`, this.baseUrl);
|
256
|
-
const response = await fetch(fetchFromUrl);
|
257
|
-
if (!response.ok) {
|
258
|
-
return void 0;
|
259
|
-
}
|
260
|
-
const bytes = new Uint8Array(await response.arrayBuffer());
|
261
|
-
return bytes;
|
262
|
-
}
|
263
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
264
|
-
async metaUpload(bytes, params) {
|
265
|
-
return void 0;
|
266
|
-
}
|
267
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
268
|
-
async metaDownload(params) {
|
269
|
-
return [];
|
250
|
+
return await unixfsFileWriter.close();
|
270
251
|
}
|
271
252
|
};
|
272
253
|
|
273
|
-
// src/blockstore/store-factory.ts
|
274
|
-
import { KeyedResolvOnce } from "@adviser/cement";
|
275
|
-
|
276
254
|
// src/blockstore/store.ts
|
277
255
|
import pLimit2 from "p-limit";
|
278
256
|
import { format, parse } from "@ipld/dag-json";
|
279
|
-
import { ResolveOnce as
|
257
|
+
import { exception2Result, ResolveOnce as ResolveOnce3, Result as Result4 } from "@adviser/cement";
|
258
|
+
|
259
|
+
// src/types.ts
|
260
|
+
function isFalsy(value) {
|
261
|
+
return value === false && value === null && value === void 0;
|
262
|
+
}
|
263
|
+
function throwFalsy(value) {
|
264
|
+
if (isFalsy(value)) {
|
265
|
+
throw new Error("value is Falsy");
|
266
|
+
}
|
267
|
+
return value;
|
268
|
+
}
|
269
|
+
function falsyToUndef(value) {
|
270
|
+
if (isFalsy(value)) {
|
271
|
+
return void 0;
|
272
|
+
}
|
273
|
+
return value;
|
274
|
+
}
|
280
275
|
|
281
276
|
// src/blockstore/loader.ts
|
282
277
|
import pLimit from "p-limit";
|
283
278
|
import { CarReader } from "@ipld/car";
|
284
|
-
import { ResolveOnce } from "@adviser/cement";
|
285
|
-
|
286
|
-
// src/blockstore/types.ts
|
287
|
-
function toCIDBlock(block) {
|
288
|
-
return block;
|
289
|
-
}
|
279
|
+
import { ResolveOnce as ResolveOnce2 } from "@adviser/cement";
|
290
280
|
|
291
281
|
// src/blockstore/loader-helpers.ts
|
292
|
-
import { encode, decode } from "multiformats/block";
|
293
282
|
import { sha256 as hasher } from "multiformats/hashes/sha2";
|
294
|
-
import * as
|
295
|
-
import * as CBW from "@ipld/car/buffer-writer";
|
296
|
-
import * as codec from "@ipld/dag-cbor";
|
297
|
-
async function encodeCarFile(roots, t) {
|
298
|
-
let size = 0;
|
299
|
-
const headerSize = CBW.headerLength({ roots });
|
300
|
-
size += headerSize;
|
301
|
-
for (const { cid, bytes } of t.entries()) {
|
302
|
-
size += CBW.blockLength({ cid, bytes });
|
303
|
-
}
|
304
|
-
const buffer = new Uint8Array(size);
|
305
|
-
const writer = CBW.createWriter(buffer, { headerSize });
|
306
|
-
for (const r of roots) {
|
307
|
-
writer.addRoot(r);
|
308
|
-
}
|
309
|
-
for (const { cid, bytes } of t.entries()) {
|
310
|
-
writer.write({ cid, bytes });
|
311
|
-
}
|
312
|
-
writer.close();
|
313
|
-
return await encode({ value: writer.bytes, hasher, codec: raw });
|
314
|
-
}
|
315
|
-
async function encodeCarHeader(fp) {
|
316
|
-
return await encode({
|
317
|
-
value: { fp },
|
318
|
-
hasher,
|
319
|
-
codec
|
320
|
-
});
|
321
|
-
}
|
283
|
+
import * as dagCodec from "@ipld/dag-cbor";
|
322
284
|
async function parseCarFile(reader, logger) {
|
323
285
|
const roots = await reader.getRoots();
|
324
286
|
const header = await reader.get(roots[0]);
|
325
287
|
if (!header) throw logger.Error().Msg("missing header block").AsError();
|
326
|
-
const
|
327
|
-
const fpvalue = value;
|
288
|
+
const dec = await decode({ bytes: header.bytes, hasher, codec: dagCodec });
|
289
|
+
const fpvalue = dec.value;
|
328
290
|
if (fpvalue && !fpvalue.fp) {
|
329
291
|
throw logger.Error().Msg("missing fp").AsError();
|
330
292
|
}
|
331
293
|
return fpvalue.fp;
|
332
294
|
}
|
333
295
|
|
334
|
-
// src/blockstore/encrypt-helpers.ts
|
335
|
-
import { sha256 } from "multiformats/hashes/sha2";
|
336
|
-
import { CID as CID2 } from "multiformats";
|
337
|
-
import { encode as encode2, decode as decode2, create as mfCreate } from "multiformats/block";
|
338
|
-
import * as dagcbor from "@ipld/dag-cbor";
|
339
|
-
import { MemoryBlockstore as MemoryBlockstore2 } from "@web3-storage/pail/block";
|
340
|
-
import { bf } from "prolly-trees/utils";
|
341
|
-
import { nocache as cache } from "prolly-trees/cache";
|
342
|
-
import { create, load } from "prolly-trees/cid-set";
|
343
|
-
|
344
|
-
// src/blockstore/encrypt-codec.ts
|
345
|
-
import { CID } from "multiformats";
|
346
|
-
function makeCodec(ilogger, crypto, randomBytes) {
|
347
|
-
const logger = ensureLogger(ilogger, "makeCodec");
|
348
|
-
const enc32 = (value) => {
|
349
|
-
value = +value;
|
350
|
-
const buff = new Uint8Array(4);
|
351
|
-
buff[3] = value >>> 24;
|
352
|
-
buff[2] = value >>> 16;
|
353
|
-
buff[1] = value >>> 8;
|
354
|
-
buff[0] = value & 255;
|
355
|
-
return buff;
|
356
|
-
};
|
357
|
-
const readUInt32LE = (buffer) => {
|
358
|
-
const offset = buffer.byteLength - 4;
|
359
|
-
return (buffer[offset] | buffer[offset + 1] << 8 | buffer[offset + 2] << 16) + buffer[offset + 3] * 16777216;
|
360
|
-
};
|
361
|
-
const concat = (buffers) => {
|
362
|
-
const uint8Arrays = buffers.map((b) => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
|
363
|
-
const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
|
364
|
-
const result = new Uint8Array(totalLength);
|
365
|
-
let offset = 0;
|
366
|
-
for (const arr of uint8Arrays) {
|
367
|
-
result.set(arr, offset);
|
368
|
-
offset += arr.length;
|
369
|
-
}
|
370
|
-
return result;
|
371
|
-
};
|
372
|
-
const encode4 = ({ iv, bytes }) => concat([iv, bytes]);
|
373
|
-
const decode4 = (bytes) => {
|
374
|
-
const iv = bytes.subarray(0, 12);
|
375
|
-
bytes = bytes.slice(12);
|
376
|
-
return { iv, bytes };
|
377
|
-
};
|
378
|
-
const code = 3145728 + 1337;
|
379
|
-
async function subtleKey(key) {
|
380
|
-
return await crypto.importKey(
|
381
|
-
"raw",
|
382
|
-
// raw or jwk
|
383
|
-
key,
|
384
|
-
// raw data
|
385
|
-
"AES-GCM",
|
386
|
-
false,
|
387
|
-
// extractable
|
388
|
-
["encrypt", "decrypt"]
|
389
|
-
);
|
390
|
-
}
|
391
|
-
const decrypt = async ({ key, value }) => {
|
392
|
-
const { bytes: inBytes, iv } = value;
|
393
|
-
const cryKey = await subtleKey(key);
|
394
|
-
const deBytes = await crypto.decrypt(
|
395
|
-
{
|
396
|
-
name: "AES-GCM",
|
397
|
-
iv,
|
398
|
-
tagLength: 128
|
399
|
-
},
|
400
|
-
cryKey,
|
401
|
-
inBytes
|
402
|
-
);
|
403
|
-
const bytes = new Uint8Array(deBytes);
|
404
|
-
const len = readUInt32LE(bytes.subarray(0, 4));
|
405
|
-
const cid = CID.decode(bytes.subarray(4, 4 + len));
|
406
|
-
return { cid, bytes: bytes.subarray(4 + len) };
|
407
|
-
};
|
408
|
-
const encrypt = async ({ key, cid, bytes }) => {
|
409
|
-
const len = enc32(cid.bytes.byteLength);
|
410
|
-
const iv = randomBytes(12);
|
411
|
-
const msg = concat([len, cid.bytes, bytes]);
|
412
|
-
try {
|
413
|
-
const cryKey = await subtleKey(key);
|
414
|
-
const deBytes = await crypto.encrypt(
|
415
|
-
{
|
416
|
-
name: "AES-GCM",
|
417
|
-
iv,
|
418
|
-
tagLength: 128
|
419
|
-
},
|
420
|
-
cryKey,
|
421
|
-
msg
|
422
|
-
);
|
423
|
-
bytes = new Uint8Array(deBytes);
|
424
|
-
} catch (e) {
|
425
|
-
throw logger.Error().Err(e).Msg("encrypt failed").AsError();
|
426
|
-
}
|
427
|
-
return { value: { bytes, iv } };
|
428
|
-
};
|
429
|
-
const cryptoFn = (key) => {
|
430
|
-
return { encrypt: (opts) => encrypt({ ...opts, key }), decrypt: (opts) => decrypt({ ...opts, key }) };
|
431
|
-
};
|
432
|
-
const name = "jchris@encrypted-block:aes-gcm";
|
433
|
-
return { encode: encode4, decode: decode4, code, name, encrypt, decrypt, crypto: cryptoFn };
|
434
|
-
}
|
435
|
-
|
436
|
-
// src/blockstore/encrypt-helpers.ts
|
437
|
-
function carLogIncludesGroup(list, cidMatch) {
|
438
|
-
return list.some((cid) => {
|
439
|
-
return cid.toString() === cidMatch.toString();
|
440
|
-
});
|
441
|
-
}
|
442
|
-
function makeEncDec(logger, crypto, randomBytes) {
|
443
|
-
const codec4 = makeCodec(logger, crypto, randomBytes);
|
444
|
-
const encrypt = async function* ({
|
445
|
-
get: get2,
|
446
|
-
cids,
|
447
|
-
hasher: hasher4,
|
448
|
-
key,
|
449
|
-
cache: cache3,
|
450
|
-
chunker: chunker2,
|
451
|
-
root: root3
|
452
|
-
}) {
|
453
|
-
const set = /* @__PURE__ */ new Set();
|
454
|
-
let eroot;
|
455
|
-
if (!carLogIncludesGroup(cids, root3)) cids.push(root3);
|
456
|
-
for (const cid of cids) {
|
457
|
-
const unencrypted = await get2(cid);
|
458
|
-
if (!unencrypted) throw logger.Error().Ref("cid", cid).Msg("missing cid block").AsError();
|
459
|
-
const encrypted = await codec4.encrypt({ ...unencrypted, key });
|
460
|
-
const block2 = await encode2({ ...encrypted, codec: codec4, hasher: hasher4 });
|
461
|
-
yield block2;
|
462
|
-
set.add(block2.cid.toString());
|
463
|
-
if (unencrypted.cid.equals(root3)) eroot = block2.cid;
|
464
|
-
}
|
465
|
-
if (!eroot) throw logger.Error().Msg("cids does not include root").AsError();
|
466
|
-
const list = [...set].map((s) => CID2.parse(s));
|
467
|
-
let last;
|
468
|
-
for await (const node of create({ list, get: get2, cache: cache3, chunker: chunker2, hasher: hasher4, codec: dagcbor })) {
|
469
|
-
const block2 = await node.block;
|
470
|
-
yield block2;
|
471
|
-
last = block2;
|
472
|
-
}
|
473
|
-
if (!last) throw logger.Error().Msg("missing last block").AsError();
|
474
|
-
const head = [eroot, last.cid];
|
475
|
-
const block = await encode2({ value: head, codec: dagcbor, hasher: hasher4 });
|
476
|
-
yield block;
|
477
|
-
};
|
478
|
-
const decrypt = async function* ({
|
479
|
-
root: root3,
|
480
|
-
get: get2,
|
481
|
-
key,
|
482
|
-
cache: cache3,
|
483
|
-
chunker: chunker2,
|
484
|
-
hasher: hasher4
|
485
|
-
}) {
|
486
|
-
const getWithDecode = async (cid) => get2(cid).then(async (block) => {
|
487
|
-
if (!block) return;
|
488
|
-
const decoded = await decode2({ ...block, codec: dagcbor, hasher: hasher4 });
|
489
|
-
return decoded;
|
490
|
-
});
|
491
|
-
const getWithDecrypt = async (cid) => get2(cid).then(async (block) => {
|
492
|
-
if (!block) return;
|
493
|
-
const decoded = await decode2({ ...block, codec: codec4, hasher: hasher4 });
|
494
|
-
return decoded;
|
495
|
-
});
|
496
|
-
const decodedRoot = await getWithDecode(root3);
|
497
|
-
if (!decodedRoot) throw logger.Error().Msg("missing root").AsError();
|
498
|
-
if (!decodedRoot.bytes) throw logger.Error().Msg("missing bytes").AsError();
|
499
|
-
const {
|
500
|
-
value: [eroot, tree]
|
501
|
-
} = decodedRoot;
|
502
|
-
const rootBlock = await get2(eroot);
|
503
|
-
if (!rootBlock) throw logger.Error().Msg("missing root block").AsError();
|
504
|
-
const cidset = await load({ cid: tree, get: getWithDecode, cache: cache3, chunker: chunker2, codec: codec4, hasher: hasher4 });
|
505
|
-
const { result: nodes } = await cidset.getAllEntries();
|
506
|
-
const unwrap = async (eblock) => {
|
507
|
-
if (!eblock) throw logger.Error().Msg("missing block").AsError();
|
508
|
-
if (!eblock.value) {
|
509
|
-
eblock = await decode2({ ...eblock, codec: codec4, hasher: hasher4 });
|
510
|
-
if (!eblock.value) throw logger.Error().Msg("missing value").AsError();
|
511
|
-
}
|
512
|
-
const { bytes, cid } = await codec4.decrypt({ ...eblock, key }).catch((e) => {
|
513
|
-
throw e;
|
514
|
-
});
|
515
|
-
const block = await mfCreate({ cid, bytes, hasher: hasher4, codec: codec4 });
|
516
|
-
return block;
|
517
|
-
};
|
518
|
-
const promises = [];
|
519
|
-
for (const { cid } of nodes) {
|
520
|
-
if (!rootBlock.cid.equals(cid)) promises.push(getWithDecrypt(cid).then(unwrap));
|
521
|
-
}
|
522
|
-
yield* promises;
|
523
|
-
yield unwrap(rootBlock);
|
524
|
-
};
|
525
|
-
return { encrypt, decrypt };
|
526
|
-
}
|
527
|
-
var chunker = bf(30);
|
528
|
-
function hexStringToUint8Array(hexString) {
|
529
|
-
const length = hexString.length;
|
530
|
-
const uint8Array = new Uint8Array(length / 2);
|
531
|
-
for (let i = 0; i < length; i += 2) {
|
532
|
-
uint8Array[i / 2] = parseInt(hexString.substring(i, i + 2), 16);
|
533
|
-
}
|
534
|
-
return uint8Array;
|
535
|
-
}
|
536
|
-
async function encryptedEncodeCarFile(logger, crypto, key, rootCid, t) {
|
537
|
-
const encryptionKey = hexStringToUint8Array(key);
|
538
|
-
const encryptedBlocks = new MemoryBlockstore2();
|
539
|
-
const cidsToEncrypt = [];
|
540
|
-
for (const { cid, bytes } of t.entries()) {
|
541
|
-
cidsToEncrypt.push(cid);
|
542
|
-
const g = await t.get(cid);
|
543
|
-
if (!g) throw logger.Error().Ref("cid", cid).Int("bytes", bytes.length).Msg("missing cid block").AsError();
|
544
|
-
}
|
545
|
-
let last = null;
|
546
|
-
const { encrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
|
547
|
-
for await (const block of encrypt({
|
548
|
-
cids: cidsToEncrypt,
|
549
|
-
get: t.get.bind(t),
|
550
|
-
key: encryptionKey,
|
551
|
-
hasher: sha256,
|
552
|
-
chunker,
|
553
|
-
cache,
|
554
|
-
root: rootCid
|
555
|
-
})) {
|
556
|
-
await encryptedBlocks.put(block.cid, block.bytes);
|
557
|
-
last = block;
|
558
|
-
}
|
559
|
-
if (!last) throw logger.Error().Msg("no blocks encrypted").AsError();
|
560
|
-
const encryptedCar = await encodeCarFile([last.cid], encryptedBlocks);
|
561
|
-
return encryptedCar;
|
562
|
-
}
|
563
|
-
async function decodeEncryptedCar(logger, crypto, key, reader) {
|
564
|
-
const roots = await reader.getRoots();
|
565
|
-
const root3 = roots[0];
|
566
|
-
return await decodeCarBlocks(logger, crypto, root3, reader.get.bind(reader), key);
|
567
|
-
}
|
568
|
-
async function decodeCarBlocks(logger, crypto, root3, get2, keyMaterial) {
|
569
|
-
const decryptionKeyUint8 = hexStringToUint8Array(keyMaterial);
|
570
|
-
const decryptionKey = decryptionKeyUint8.buffer.slice(0, decryptionKeyUint8.byteLength);
|
571
|
-
const decryptedBlocks = new MemoryBlockstore2();
|
572
|
-
let last = null;
|
573
|
-
const { decrypt } = makeEncDec(logger, crypto, crypto.randomBytes);
|
574
|
-
for await (const block of decrypt({
|
575
|
-
root: root3,
|
576
|
-
get: get2,
|
577
|
-
key: decryptionKey,
|
578
|
-
hasher: sha256,
|
579
|
-
chunker,
|
580
|
-
cache
|
581
|
-
})) {
|
582
|
-
await decryptedBlocks.put(block.cid, block.bytes);
|
583
|
-
last = block;
|
584
|
-
}
|
585
|
-
if (!last) throw logger.Error().Msg("no blocks decrypted").AsError();
|
586
|
-
return { blocks: decryptedBlocks, root: last.cid };
|
587
|
-
}
|
588
|
-
|
589
296
|
// src/blockstore/transaction.ts
|
590
|
-
import { MemoryBlockstore
|
591
|
-
|
592
|
-
|
297
|
+
import { MemoryBlockstore } from "@web3-storage/pail/block";
|
298
|
+
import { toCryptoRuntime } from "@adviser/cement";
|
299
|
+
var CarTransaction = class extends MemoryBlockstore {
|
300
|
+
constructor(parent, opts = { add: true, noLoader: false }) {
|
593
301
|
super();
|
594
302
|
if (opts.add) {
|
595
303
|
parent.transactions.add(this);
|
@@ -603,8 +311,8 @@ var CarTransaction = class extends MemoryBlockstore3 {
|
|
603
311
|
return super.get(cid);
|
604
312
|
}
|
605
313
|
};
|
606
|
-
function defaultedBlockstoreRuntime(opts, component, ctx) {
|
607
|
-
const logger = ensureLogger(
|
314
|
+
function defaultedBlockstoreRuntime(sthis, opts, component, ctx) {
|
315
|
+
const logger = ensureLogger(sthis, component, ctx);
|
608
316
|
const store = opts.store || {};
|
609
317
|
return {
|
610
318
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
@@ -621,22 +329,24 @@ function defaultedBlockstoreRuntime(opts, component, ctx) {
|
|
621
329
|
threshold: 1e3 * 1e3,
|
622
330
|
...opts,
|
623
331
|
logger,
|
624
|
-
|
332
|
+
keyBag: opts.keyBag || {},
|
333
|
+
crypto: toCryptoRuntime(opts.crypto),
|
625
334
|
store,
|
626
|
-
storeRuntime: toStoreRuntime(store,
|
335
|
+
storeRuntime: toStoreRuntime(store, sthis)
|
627
336
|
};
|
628
337
|
}
|
629
|
-
|
338
|
+
function blockstoreFactory(sthis, opts) {
|
630
339
|
if (opts.name) {
|
631
|
-
return new EncryptedBlockstore(opts);
|
340
|
+
return new EncryptedBlockstore(sthis, opts);
|
632
341
|
} else {
|
633
342
|
return new BaseBlockstore(opts);
|
634
343
|
}
|
635
|
-
}
|
344
|
+
}
|
636
345
|
var BaseBlockstore = class {
|
637
346
|
constructor(ebOpts = {}) {
|
638
347
|
this.transactions = /* @__PURE__ */ new Set();
|
639
|
-
this.
|
348
|
+
this.sthis = ensureSuperThis(ebOpts);
|
349
|
+
this.ebOpts = defaultedBlockstoreRuntime(this.sthis, ebOpts, "BaseBlockstore");
|
640
350
|
this.logger = this.ebOpts.logger;
|
641
351
|
}
|
642
352
|
// ready: Promise<void>;
|
@@ -659,8 +369,8 @@ var BaseBlockstore = class {
|
|
659
369
|
throw this.logger.Error().Msg("use a transaction to put").AsError();
|
660
370
|
}
|
661
371
|
// TransactionMeta
|
662
|
-
async transaction(fn, _opts
|
663
|
-
const t = new CarTransaction(this);
|
372
|
+
async transaction(fn, _opts) {
|
373
|
+
const t = new CarTransaction(this, _opts);
|
664
374
|
const done = await fn(t);
|
665
375
|
this.lastTxMeta = done;
|
666
376
|
return { t, meta: done };
|
@@ -677,16 +387,16 @@ var BaseBlockstore = class {
|
|
677
387
|
}
|
678
388
|
};
|
679
389
|
var EncryptedBlockstore = class extends BaseBlockstore {
|
680
|
-
constructor(ebOpts) {
|
390
|
+
constructor(sthis, ebOpts) {
|
681
391
|
super(ebOpts);
|
682
392
|
this.compacting = false;
|
683
|
-
this.logger = ensureLogger(
|
393
|
+
this.logger = ensureLogger(this.sthis, "EncryptedBlockstore");
|
684
394
|
const { name } = ebOpts;
|
685
395
|
if (!name) {
|
686
396
|
throw this.logger.Error().Msg("name required").AsError();
|
687
397
|
}
|
688
398
|
this.name = name;
|
689
|
-
this.loader = new Loader(this.name, ebOpts);
|
399
|
+
this.loader = new Loader(this.name, ebOpts, sthis);
|
690
400
|
}
|
691
401
|
ready() {
|
692
402
|
return this.loader.ready();
|
@@ -717,10 +427,13 @@ var EncryptedBlockstore = class extends BaseBlockstore {
|
|
717
427
|
}
|
718
428
|
throw this.logger.Error().Msg("failed to commit car files").AsError();
|
719
429
|
}
|
720
|
-
async getFile(car, cid
|
430
|
+
async getFile(car, cid) {
|
721
431
|
await this.ready();
|
722
432
|
if (!this.loader) throw this.logger.Error().Msg("loader required to get file, database must be named").AsError();
|
723
|
-
const reader = await this.loader.loadFileCar(
|
433
|
+
const reader = await this.loader.loadFileCar(
|
434
|
+
car
|
435
|
+
/*, isPublic */
|
436
|
+
);
|
724
437
|
const block = await reader.get(cid);
|
725
438
|
if (!block) throw this.logger.Error().Str("cid", cid.toString()).Msg(`Missing block`).AsError();
|
726
439
|
return block.bytes;
|
@@ -776,10 +489,20 @@ var CompactionFetcher = class {
|
|
776
489
|
};
|
777
490
|
|
778
491
|
// src/blockstore/commit-queue.ts
|
492
|
+
import { Future } from "@adviser/cement";
|
779
493
|
var CommitQueue = class {
|
780
494
|
constructor() {
|
781
495
|
this.queue = [];
|
782
496
|
this.processing = false;
|
497
|
+
this._waitIdleItems = /* @__PURE__ */ new Set();
|
498
|
+
}
|
499
|
+
waitIdle() {
|
500
|
+
if (this.queue.length === 0 && !this.processing) {
|
501
|
+
return Promise.resolve();
|
502
|
+
}
|
503
|
+
const fn = new Future();
|
504
|
+
this._waitIdleItems.add(fn);
|
505
|
+
return fn.asPromise();
|
783
506
|
}
|
784
507
|
async enqueue(fn) {
|
785
508
|
return new Promise((resolve, reject) => {
|
@@ -804,38 +527,384 @@ var CommitQueue = class {
|
|
804
527
|
this.processing = true;
|
805
528
|
const queueFn = this.queue.shift();
|
806
529
|
if (queueFn) {
|
807
|
-
queueFn()
|
530
|
+
queueFn().finally(() => {
|
531
|
+
});
|
808
532
|
}
|
809
533
|
}
|
534
|
+
if (this.queue.length === 0 && !this.processing) {
|
535
|
+
const toResolve = Array.from(this._waitIdleItems);
|
536
|
+
this._waitIdleItems.clear();
|
537
|
+
toResolve.map((fn) => fn.resolve());
|
538
|
+
}
|
810
539
|
}
|
811
540
|
};
|
812
541
|
|
813
|
-
// src/
|
814
|
-
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
}
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
542
|
+
// src/runtime/key-bag.ts
|
543
|
+
var key_bag_exports = {};
|
544
|
+
__export(key_bag_exports, {
|
545
|
+
KeyBag: () => KeyBag,
|
546
|
+
getKeyBag: () => getKeyBag,
|
547
|
+
registerKeyBagProviderFactory: () => registerKeyBagProviderFactory
|
548
|
+
});
|
549
|
+
import {
|
550
|
+
KeyedResolvOnce,
|
551
|
+
ResolveOnce,
|
552
|
+
ResolveSeq,
|
553
|
+
Result as Result2,
|
554
|
+
runtimeFn,
|
555
|
+
toCryptoRuntime as toCryptoRuntime2,
|
556
|
+
URI
|
557
|
+
} from "@adviser/cement";
|
558
|
+
import { base58btc } from "multiformats/bases/base58";
|
559
|
+
var KeyBag = class {
|
560
|
+
constructor(rt) {
|
561
|
+
this.rt = rt;
|
562
|
+
this._warnOnce = new ResolveOnce();
|
563
|
+
this._seq = new ResolveSeq();
|
564
|
+
this.logger = ensureLogger(rt.sthis, "KeyBag");
|
565
|
+
this.logger.Debug().Msg("KeyBag created");
|
566
|
+
}
|
567
|
+
async subtleKey(key) {
|
568
|
+
const extractable = this.rt.url.getParam("extractKey") === "_deprecated_internal_api";
|
569
|
+
if (extractable) {
|
570
|
+
this._warnOnce.once(
|
571
|
+
() => this.logger.Warn().Msg("extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!")
|
572
|
+
);
|
573
|
+
}
|
574
|
+
return await this.rt.crypto.importKey(
|
575
|
+
"raw",
|
576
|
+
// raw or jwk
|
577
|
+
base58btc.decode(key),
|
578
|
+
// hexStringToUint8Array(key), // raw data
|
579
|
+
"AES-GCM",
|
580
|
+
extractable,
|
581
|
+
["encrypt", "decrypt"]
|
582
|
+
);
|
825
583
|
}
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
}
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
584
|
+
async ensureKeyFromUrl(url, keyFactory) {
|
585
|
+
const storeKey = url.getParam("storekey");
|
586
|
+
if (storeKey === "insecure") {
|
587
|
+
return Result2.Ok(url);
|
588
|
+
}
|
589
|
+
if (!storeKey) {
|
590
|
+
const keyName = `@${keyFactory()}@`;
|
591
|
+
const ret = await this.getNamedKey(keyName);
|
592
|
+
if (ret.isErr()) {
|
593
|
+
return ret;
|
594
|
+
}
|
595
|
+
const urb = url.build().setParam("storekey", keyName);
|
596
|
+
return Result2.Ok(urb.URI());
|
597
|
+
}
|
598
|
+
if (storeKey.startsWith("@") && storeKey.endsWith("@")) {
|
599
|
+
const ret = await this.getNamedKey(storeKey);
|
600
|
+
if (ret.isErr()) {
|
601
|
+
return ret;
|
602
|
+
}
|
603
|
+
}
|
604
|
+
return Result2.Ok(url);
|
605
|
+
}
|
606
|
+
async toKeyWithFingerPrint(keyStr) {
|
607
|
+
const material = base58btc.decode(keyStr);
|
608
|
+
const key = await this.subtleKey(keyStr);
|
609
|
+
const fpr = await this.rt.crypto.digestSHA256(material);
|
610
|
+
return Result2.Ok({
|
611
|
+
key,
|
612
|
+
fingerPrint: base58btc.encode(new Uint8Array(fpr))
|
613
|
+
});
|
614
|
+
}
|
615
|
+
async setNamedKey(name, key) {
|
616
|
+
return this._seq.add(() => this._setNamedKey(name, key));
|
617
|
+
}
|
618
|
+
// avoid deadlock
|
619
|
+
async _setNamedKey(name, key) {
|
620
|
+
const item = {
|
621
|
+
name,
|
622
|
+
key
|
623
|
+
};
|
624
|
+
const bag = await this.rt.getBag();
|
625
|
+
this.logger.Debug().Str("name", name).Msg("setNamedKey");
|
626
|
+
await bag.set(name, item);
|
627
|
+
return await this.toKeyWithFingerPrint(item.key);
|
628
|
+
}
|
629
|
+
async getNamedExtractableKey(name, failIfNotFound = false) {
|
630
|
+
const ret = await this.getNamedKey(name, failIfNotFound);
|
631
|
+
if (ret.isErr()) {
|
632
|
+
return ret;
|
633
|
+
}
|
634
|
+
const named = ret.Ok();
|
635
|
+
return Result2.Ok({
|
636
|
+
...named,
|
637
|
+
extract: async () => {
|
638
|
+
const ext = new Uint8Array(await this.rt.crypto.exportKey("raw", named.key));
|
639
|
+
return {
|
640
|
+
key: ext,
|
641
|
+
keyStr: base58btc.encode(ext)
|
642
|
+
};
|
643
|
+
}
|
644
|
+
});
|
645
|
+
}
|
646
|
+
async getNamedKey(name, failIfNotFound = false) {
|
647
|
+
const id = this.rt.sthis.nextId(4).str;
|
648
|
+
return this._seq.add(async () => {
|
649
|
+
const bag = await this.rt.getBag();
|
650
|
+
const named = await bag.get(name);
|
651
|
+
if (named) {
|
652
|
+
const fpr = await this.toKeyWithFingerPrint(named.key);
|
653
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", fpr).Msg("fingerPrint getNamedKey");
|
654
|
+
return fpr;
|
655
|
+
}
|
656
|
+
if (failIfNotFound) {
|
657
|
+
this.logger.Debug().Str("id", id).Str("name", name).Msg("failIfNotFound getNamedKey");
|
658
|
+
return Result2.Err(new Error(`Key not found: ${name}`));
|
659
|
+
}
|
660
|
+
const ret = await this._setNamedKey(name, base58btc.encode(this.rt.crypto.randomBytes(this.rt.keyLength)));
|
661
|
+
this.logger.Debug().Str("id", id).Str("name", name).Result("fpr", ret).Msg("createKey getNamedKey-post");
|
662
|
+
return ret;
|
663
|
+
});
|
664
|
+
}
|
665
|
+
};
|
666
|
+
var keyBagProviderFactories = new Map(
|
667
|
+
[
|
668
|
+
{
|
669
|
+
protocol: "file:",
|
670
|
+
factory: async (url, sthis) => {
|
671
|
+
const { KeyBagProviderFile } = await import("./key-bag-file-DFMW6ZM6.js");
|
672
|
+
return new KeyBagProviderFile(url, sthis);
|
673
|
+
}
|
674
|
+
},
|
675
|
+
{
|
676
|
+
protocol: "indexdb:",
|
677
|
+
factory: async (url, sthis) => {
|
678
|
+
const { KeyBagProviderIndexDB } = await import("./key-bag-indexdb-R2RWGSQ4.js");
|
679
|
+
return new KeyBagProviderIndexDB(url, sthis);
|
680
|
+
}
|
681
|
+
}
|
682
|
+
].map((i) => [i.protocol, i])
|
683
|
+
);
|
684
|
+
function registerKeyBagProviderFactory(item) {
|
685
|
+
const protocol = item.protocol.endsWith(":") ? item.protocol : item.protocol + ":";
|
686
|
+
keyBagProviderFactories.set(protocol, {
|
687
|
+
...item,
|
688
|
+
protocol
|
689
|
+
});
|
690
|
+
}
|
691
|
+
function defaultKeyBagOpts(sthis, kbo) {
|
692
|
+
if (kbo.keyRuntime) {
|
693
|
+
return kbo.keyRuntime;
|
694
|
+
}
|
695
|
+
const logger = ensureLogger(sthis, "KeyBag");
|
696
|
+
let url;
|
697
|
+
if (kbo.url) {
|
698
|
+
url = URI.from(kbo.url);
|
699
|
+
logger.Debug().Url(url).Msg("from opts");
|
700
|
+
} else {
|
701
|
+
let bagFnameOrUrl = sthis.env.get("FP_KEYBAG_URL");
|
702
|
+
if (runtimeFn().isBrowser) {
|
703
|
+
url = URI.from(bagFnameOrUrl || "indexdb://fp-keybag");
|
704
|
+
} else {
|
705
|
+
if (!bagFnameOrUrl) {
|
706
|
+
const home = sthis.env.get("HOME");
|
707
|
+
bagFnameOrUrl = `${home}/.fireproof/keybag`;
|
708
|
+
url = URI.from(`file://${bagFnameOrUrl}`);
|
709
|
+
} else {
|
710
|
+
url = URI.from(bagFnameOrUrl);
|
711
|
+
}
|
712
|
+
}
|
713
|
+
logger.Debug().Url(url).Msg("from env");
|
714
|
+
}
|
715
|
+
const kitem = keyBagProviderFactories.get(url.protocol);
|
716
|
+
if (!kitem) {
|
717
|
+
throw logger.Error().Url(url).Msg("unsupported protocol").AsError();
|
718
|
+
}
|
719
|
+
const getBag = async () => kitem.factory(url, sthis);
|
720
|
+
if (url.hasParam("masterkey")) {
|
721
|
+
throw logger.Error().Url(url).Msg("masterkey is not supported").AsError();
|
722
|
+
}
|
723
|
+
return {
|
724
|
+
url,
|
725
|
+
crypto: kbo.crypto || toCryptoRuntime2({}),
|
726
|
+
sthis,
|
727
|
+
logger,
|
728
|
+
keyLength: kbo.keyLength || 16,
|
729
|
+
getBag,
|
730
|
+
id: () => {
|
731
|
+
return url.toString();
|
732
|
+
}
|
733
|
+
};
|
734
|
+
}
|
735
|
+
var _keyBags = new KeyedResolvOnce();
|
736
|
+
async function getKeyBag(sthis, kbo = {}) {
|
737
|
+
await sthis.start();
|
738
|
+
const rt = defaultKeyBagOpts(sthis, kbo);
|
739
|
+
return _keyBags.get(rt.id()).once(async () => new KeyBag(rt));
|
740
|
+
}
|
741
|
+
|
742
|
+
// src/blockstore/commitor.ts
|
743
|
+
import * as CBW from "@ipld/car/buffer-writer";
|
744
|
+
import { sha256 as hasher2 } from "multiformats/hashes/sha2";
|
745
|
+
import * as dagCodec2 from "@ipld/dag-cbor";
|
746
|
+
async function encodeCarFile(roots, t, codec3) {
|
747
|
+
let size = 0;
|
748
|
+
const headerSize = CBW.headerLength({ roots });
|
749
|
+
size += headerSize;
|
750
|
+
for (const { cid, bytes } of t.entries()) {
|
751
|
+
size += CBW.blockLength({ cid, bytes });
|
752
|
+
}
|
753
|
+
const buffer = new Uint8Array(size);
|
754
|
+
const writer = CBW.createWriter(buffer, { headerSize });
|
755
|
+
for (const r of roots) {
|
756
|
+
writer.addRoot(r);
|
757
|
+
}
|
758
|
+
for (const { cid, bytes } of t.entries()) {
|
759
|
+
writer.write({ cid, bytes });
|
760
|
+
}
|
761
|
+
writer.close();
|
762
|
+
return await encode({ value: writer.bytes, hasher: hasher2, codec: codec3 });
|
763
|
+
}
|
764
|
+
async function createCarFile(encoder, cid, t) {
|
765
|
+
return encodeCarFile([cid], t, encoder);
|
766
|
+
}
|
767
|
+
async function commitFiles(fileStore, walStore, t, done) {
|
768
|
+
const { files: roots } = makeFileCarHeader(done);
|
769
|
+
const cids = [];
|
770
|
+
const codec3 = (await fileStore.keyedCrypto()).codec();
|
771
|
+
const cars = await prepareCarFilesFiles(codec3, roots, t);
|
772
|
+
for (const car of cars) {
|
773
|
+
const { cid, bytes } = car;
|
774
|
+
await fileStore.save({ cid, bytes });
|
775
|
+
await walStore.enqueueFile(
|
776
|
+
cid
|
777
|
+
/*, !!opts.public*/
|
778
|
+
);
|
779
|
+
cids.push(cid);
|
780
|
+
}
|
781
|
+
return cids;
|
782
|
+
}
|
783
|
+
function makeFileCarHeader(result) {
|
784
|
+
const files = [];
|
785
|
+
for (const [, meta] of Object.entries(result.files || {})) {
|
786
|
+
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
787
|
+
files.push(meta.cid);
|
788
|
+
}
|
789
|
+
}
|
790
|
+
return { ...result, files };
|
791
|
+
}
|
792
|
+
async function prepareCarFilesFiles(encoder, roots, t) {
|
793
|
+
return [await encodeCarFile(roots, t, encoder)];
|
794
|
+
}
|
795
|
+
function makeCarHeader(meta, cars, compact = false) {
|
796
|
+
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
797
|
+
return { ...coreHeader, meta };
|
798
|
+
}
|
799
|
+
async function encodeCarHeader(fp) {
|
800
|
+
return await encode({
|
801
|
+
value: { fp },
|
802
|
+
hasher: hasher2,
|
803
|
+
codec: dagCodec2
|
804
|
+
});
|
805
|
+
}
|
806
|
+
async function commit(params, t, done, opts = { noLoader: false, compact: false }) {
|
807
|
+
const fp = makeCarHeader(done, params.carLog, !!opts.compact);
|
808
|
+
const rootBlock = await encodeCarHeader(fp);
|
809
|
+
const cars = await prepareCarFiles(params.encoder, params.threshold, rootBlock, t);
|
810
|
+
const cids = [];
|
811
|
+
for (const car of cars) {
|
812
|
+
const { cid, bytes } = car;
|
813
|
+
await params.carStore.save({ cid, bytes });
|
814
|
+
cids.push(cid);
|
815
|
+
}
|
816
|
+
const newDbMeta = { cars: cids };
|
817
|
+
await params.WALStore.enqueue(newDbMeta, opts);
|
818
|
+
await params.metaStore.save(newDbMeta);
|
819
|
+
return { cgrp: cids, header: fp };
|
820
|
+
}
|
821
|
+
async function prepareCarFiles(encoder, threshold, rootBlock, t) {
|
822
|
+
const carFiles = [];
|
823
|
+
threshold = threshold || 1e3 * 1e3;
|
824
|
+
let clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
825
|
+
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
826
|
+
let newsize = CBW.blockLength(toCIDBlock(rootBlock));
|
827
|
+
let cidRootBlock = rootBlock;
|
828
|
+
for (const { cid, bytes } of t.entries()) {
|
829
|
+
newsize += CBW.blockLength(toCIDBlock({ cid, bytes }));
|
830
|
+
if (newsize >= threshold) {
|
831
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
832
|
+
clonedt = new CarTransaction(t.parent, { add: false, noLoader: false });
|
833
|
+
clonedt.putSync(cid, bytes);
|
834
|
+
cidRootBlock = { cid, bytes };
|
835
|
+
newsize = CBW.blockLength(toCIDBlock({ cid, bytes }));
|
836
|
+
} else {
|
837
|
+
clonedt.putSync(cid, bytes);
|
838
|
+
}
|
839
|
+
}
|
840
|
+
carFiles.push(await createCarFile(encoder, cidRootBlock.cid, clonedt));
|
841
|
+
return carFiles;
|
842
|
+
}
|
843
|
+
|
844
|
+
// src/blockstore/loader.ts
|
845
|
+
import { sha256 as hasher3 } from "multiformats/hashes/sha2";
|
846
|
+
|
847
|
+
// src/blockstore/task-manager.ts
|
848
|
+
var TaskManager = class {
|
849
|
+
constructor(sthis, callback) {
|
850
|
+
this.eventsWeHandled = /* @__PURE__ */ new Set();
|
851
|
+
this.queue = [];
|
852
|
+
this.isProcessing = false;
|
853
|
+
this.logger = ensureLogger(sthis, "TaskManager");
|
854
|
+
this.callback = callback;
|
855
|
+
}
|
856
|
+
async handleEvent(cid, parents, dbMeta) {
|
857
|
+
for (const parent of parents) {
|
858
|
+
this.eventsWeHandled.add(parent.toString());
|
859
|
+
}
|
860
|
+
this.queue.push({ cid: cid.toString(), dbMeta, retries: 0 });
|
861
|
+
this.queue = this.queue.filter(({ cid: cid2 }) => !this.eventsWeHandled.has(cid2));
|
862
|
+
void this.processQueue();
|
863
|
+
}
|
864
|
+
async processQueue() {
|
865
|
+
if (this.isProcessing) return;
|
866
|
+
this.isProcessing = true;
|
867
|
+
const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
868
|
+
const first = filteredQueue[0];
|
869
|
+
if (!first) {
|
870
|
+
return;
|
871
|
+
}
|
872
|
+
try {
|
873
|
+
await this.callback(first.dbMeta);
|
874
|
+
this.eventsWeHandled.add(first.cid);
|
875
|
+
this.queue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid));
|
876
|
+
} catch (err) {
|
877
|
+
if (first.retries++ > 3) {
|
878
|
+
this.logger.Error().Str("cid", first.cid).Msg("failed to process event block after 3 retries");
|
879
|
+
this.queue = this.queue.filter(({ cid }) => cid !== first.cid);
|
880
|
+
}
|
881
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
882
|
+
throw this.logger.Error().Err(err).Msg("failed to process event block").AsError();
|
883
|
+
} finally {
|
884
|
+
this.isProcessing = false;
|
885
|
+
if (this.queue.length > 0) {
|
886
|
+
void this.processQueue();
|
887
|
+
}
|
888
|
+
}
|
835
889
|
}
|
836
890
|
};
|
891
|
+
|
892
|
+
// src/blockstore/loader.ts
|
893
|
+
function carLogIncludesGroup(list, cids) {
|
894
|
+
return list.some((arr) => {
|
895
|
+
return arr.toString() === cids.toString();
|
896
|
+
});
|
897
|
+
}
|
898
|
+
function uniqueCids(list, remove = /* @__PURE__ */ new Set()) {
|
899
|
+
const byString = /* @__PURE__ */ new Map();
|
900
|
+
for (const cid of list) {
|
901
|
+
if (remove.has(cid.toString())) continue;
|
902
|
+
byString.set(cid.toString(), cid);
|
903
|
+
}
|
904
|
+
return [...byString.values()];
|
905
|
+
}
|
837
906
|
var Loader = class {
|
838
|
-
constructor(name, ebOpts) {
|
907
|
+
constructor(name, ebOpts, sthis) {
|
839
908
|
this.commitQueue = new CommitQueue();
|
840
909
|
this.isCompacting = false;
|
841
910
|
this.carReaders = /* @__PURE__ */ new Map();
|
@@ -845,9 +914,11 @@ var Loader = class {
|
|
845
914
|
this.getBlockCache = /* @__PURE__ */ new Map();
|
846
915
|
this.seenMeta = /* @__PURE__ */ new Set();
|
847
916
|
this.writeLimit = pLimit(1);
|
848
|
-
this.onceReady = new
|
917
|
+
this.onceReady = new ResolveOnce2();
|
849
918
|
this.name = name;
|
919
|
+
this.sthis = sthis;
|
850
920
|
this.ebOpts = defaultedBlockstoreRuntime(
|
921
|
+
sthis,
|
851
922
|
{
|
852
923
|
...ebOpts,
|
853
924
|
name
|
@@ -855,34 +926,42 @@ var Loader = class {
|
|
855
926
|
"Loader"
|
856
927
|
);
|
857
928
|
this.logger = this.ebOpts.logger;
|
929
|
+
this.taskManager = new TaskManager(sthis, async (dbMeta) => {
|
930
|
+
await this.handleDbMetasFromStore([dbMeta]);
|
931
|
+
});
|
858
932
|
}
|
859
933
|
// readonly id = uuidv4();
|
934
|
+
async keyBag() {
|
935
|
+
return getKeyBag(this.sthis, this.ebOpts.keyBag);
|
936
|
+
}
|
860
937
|
async carStore() {
|
861
938
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
862
939
|
}
|
863
940
|
async fileStore() {
|
864
941
|
return this.ebOpts.storeRuntime.makeDataStore(this);
|
865
942
|
}
|
866
|
-
async
|
867
|
-
return this.ebOpts.storeRuntime.
|
943
|
+
async WALStore() {
|
944
|
+
return this.ebOpts.storeRuntime.makeWALStore(this);
|
868
945
|
}
|
869
946
|
async metaStore() {
|
870
947
|
return this.ebOpts.storeRuntime.makeMetaStore(this);
|
871
948
|
}
|
872
949
|
async ready() {
|
873
950
|
return this.onceReady.once(async () => {
|
874
|
-
const metas =
|
875
|
-
if (
|
951
|
+
const metas = await (await this.metaStore()).load();
|
952
|
+
if (this.ebOpts.meta) {
|
953
|
+
await this.handleDbMetasFromStore([this.ebOpts.meta]);
|
954
|
+
} else if (metas) {
|
876
955
|
await this.handleDbMetasFromStore(metas);
|
877
956
|
}
|
878
957
|
});
|
879
958
|
}
|
880
959
|
async close() {
|
881
|
-
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
960
|
+
const toClose = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
882
961
|
await Promise.all(toClose.map((store) => store.close()));
|
883
962
|
}
|
884
963
|
async destroy() {
|
885
|
-
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.
|
964
|
+
const toDestroy = await Promise.all([this.carStore(), this.metaStore(), this.fileStore(), this.WALStore()]);
|
886
965
|
await Promise.all(toDestroy.map((store) => store.destroy()));
|
887
966
|
}
|
888
967
|
// async snapToCar(carCid: AnyLink | string) {
|
@@ -896,6 +975,7 @@ var Loader = class {
|
|
896
975
|
// await this._applyCarHeader(carHeader, true)
|
897
976
|
// }
|
898
977
|
async handleDbMetasFromStore(metas) {
|
978
|
+
this.logger.Debug().Any("metas", metas).Msg("handleDbMetasFromStore");
|
899
979
|
for (const meta of metas) {
|
900
980
|
await this.writeLimit(async () => {
|
901
981
|
await this.mergeDbMetaIntoClock(meta);
|
@@ -908,10 +988,7 @@ var Loader = class {
|
|
908
988
|
}
|
909
989
|
if (this.seenMeta.has(meta.cars.toString())) return;
|
910
990
|
this.seenMeta.add(meta.cars.toString());
|
911
|
-
if (meta.
|
912
|
-
await this.setKey(meta.key);
|
913
|
-
}
|
914
|
-
if (carLogIncludesGroup2(this.carLog, meta.cars)) {
|
991
|
+
if (carLogIncludesGroup(this.carLog, meta.cars)) {
|
915
992
|
return;
|
916
993
|
}
|
917
994
|
const carHeader = await this.loadCarHeaderFromMeta(meta);
|
@@ -920,45 +997,59 @@ var Loader = class {
|
|
920
997
|
this.carLog = [...uniqueCids([meta.cars, ...this.carLog, ...carHeader.cars], this.seenCompacted)];
|
921
998
|
await this.ebOpts.applyMeta?.(carHeader.meta);
|
922
999
|
}
|
923
|
-
async ingestKeyFromMeta(meta) {
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
}
|
1000
|
+
// protected async ingestKeyFromMeta(meta: DbMeta): Promise<void> {
|
1001
|
+
// const { key } = meta;
|
1002
|
+
// if (key) {
|
1003
|
+
// await this.setKey(key);
|
1004
|
+
// }
|
1005
|
+
// }
|
929
1006
|
async loadCarHeaderFromMeta({ cars: cids }) {
|
930
1007
|
const reader = await this.loadCar(cids[0]);
|
931
1008
|
return await parseCarFile(reader, this.logger);
|
932
1009
|
}
|
933
|
-
async _getKey() {
|
934
|
-
|
935
|
-
|
936
|
-
|
937
|
-
|
938
|
-
|
939
|
-
|
940
|
-
|
941
|
-
|
942
|
-
}
|
943
|
-
// can these skip the queue? or have a file queue?
|
944
|
-
async _commitInternalFiles(t, done, opts = { noLoader: false, compact: false }) {
|
1010
|
+
// async _getKey(): Promise<string | undefined> {
|
1011
|
+
// if (this.key) return this.key;
|
1012
|
+
// // generate a random key
|
1013
|
+
// if (!this.ebOpts.public) {
|
1014
|
+
// await this.setKey(toHexString(this.ebOpts.crypto.randomBytes(32)));
|
1015
|
+
// }
|
1016
|
+
// return this.key || undefined;
|
1017
|
+
// }
|
1018
|
+
async commitFiles(t, done) {
|
945
1019
|
await this.ready();
|
946
|
-
const
|
947
|
-
const
|
948
|
-
|
949
|
-
for (const car of cars) {
|
950
|
-
const { cid, bytes } = car;
|
951
|
-
await (await this.fileStore()).save({ cid, bytes });
|
952
|
-
await (await this.remoteWAL()).enqueueFile(cid, !!opts.public);
|
953
|
-
cids.push(cid);
|
954
|
-
}
|
955
|
-
return cids;
|
1020
|
+
const fstore = await this.fileStore();
|
1021
|
+
const wstore = await this.WALStore();
|
1022
|
+
return this.commitQueue.enqueue(() => commitFiles(fstore, wstore, t, done));
|
956
1023
|
}
|
957
|
-
async loadFileCar(cid
|
958
|
-
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore
|
1024
|
+
async loadFileCar(cid) {
|
1025
|
+
return await this.storesLoadCar(cid, await this.fileStore(), this.remoteFileStore);
|
959
1026
|
}
|
960
1027
|
async commit(t, done, opts = { noLoader: false, compact: false }) {
|
961
|
-
|
1028
|
+
await this.ready();
|
1029
|
+
const fstore = await this.fileStore();
|
1030
|
+
const params = {
|
1031
|
+
encoder: (await fstore.keyedCrypto()).codec(),
|
1032
|
+
carLog: this.carLog,
|
1033
|
+
carStore: fstore,
|
1034
|
+
WALStore: await this.WALStore(),
|
1035
|
+
metaStore: await this.metaStore()
|
1036
|
+
};
|
1037
|
+
return this.commitQueue.enqueue(async () => {
|
1038
|
+
await this.cacheTransaction(t);
|
1039
|
+
const ret = await commit(params, t, done, opts);
|
1040
|
+
await this.updateCarLog(ret.cgrp, ret.header, !!opts.compact);
|
1041
|
+
return ret.cgrp;
|
1042
|
+
});
|
1043
|
+
}
|
1044
|
+
async updateCarLog(cids, fp, compact) {
|
1045
|
+
if (compact) {
|
1046
|
+
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1047
|
+
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1048
|
+
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1049
|
+
await this.removeCidsForCompact(previousCompactCid[0]).catch((e) => e);
|
1050
|
+
} else {
|
1051
|
+
this.carLog.unshift(cids);
|
1052
|
+
}
|
962
1053
|
}
|
963
1054
|
async cacheTransaction(t) {
|
964
1055
|
for await (const block of t.entries()) {
|
@@ -978,79 +1069,6 @@ var Loader = class {
|
|
978
1069
|
}
|
979
1070
|
}
|
980
1071
|
}
|
981
|
-
async _commitInternal(t, done, opts = { noLoader: false, compact: false }) {
|
982
|
-
await this.ready();
|
983
|
-
const fp = this.makeCarHeader(done, this.carLog, !!opts.compact);
|
984
|
-
const rootBlock = await encodeCarHeader(fp);
|
985
|
-
const cars = await this.prepareCarFiles(rootBlock, t, !!opts.public);
|
986
|
-
const cids = [];
|
987
|
-
for (const car of cars) {
|
988
|
-
const { cid, bytes } = car;
|
989
|
-
await (await this.carStore()).save({ cid, bytes });
|
990
|
-
cids.push(cid);
|
991
|
-
}
|
992
|
-
await this.cacheTransaction(t);
|
993
|
-
const newDbMeta = { cars: cids, key: this.key || null };
|
994
|
-
await (await this.remoteWAL()).enqueue(newDbMeta, opts);
|
995
|
-
await (await this.metaStore()).save(newDbMeta);
|
996
|
-
await this.updateCarLog(cids, fp, !!opts.compact);
|
997
|
-
return cids;
|
998
|
-
}
|
999
|
-
async prepareCarFilesFiles(roots, t, isPublic) {
|
1000
|
-
const theKey = isPublic ? null : await this._getKey();
|
1001
|
-
const car = theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, roots[0], t) : await encodeCarFile(roots, t);
|
1002
|
-
return [car];
|
1003
|
-
}
|
1004
|
-
async prepareCarFiles(rootBlock, t, isPublic) {
|
1005
|
-
const theKey = isPublic ? void 0 : await this._getKey();
|
1006
|
-
const carFiles = [];
|
1007
|
-
const threshold = this.ebOpts.threshold || 1e3 * 1e3;
|
1008
|
-
let clonedt = new CarTransaction(t.parent, { add: false });
|
1009
|
-
clonedt.putSync(rootBlock.cid, rootBlock.bytes);
|
1010
|
-
let newsize = CBW2.blockLength(toCIDBlock(rootBlock));
|
1011
|
-
let cidRootBlock = rootBlock;
|
1012
|
-
for (const { cid, bytes } of t.entries()) {
|
1013
|
-
newsize += CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
1014
|
-
if (newsize >= threshold) {
|
1015
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
1016
|
-
clonedt = new CarTransaction(t.parent, { add: false });
|
1017
|
-
clonedt.putSync(cid, bytes);
|
1018
|
-
cidRootBlock = { cid, bytes };
|
1019
|
-
newsize = CBW2.blockLength(toCIDBlock({ cid, bytes }));
|
1020
|
-
} else {
|
1021
|
-
clonedt.putSync(cid, bytes);
|
1022
|
-
}
|
1023
|
-
}
|
1024
|
-
carFiles.push(await this.createCarFile(theKey, cidRootBlock.cid, clonedt));
|
1025
|
-
return carFiles;
|
1026
|
-
}
|
1027
|
-
async createCarFile(theKey, cid, t) {
|
1028
|
-
try {
|
1029
|
-
return theKey && this.ebOpts.crypto ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t) : await encodeCarFile([cid], t);
|
1030
|
-
} catch (e) {
|
1031
|
-
console.error("error creating car file", e);
|
1032
|
-
throw e;
|
1033
|
-
}
|
1034
|
-
}
|
1035
|
-
makeFileCarHeader(result) {
|
1036
|
-
const files = [];
|
1037
|
-
for (const [, meta] of Object.entries(result.files || {})) {
|
1038
|
-
if (meta && typeof meta === "object" && "cid" in meta && meta !== null) {
|
1039
|
-
files.push(meta.cid);
|
1040
|
-
}
|
1041
|
-
}
|
1042
|
-
return { ...result, files };
|
1043
|
-
}
|
1044
|
-
async updateCarLog(cids, fp, compact) {
|
1045
|
-
if (compact) {
|
1046
|
-
const previousCompactCid = fp.compact[fp.compact.length - 1];
|
1047
|
-
fp.compact.map((c) => c.toString()).forEach(this.seenCompacted.add, this.seenCompacted);
|
1048
|
-
this.carLog = [...uniqueCids([...this.carLog, ...fp.cars, cids], this.seenCompacted)];
|
1049
|
-
await this.removeCidsForCompact(previousCompactCid[0]);
|
1050
|
-
} else {
|
1051
|
-
this.carLog.unshift(cids);
|
1052
|
-
}
|
1053
|
-
}
|
1054
1072
|
async removeCidsForCompact(cid) {
|
1055
1073
|
const carHeader = await this.loadCarHeaderFromMeta({
|
1056
1074
|
cars: [cid]
|
@@ -1069,9 +1087,9 @@ var Loader = class {
|
|
1069
1087
|
// await this.remoteWAL!.enqueue(dbMeta, { public: false })
|
1070
1088
|
// }
|
1071
1089
|
// }
|
1072
|
-
async *entries(
|
1090
|
+
async *entries(cache2 = true) {
|
1073
1091
|
await this.ready();
|
1074
|
-
if (
|
1092
|
+
if (cache2) {
|
1075
1093
|
for (const [, block] of this.getBlockCache) {
|
1076
1094
|
yield block;
|
1077
1095
|
}
|
@@ -1153,10 +1171,6 @@ var Loader = class {
|
|
1153
1171
|
}
|
1154
1172
|
return got;
|
1155
1173
|
}
|
1156
|
-
makeCarHeader(meta, cars, compact = false) {
|
1157
|
-
const coreHeader = compact ? { cars: [], compact: cars } : { cars, compact: [] };
|
1158
|
-
return { ...coreHeader, meta };
|
1159
|
-
}
|
1160
1174
|
async loadCar(cid) {
|
1161
1175
|
if (!this.carStore) {
|
1162
1176
|
throw this.logger.Error().Msg("car store not initialized").AsError();
|
@@ -1164,72 +1178,52 @@ var Loader = class {
|
|
1164
1178
|
const loaded = await this.storesLoadCar(cid, await this.carStore(), this.remoteCarStore);
|
1165
1179
|
return loaded;
|
1166
1180
|
}
|
1167
|
-
|
1168
|
-
async storesLoadCar(cid, local, remote, publicFiles) {
|
1181
|
+
async makeDecoderAndCarReader(cid, local, remote) {
|
1169
1182
|
const cidsString = cid.toString();
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
|
1182
|
-
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
|
1187
|
-
|
1188
|
-
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1189
|
-
}
|
1190
|
-
}
|
1191
|
-
if (!loadedCar) {
|
1192
|
-
throw this.logger.Error().Url(local.url).Str("cid", cidsString).Msg("missing car files").AsError();
|
1193
|
-
}
|
1194
|
-
const rawReader = await CarReader.fromBytes(loadedCar.bytes);
|
1195
|
-
const readerP = publicFiles ? Promise.resolve(rawReader) : this.ensureDecryptedReader(rawReader);
|
1196
|
-
const cachedReaderP = readerP.then(async (reader) => {
|
1197
|
-
await this.cacheCarReader(cidsString, reader).catch(() => {
|
1198
|
-
return;
|
1199
|
-
});
|
1200
|
-
return reader;
|
1201
|
-
});
|
1202
|
-
this.carReaders.set(cidsString, cachedReaderP);
|
1203
|
-
return readerP;
|
1204
|
-
})().catch((e) => {
|
1205
|
-
this.carReaders.delete(cidsString);
|
1206
|
-
throw e;
|
1207
|
-
})
|
1208
|
-
);
|
1183
|
+
let loadedCar = void 0;
|
1184
|
+
let activeStore = local;
|
1185
|
+
try {
|
1186
|
+
this.logger.Debug().Str("cid", cidsString).Msg("loading car");
|
1187
|
+
loadedCar = await local.load(cid);
|
1188
|
+
this.logger.Debug().Bool("loadedCar", loadedCar).Msg("loaded");
|
1189
|
+
} catch (e) {
|
1190
|
+
if (remote) {
|
1191
|
+
const remoteCar = await remote.load(cid);
|
1192
|
+
if (remoteCar) {
|
1193
|
+
this.logger.Debug().Ref("cid", remoteCar.cid).Msg("saving remote car locally");
|
1194
|
+
await local.save(remoteCar);
|
1195
|
+
loadedCar = remoteCar;
|
1196
|
+
activeStore = remote;
|
1197
|
+
}
|
1198
|
+
} else {
|
1199
|
+
this.logger.Error().Str("cid", cidsString).Err(e).Msg("loading car");
|
1200
|
+
}
|
1209
1201
|
}
|
1210
|
-
|
1211
|
-
|
1212
|
-
async ensureDecryptedReader(reader) {
|
1213
|
-
const theKey = await this._getKey();
|
1214
|
-
if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) {
|
1215
|
-
return reader;
|
1202
|
+
if (!loadedCar) {
|
1203
|
+
throw this.logger.Error().Url(local.url()).Str("cid", cidsString).Msg("missing car files").AsError();
|
1216
1204
|
}
|
1217
|
-
const {
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1205
|
+
const bytes = await decode({ bytes: loadedCar.bytes, hasher: hasher3, codec: (await activeStore.keyedCrypto()).codec() });
|
1206
|
+
const rawReader = await CarReader.fromBytes(bytes.value);
|
1207
|
+
const readerP = Promise.resolve(rawReader);
|
1208
|
+
const cachedReaderP = readerP.then(async (reader) => {
|
1209
|
+
await this.cacheCarReader(cidsString, reader).catch((e) => {
|
1210
|
+
this.logger.Error().Err(e).Str("cid", cidsString).Msg("error caching car reader");
|
1211
|
+
return;
|
1212
|
+
});
|
1213
|
+
return reader;
|
1214
|
+
});
|
1215
|
+
this.carReaders.set(cidsString, cachedReaderP);
|
1216
|
+
return readerP;
|
1223
1217
|
}
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1232
|
-
|
1218
|
+
//What if instead it returns an Array of CarHeader
|
1219
|
+
async storesLoadCar(cid, local, remote) {
|
1220
|
+
const cidsString = cid.toString();
|
1221
|
+
let dacr = this.carReaders.get(cidsString);
|
1222
|
+
if (!dacr) {
|
1223
|
+
dacr = this.makeDecoderAndCarReader(cid, local, remote);
|
1224
|
+
this.carReaders.set(cidsString, dacr);
|
1225
|
+
}
|
1226
|
+
return dacr;
|
1233
1227
|
}
|
1234
1228
|
async getMoreReaders(cids) {
|
1235
1229
|
const limit = pLimit(5);
|
@@ -1238,127 +1232,441 @@ var Loader = class {
|
|
1238
1232
|
}
|
1239
1233
|
};
|
1240
1234
|
|
1241
|
-
// src/
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1235
|
+
// src/runtime/keyed-crypto.ts
|
1236
|
+
var keyed_crypto_exports = {};
|
1237
|
+
__export(keyed_crypto_exports, {
|
1238
|
+
BlockIvKeyIdCodec: () => BlockIvKeyIdCodec,
|
1239
|
+
keyedCryptoFactory: () => keyedCryptoFactory
|
1240
|
+
});
|
1241
|
+
import { base58btc as base58btc2 } from "multiformats/bases/base58";
|
1242
|
+
import { sha256 as hasher4 } from "multiformats/hashes/sha2";
|
1243
|
+
import * as CBOR from "cborg";
|
1244
|
+
var generateIV = {
|
1245
|
+
random: {
|
1246
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1247
|
+
calc: async (ko, crypto, data) => {
|
1248
|
+
return crypto.randomBytes(ko.ivLength);
|
1249
|
+
},
|
1250
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1251
|
+
verify: async (ko, crypto, iv, data) => {
|
1252
|
+
return true;
|
1253
|
+
}
|
1254
|
+
},
|
1255
|
+
hash: {
|
1256
|
+
calc: async (ko, crypto, data) => {
|
1257
|
+
const hash = await hasher4.digest(data);
|
1258
|
+
const hashBytes = new Uint8Array(hash.bytes);
|
1259
|
+
const hashArray = new Uint8Array(ko.ivLength);
|
1260
|
+
for (let i = 0; i < hashBytes.length; i++) {
|
1261
|
+
hashArray[i % ko.ivLength] ^= hashBytes[i];
|
1262
|
+
}
|
1263
|
+
return hashArray;
|
1264
|
+
},
|
1265
|
+
verify: async function(ko, crypto, iv, data) {
|
1266
|
+
return ko.url.getParam("ivverify") !== "disable" && UInt8ArrayEqual(iv, await this.calc(ko, crypto, data));
|
1267
|
+
}
|
1245
1268
|
}
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
|
1252
|
-
|
1269
|
+
};
|
1270
|
+
function getGenerateIVFn(url, opts) {
|
1271
|
+
const ivhash = opts.ivCalc || url.getParam("ivhash") || "hash";
|
1272
|
+
return generateIV[ivhash] || generateIV["hash"];
|
1273
|
+
}
|
1274
|
+
var BlockIvKeyIdCodec = class {
|
1275
|
+
constructor(ko, iv, opts) {
|
1276
|
+
this.code = 3147065;
|
1277
|
+
this.name = "Fireproof@encrypted-block:aes-gcm";
|
1278
|
+
this.ko = ko;
|
1279
|
+
this.iv = iv;
|
1280
|
+
this.opts = opts || {};
|
1281
|
+
}
|
1282
|
+
async encode(data) {
|
1283
|
+
const calcIv = this.iv || await getGenerateIVFn(this.ko.url, this.opts).calc(this.ko, this.ko.crypto, data);
|
1284
|
+
const { iv } = this.ko.algo(calcIv);
|
1285
|
+
const fprt = await this.ko.fingerPrint();
|
1286
|
+
const keyId = base58btc2.decode(fprt);
|
1287
|
+
this.ko.logger.Debug().Str("fp", fprt).Msg("encode");
|
1288
|
+
return CBOR.encode({
|
1289
|
+
iv,
|
1290
|
+
keyId,
|
1291
|
+
data: await this.ko._encrypt({ iv, bytes: data })
|
1292
|
+
});
|
1293
|
+
}
|
1294
|
+
async decode(abytes) {
|
1295
|
+
let bytes;
|
1296
|
+
if (abytes instanceof Uint8Array) {
|
1297
|
+
bytes = abytes;
|
1298
|
+
} else {
|
1299
|
+
bytes = new Uint8Array(abytes);
|
1300
|
+
}
|
1301
|
+
const { iv, keyId, data } = CBOR.decode(bytes);
|
1302
|
+
const fprt = await this.ko.fingerPrint();
|
1303
|
+
this.ko.logger.Debug().Str("fp", base58btc2.encode(keyId)).Msg("decode");
|
1304
|
+
if (base58btc2.encode(keyId) !== fprt) {
|
1305
|
+
throw this.ko.logger.Error().Str("fp", fprt).Str("keyId", base58btc2.encode(keyId)).Msg("keyId mismatch").AsError();
|
1306
|
+
}
|
1307
|
+
const result = await this.ko._decrypt({ iv, bytes: data });
|
1308
|
+
if (!this.opts?.noIVVerify && !await getGenerateIVFn(this.ko.url, this.opts).verify(this.ko, this.ko.crypto, iv, result)) {
|
1309
|
+
throw this.ko.logger.Error().Msg("iv missmatch").AsError();
|
1310
|
+
}
|
1311
|
+
return result;
|
1312
|
+
}
|
1313
|
+
};
|
1314
|
+
var keyedCrypto = class {
|
1315
|
+
constructor(url, key, cyopt, sthis) {
|
1316
|
+
this.ivLength = 12;
|
1317
|
+
this.isEncrypting = true;
|
1318
|
+
this.logger = ensureLogger(sthis, "keyedCrypto");
|
1319
|
+
this.crypto = cyopt;
|
1320
|
+
this.key = key;
|
1253
1321
|
this.url = url;
|
1254
|
-
this.logger = logger;
|
1255
1322
|
}
|
1256
|
-
|
1257
|
-
this.
|
1323
|
+
fingerPrint() {
|
1324
|
+
return Promise.resolve(this.key.fingerPrint);
|
1258
1325
|
}
|
1259
|
-
|
1260
|
-
this
|
1326
|
+
codec(iv, opts) {
|
1327
|
+
return new BlockIvKeyIdCodec(this, iv, opts);
|
1328
|
+
}
|
1329
|
+
algo(iv) {
|
1330
|
+
return {
|
1331
|
+
name: "AES-GCM",
|
1332
|
+
iv: iv || this.crypto.randomBytes(this.ivLength),
|
1333
|
+
tagLength: 128
|
1334
|
+
};
|
1335
|
+
}
|
1336
|
+
async _decrypt(data) {
|
1337
|
+
this.logger.Debug().Len(data.bytes, "bytes").Len(data.iv, "iv").Str("fp", this.key.fingerPrint).Msg("decrypting");
|
1338
|
+
return new Uint8Array(await this.crypto.decrypt(this.algo(data.iv), this.key.key, data.bytes));
|
1339
|
+
}
|
1340
|
+
async _encrypt(data) {
|
1341
|
+
this.logger.Debug().Len(data.bytes).Str("fp", this.key.fingerPrint).Msg("encrypting");
|
1342
|
+
const a = this.algo(data.iv);
|
1343
|
+
return new Uint8Array(await this.crypto.encrypt(a, this.key.key, data.bytes));
|
1261
1344
|
}
|
1262
1345
|
};
|
1263
|
-
var
|
1264
|
-
|
1265
|
-
|
1266
|
-
|
1267
|
-
super(name, url, ensureLogger(logger, "MetaStore", {}));
|
1268
|
-
this.tag = "header-base";
|
1269
|
-
this.gateway = gateway;
|
1270
|
-
}
|
1271
|
-
makeHeader({ cars, key }) {
|
1272
|
-
const toEncode = { cars };
|
1273
|
-
if (key) toEncode.key = key;
|
1274
|
-
return format(toEncode);
|
1275
|
-
}
|
1276
|
-
parseHeader(headerData) {
|
1277
|
-
const got = parse(headerData);
|
1278
|
-
return got;
|
1346
|
+
var nullCodec = class {
|
1347
|
+
constructor() {
|
1348
|
+
this.code = 0;
|
1349
|
+
this.name = "Fireproof@unencrypted-block";
|
1279
1350
|
}
|
1280
|
-
|
1281
|
-
|
1282
|
-
const res = await this.gateway.start(this.url);
|
1283
|
-
if (res.isErr()) {
|
1284
|
-
return res;
|
1285
|
-
}
|
1286
|
-
this._onStarted.forEach((fn) => fn());
|
1287
|
-
return guardVersion(this.url);
|
1351
|
+
encode(data) {
|
1352
|
+
return data;
|
1288
1353
|
}
|
1289
|
-
|
1290
|
-
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1297
|
-
|
1298
|
-
|
1354
|
+
decode(data) {
|
1355
|
+
return data;
|
1356
|
+
}
|
1357
|
+
};
|
1358
|
+
var noCrypto = class {
|
1359
|
+
constructor(url, cyrt, sthis) {
|
1360
|
+
this.ivLength = 0;
|
1361
|
+
this.code = 0;
|
1362
|
+
this.name = "Fireproof@unencrypted-block";
|
1363
|
+
this.isEncrypting = false;
|
1364
|
+
this._fingerPrint = "noCrypto:" + Math.random();
|
1365
|
+
this.logger = ensureLogger(sthis, "noCrypto");
|
1366
|
+
this.crypto = cyrt;
|
1367
|
+
this.url = url;
|
1368
|
+
}
|
1369
|
+
fingerPrint() {
|
1370
|
+
return Promise.resolve(this._fingerPrint);
|
1371
|
+
}
|
1372
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1373
|
+
codec(iv) {
|
1374
|
+
return new nullCodec();
|
1375
|
+
}
|
1376
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1377
|
+
algo(iv) {
|
1378
|
+
return {
|
1379
|
+
name: "noCrypto",
|
1380
|
+
iv: new Uint8Array(),
|
1381
|
+
tagLength: 0
|
1382
|
+
};
|
1383
|
+
}
|
1384
|
+
_decrypt() {
|
1385
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1386
|
+
}
|
1387
|
+
_encrypt() {
|
1388
|
+
throw this.logger.Error().Msg("noCrypto.decrypt not implemented").AsError();
|
1389
|
+
}
|
1390
|
+
};
|
1391
|
+
async function keyedCryptoFactory(url, kb, sthis) {
|
1392
|
+
const storekey = url.getParam("storekey");
|
1393
|
+
if (storekey && storekey !== "insecure") {
|
1394
|
+
let rkey = await kb.getNamedKey(storekey, true);
|
1395
|
+
if (rkey.isErr()) {
|
1396
|
+
try {
|
1397
|
+
rkey = await kb.toKeyWithFingerPrint(storekey);
|
1398
|
+
} catch (e) {
|
1399
|
+
throw sthis.logger.Error().Err(e).Str("keybag", kb.rt.id()).Str("name", storekey).Msg("getNamedKey failed").AsError();
|
1299
1400
|
}
|
1300
|
-
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
1301
|
-
}
|
1302
|
-
try {
|
1303
|
-
return [this.parseHeader(textDecoder.decode(bytes.Ok()))];
|
1304
|
-
} catch (e) {
|
1305
|
-
throw this.logger.Error().Err(e).Msg("parseHeader").AsError();
|
1306
1401
|
}
|
1402
|
+
return new keyedCrypto(url, rkey.Ok(), kb.rt.crypto, sthis);
|
1307
1403
|
}
|
1308
|
-
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
|
1314
|
-
|
1315
|
-
|
1404
|
+
return new noCrypto(url, kb.rt.crypto, sthis);
|
1405
|
+
}
|
1406
|
+
|
1407
|
+
// src/blockstore/fragment-gateway.ts
|
1408
|
+
import { Result as Result3 } from "@adviser/cement";
|
1409
|
+
import { base58btc as base58btc3 } from "multiformats/bases/base58";
|
1410
|
+
import { encode as encode3, decode as decode3 } from "cborg";
|
1411
|
+
function getFragSize(url) {
|
1412
|
+
const fragSize = url.getParam("fragSize");
|
1413
|
+
let ret = 0;
|
1414
|
+
if (fragSize) {
|
1415
|
+
ret = parseInt(fragSize);
|
1416
|
+
}
|
1417
|
+
if (isNaN(ret) || ret <= 0) {
|
1418
|
+
ret = 0;
|
1419
|
+
}
|
1420
|
+
return ret;
|
1421
|
+
}
|
1422
|
+
async function getFrags(url, innerGW, headerSize, logger) {
|
1423
|
+
const fragSize = getFragSize(url);
|
1424
|
+
if (!fragSize) {
|
1425
|
+
const res = await innerGW.get(url);
|
1316
1426
|
if (res.isErr()) {
|
1317
|
-
|
1427
|
+
return [res];
|
1428
|
+
}
|
1429
|
+
const data = res.unwrap();
|
1430
|
+
return [
|
1431
|
+
Result3.Ok({
|
1432
|
+
fid: new Uint8Array(0),
|
1433
|
+
ofs: 0,
|
1434
|
+
len: data.length,
|
1435
|
+
data
|
1436
|
+
})
|
1437
|
+
];
|
1438
|
+
}
|
1439
|
+
const firstRaw = await innerGW.get(url.build().setParam("ofs", "0").URI());
|
1440
|
+
if (firstRaw.isErr()) {
|
1441
|
+
return [firstRaw];
|
1442
|
+
}
|
1443
|
+
const firstFragment = decode3(firstRaw.unwrap());
|
1444
|
+
const blockSize = firstFragment.data.length;
|
1445
|
+
const ops = [Promise.resolve(Result3.Ok(firstFragment))];
|
1446
|
+
const fidStr = base58btc3.encode(firstFragment.fid);
|
1447
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", firstFragment.len.toString()).setParam("headerSize", headerSize.toString());
|
1448
|
+
for (let ofs = blockSize; ofs < firstFragment.len; ofs += blockSize) {
|
1449
|
+
ops.push(
|
1450
|
+
(async (furl, ofs2) => {
|
1451
|
+
const raw2 = await innerGW.get(furl);
|
1452
|
+
if (raw2.isErr()) {
|
1453
|
+
return raw2;
|
1454
|
+
}
|
1455
|
+
const fragment = decode3(raw2.unwrap());
|
1456
|
+
if (base58btc3.encode(fragment.fid) !== fidStr) {
|
1457
|
+
return Result3.Err(logger.Error().Msg("Fragment fid mismatch").AsError());
|
1458
|
+
}
|
1459
|
+
if (fragment.ofs !== ofs2) {
|
1460
|
+
return Result3.Err(logger.Error().Uint64("ofs", ofs2).Msg("Fragment ofs mismatch").AsError());
|
1461
|
+
}
|
1462
|
+
return Result3.Ok(fragment);
|
1463
|
+
})(fragUrl.setParam("ofs", ofs.toString()).URI(), ofs)
|
1464
|
+
);
|
1465
|
+
}
|
1466
|
+
return Promise.all(ops);
|
1467
|
+
}
|
1468
|
+
var FragmentGateway = class {
|
1469
|
+
constructor(sthis, innerGW) {
|
1470
|
+
this.fidLength = 4;
|
1471
|
+
this.headerSize = 32;
|
1472
|
+
this.sthis = ensureSuperLog(sthis, "FragmentGateway");
|
1473
|
+
this.logger = this.sthis.logger;
|
1474
|
+
this.innerGW = innerGW;
|
1475
|
+
}
|
1476
|
+
slicer(url, body) {
|
1477
|
+
const fragSize = getFragSize(url);
|
1478
|
+
if (!fragSize) {
|
1479
|
+
return [this.innerGW.put(url, body)];
|
1480
|
+
}
|
1481
|
+
const blocksize = fragSize - this.headerSize;
|
1482
|
+
if (blocksize <= 0) {
|
1483
|
+
throw this.logger.Error().Uint64("fragSize", fragSize).Uint64("headerSize", this.headerSize).Msg("Fragment size is too small").AsError();
|
1484
|
+
}
|
1485
|
+
const ops = [];
|
1486
|
+
const fid = this.sthis.nextId(this.fidLength);
|
1487
|
+
const fragUrl = url.build().setParam("fid", fid.str).setParam("len", body.length.toString()).setParam("headerSize", this.headerSize.toString());
|
1488
|
+
for (let ofs = 0; ofs < body.length; ofs += blocksize) {
|
1489
|
+
const block = encode3({
|
1490
|
+
fid: fid.bin,
|
1491
|
+
ofs,
|
1492
|
+
len: body.length,
|
1493
|
+
data: body.slice(ofs, ofs + blocksize)
|
1494
|
+
});
|
1495
|
+
if (block.length > fragSize) {
|
1496
|
+
throw this.logger.Error().Uint64("block", block.length).Uint64("fragSize", fragSize).Msg("Block size to big").AsError();
|
1497
|
+
}
|
1498
|
+
ops.push(this.innerGW.put(fragUrl.setParam("ofs", ofs.toString()).URI(), block));
|
1499
|
+
}
|
1500
|
+
return ops;
|
1501
|
+
}
|
1502
|
+
buildUrl(baseUrl, key) {
|
1503
|
+
return this.innerGW.buildUrl(baseUrl, key);
|
1504
|
+
}
|
1505
|
+
async destroy(iurl) {
|
1506
|
+
return this.innerGW.destroy(iurl);
|
1507
|
+
}
|
1508
|
+
async start(url) {
|
1509
|
+
this.headerSize = encode3({
|
1510
|
+
fid: this.sthis.nextId(this.fidLength).bin,
|
1511
|
+
ofs: 1024 * 1024,
|
1512
|
+
// 32bit
|
1513
|
+
len: 16 * 1024 * 1024,
|
1514
|
+
// 32bit
|
1515
|
+
data: new Uint8Array(1024)
|
1516
|
+
}).length - 1024;
|
1517
|
+
return this.innerGW.start(url);
|
1518
|
+
}
|
1519
|
+
async close(url) {
|
1520
|
+
return this.innerGW.close(url);
|
1521
|
+
}
|
1522
|
+
async put(url, body) {
|
1523
|
+
await Promise.all(this.slicer(url, body));
|
1524
|
+
return Result3.Ok(void 0);
|
1525
|
+
}
|
1526
|
+
async get(url) {
|
1527
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1528
|
+
let buffer = void 0;
|
1529
|
+
for (const rfrag of rfrags) {
|
1530
|
+
if (rfrag.isErr()) {
|
1531
|
+
return Result3.Err(rfrag.Err());
|
1532
|
+
}
|
1533
|
+
const frag = rfrag.Ok();
|
1534
|
+
buffer = buffer || new Uint8Array(frag.len);
|
1535
|
+
buffer.set(frag.data, frag.ofs);
|
1318
1536
|
}
|
1319
|
-
return
|
1537
|
+
return Result3.Ok(buffer || new Uint8Array(0));
|
1320
1538
|
}
|
1321
|
-
async
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1539
|
+
async subscribe(url, callback) {
|
1540
|
+
if (this.innerGW.subscribe) {
|
1541
|
+
return this.innerGW.subscribe(url, callback);
|
1542
|
+
} else {
|
1543
|
+
let lastData = void 0;
|
1544
|
+
let interval = 100;
|
1545
|
+
const fetchData = async () => {
|
1546
|
+
const result = await this.innerGW.get(url);
|
1547
|
+
if (result.isOk()) {
|
1548
|
+
const data = result.Ok();
|
1549
|
+
if (!lastData || !data.every((value, index2) => lastData && value === lastData[index2])) {
|
1550
|
+
lastData = data;
|
1551
|
+
callback(data);
|
1552
|
+
interval = 100;
|
1553
|
+
} else {
|
1554
|
+
interval *= 2;
|
1555
|
+
}
|
1556
|
+
}
|
1557
|
+
timeoutId = setTimeout(fetchData, interval);
|
1558
|
+
};
|
1559
|
+
let timeoutId = setTimeout(fetchData, interval);
|
1560
|
+
return Result3.Ok(() => {
|
1561
|
+
clearTimeout(timeoutId);
|
1562
|
+
});
|
1563
|
+
}
|
1325
1564
|
}
|
1326
|
-
async
|
1327
|
-
|
1565
|
+
async delete(url) {
|
1566
|
+
const rfrags = await getFrags(url, this.innerGW, this.headerSize, this.logger);
|
1567
|
+
for (const rfrag of rfrags) {
|
1568
|
+
if (rfrag.isErr()) {
|
1569
|
+
return Result3.Err(rfrag.Err());
|
1570
|
+
}
|
1571
|
+
const frag = rfrag.Ok();
|
1572
|
+
const fidStr = base58btc3.encode(frag.fid);
|
1573
|
+
const fragUrl = url.build().setParam("fid", fidStr).setParam("len", frag.len.toString()).setParam("headerSize", this.headerSize.toString()).URI();
|
1574
|
+
await this.innerGW.delete(fragUrl);
|
1575
|
+
}
|
1576
|
+
return Result3.Ok(void 0);
|
1328
1577
|
}
|
1329
1578
|
};
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
this.
|
1579
|
+
|
1580
|
+
// src/blockstore/store.ts
|
1581
|
+
function guardVersion(url) {
|
1582
|
+
if (!url.hasParam("version")) {
|
1583
|
+
return Result4.Err(`missing version: ${url.toString()}`);
|
1584
|
+
}
|
1585
|
+
return Result4.Ok(url);
|
1586
|
+
}
|
1587
|
+
var BaseStoreImpl = class {
|
1588
|
+
constructor(name, url, opts, sthis, logger) {
|
1589
|
+
this._onStarted = [];
|
1590
|
+
this._onClosed = [];
|
1591
|
+
this.name = name;
|
1592
|
+
this._url = url;
|
1593
|
+
this.keybag = opts.keybag;
|
1594
|
+
this.sthis = sthis;
|
1595
|
+
this.logger = logger.With().Ref("url", () => this._url.toString()).Str("name", name).Logger();
|
1596
|
+
this.gateway = new FragmentGateway(this.sthis, opts.gateway);
|
1597
|
+
this.loader = opts.loader;
|
1598
|
+
}
|
1599
|
+
url() {
|
1600
|
+
return this._url;
|
1601
|
+
}
|
1602
|
+
onStarted(fn) {
|
1603
|
+
this._onStarted.push(fn);
|
1604
|
+
}
|
1605
|
+
onClosed(fn) {
|
1606
|
+
this._onClosed.push(fn);
|
1607
|
+
}
|
1608
|
+
async keyedCrypto() {
|
1609
|
+
return keyedCryptoFactory(this._url, await this.keybag(), this.sthis);
|
1341
1610
|
}
|
1342
1611
|
async start() {
|
1343
|
-
this.logger.Debug().Msg("starting-gateway");
|
1344
|
-
|
1612
|
+
this.logger.Debug().Str("storeType", this.storeType).Msg("starting-gateway-pre");
|
1613
|
+
this._url = this._url.build().setParam("store", this.storeType).URI();
|
1614
|
+
const res = await this.gateway.start(this._url);
|
1345
1615
|
if (res.isErr()) {
|
1346
1616
|
this.logger.Error().Result("gw-start", res).Msg("started-gateway");
|
1347
1617
|
return res;
|
1348
1618
|
}
|
1349
|
-
this.
|
1350
|
-
const
|
1619
|
+
this._url = res.Ok();
|
1620
|
+
const kb = await this.keybag();
|
1621
|
+
const skRes = await kb.ensureKeyFromUrl(this._url, () => {
|
1622
|
+
const idx = this._url.getParam("index");
|
1623
|
+
const storeKeyName = [this.name];
|
1624
|
+
if (idx) {
|
1625
|
+
storeKeyName.push(idx);
|
1626
|
+
}
|
1627
|
+
storeKeyName.push(this.storeType);
|
1628
|
+
return storeKeyName.join(":");
|
1629
|
+
});
|
1630
|
+
if (skRes.isErr()) {
|
1631
|
+
return skRes;
|
1632
|
+
}
|
1633
|
+
this._url = skRes.Ok();
|
1634
|
+
const version = guardVersion(this._url);
|
1351
1635
|
if (version.isErr()) {
|
1352
1636
|
this.logger.Error().Result("version", version).Msg("guardVersion");
|
1353
1637
|
await this.close();
|
1354
1638
|
return version;
|
1355
1639
|
}
|
1640
|
+
if (this.ready) {
|
1641
|
+
const fn = this.ready.bind(this);
|
1642
|
+
const ready = await exception2Result(fn);
|
1643
|
+
if (ready.isErr()) {
|
1644
|
+
await this.close();
|
1645
|
+
return ready;
|
1646
|
+
}
|
1647
|
+
}
|
1648
|
+
this._onStarted.forEach((fn) => fn());
|
1356
1649
|
this.logger.Debug().Msg("started");
|
1357
1650
|
return version;
|
1358
1651
|
}
|
1652
|
+
};
|
1653
|
+
var DataStoreImpl = class extends BaseStoreImpl {
|
1654
|
+
// readonly tag: string = "car-base";
|
1655
|
+
constructor(sthis, name, url, opts) {
|
1656
|
+
super(
|
1657
|
+
name,
|
1658
|
+
url,
|
1659
|
+
{
|
1660
|
+
...opts
|
1661
|
+
},
|
1662
|
+
sthis,
|
1663
|
+
ensureLogger(sthis, "DataStoreImpl")
|
1664
|
+
);
|
1665
|
+
this.storeType = "data";
|
1666
|
+
}
|
1359
1667
|
async load(cid) {
|
1360
1668
|
this.logger.Debug().Any("cid", cid).Msg("loading");
|
1361
|
-
const url = await this.gateway.buildUrl(this.url, cid.toString());
|
1669
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
1362
1670
|
if (url.isErr()) {
|
1363
1671
|
throw this.logger.Error().Err(url.Err()).Str("cid", cid.toString()).Msg("got error from gateway.buildUrl").AsError();
|
1364
1672
|
}
|
@@ -1371,7 +1679,7 @@ var DataStore = class extends VersionedStore {
|
|
1371
1679
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
1372
1680
|
async save(car, opts) {
|
1373
1681
|
this.logger.Debug().Any("cid", car.cid.toString()).Msg("saving");
|
1374
|
-
const url = await this.gateway.buildUrl(this.url, car.cid.toString());
|
1682
|
+
const url = await this.gateway.buildUrl(this.url(), car.cid.toString());
|
1375
1683
|
if (url.isErr()) {
|
1376
1684
|
throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError();
|
1377
1685
|
}
|
@@ -1382,46 +1690,53 @@ var DataStore = class extends VersionedStore {
|
|
1382
1690
|
return res.Ok();
|
1383
1691
|
}
|
1384
1692
|
async remove(cid) {
|
1385
|
-
const url = await this.gateway.buildUrl(this.url, cid.toString());
|
1693
|
+
const url = await this.gateway.buildUrl(this.url(), cid.toString());
|
1386
1694
|
if (url.isErr()) {
|
1387
1695
|
return url;
|
1388
1696
|
}
|
1389
1697
|
return this.gateway.delete(url.Ok());
|
1390
1698
|
}
|
1391
1699
|
async close() {
|
1392
|
-
await this.gateway.close(this.url);
|
1700
|
+
await this.gateway.close(this.url());
|
1393
1701
|
this._onClosed.forEach((fn) => fn());
|
1394
|
-
return
|
1702
|
+
return Result4.Ok(void 0);
|
1395
1703
|
}
|
1396
1704
|
destroy() {
|
1397
|
-
return this.gateway.destroy(this.url);
|
1705
|
+
return this.gateway.destroy(this.url());
|
1398
1706
|
}
|
1399
1707
|
};
|
1400
|
-
var
|
1401
|
-
constructor(loader, url,
|
1402
|
-
super(
|
1403
|
-
|
1404
|
-
|
1708
|
+
var WALStoreImpl = class extends BaseStoreImpl {
|
1709
|
+
constructor(loader, url, opts) {
|
1710
|
+
super(
|
1711
|
+
loader.name,
|
1712
|
+
url,
|
1713
|
+
{
|
1714
|
+
...opts
|
1715
|
+
},
|
1716
|
+
loader.sthis,
|
1717
|
+
ensureLogger(loader.sthis, "WALStoreImpl")
|
1718
|
+
);
|
1719
|
+
this.storeType = "wal";
|
1720
|
+
this._ready = new ResolveOnce3();
|
1405
1721
|
this.walState = { operations: [], noLoaderOps: [], fileOperations: [] };
|
1406
1722
|
this.processing = void 0;
|
1407
1723
|
this.processQueue = new CommitQueue();
|
1408
|
-
this.
|
1409
|
-
|
1410
|
-
|
1411
|
-
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
|
1724
|
+
this.ready = async () => {
|
1725
|
+
return this._ready.once(async () => {
|
1726
|
+
const walState = await this.load().catch((e) => {
|
1727
|
+
this.logger.Error().Any("error", e).Msg("error loading wal");
|
1728
|
+
return void 0;
|
1729
|
+
});
|
1730
|
+
if (!walState) {
|
1731
|
+
this.walState.operations = [];
|
1732
|
+
this.walState.fileOperations = [];
|
1733
|
+
} else {
|
1734
|
+
this.walState.operations = walState.operations || [];
|
1735
|
+
this.walState.fileOperations = walState.fileOperations || [];
|
1736
|
+
}
|
1416
1737
|
});
|
1417
|
-
|
1418
|
-
|
1419
|
-
this.walState.fileOperations = [];
|
1420
|
-
} else {
|
1421
|
-
this.walState.operations = walState.operations || [];
|
1422
|
-
this.walState.fileOperations = walState.fileOperations || [];
|
1423
|
-
}
|
1424
|
-
});
|
1738
|
+
};
|
1739
|
+
this.loader = loader;
|
1425
1740
|
}
|
1426
1741
|
async enqueue(dbMeta, opts) {
|
1427
1742
|
await this.ready();
|
@@ -1431,19 +1746,23 @@ var RemoteWAL = class extends VersionedStore {
|
|
1431
1746
|
this.walState.operations.push(dbMeta);
|
1432
1747
|
}
|
1433
1748
|
await this.save(this.walState);
|
1434
|
-
void this.
|
1749
|
+
void this.process();
|
1435
1750
|
}
|
1436
1751
|
async enqueueFile(fileCid, publicFile = false) {
|
1437
1752
|
await this.ready();
|
1438
1753
|
this.walState.fileOperations.push({ cid: fileCid, public: publicFile });
|
1439
1754
|
}
|
1440
|
-
async
|
1755
|
+
async process() {
|
1441
1756
|
await this.ready();
|
1442
1757
|
if (!this.loader.remoteCarStore) return;
|
1443
1758
|
await this.processQueue.enqueue(async () => {
|
1444
|
-
|
1759
|
+
try {
|
1760
|
+
await this._doProcess();
|
1761
|
+
} catch (e) {
|
1762
|
+
this.logger.Error().Any("error", e).Msg("error processing wal");
|
1763
|
+
}
|
1445
1764
|
if (this.walState.operations.length || this.walState.fileOperations.length || this.walState.noLoaderOps.length) {
|
1446
|
-
setTimeout(() => void this.
|
1765
|
+
setTimeout(() => void this.process(), 0);
|
1447
1766
|
}
|
1448
1767
|
});
|
1449
1768
|
}
|
@@ -1461,7 +1780,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
1461
1780
|
for (const cid of dbMeta.cars) {
|
1462
1781
|
const car = await (await this.loader.carStore()).load(cid);
|
1463
1782
|
if (!car) {
|
1464
|
-
if (
|
1783
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
1465
1784
|
throw this.logger.Error().Ref("cid", cid).Msg("missing local car").AsError();
|
1466
1785
|
} else {
|
1467
1786
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -1476,7 +1795,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
1476
1795
|
for (const cid of dbMeta.cars) {
|
1477
1796
|
const car = await (await this.loader.carStore()).load(cid).catch(() => null);
|
1478
1797
|
if (!car) {
|
1479
|
-
if (
|
1798
|
+
if (carLogIncludesGroup(this.loader.carLog, dbMeta.cars))
|
1480
1799
|
throw this.logger.Error().Ref("cid", cid).Msg(`missing local car`).AsError();
|
1481
1800
|
} else {
|
1482
1801
|
await throwFalsy(this.loader.remoteCarStore).save(car);
|
@@ -1501,11 +1820,7 @@ var RemoteWAL = class extends VersionedStore {
|
|
1501
1820
|
const res = await Promise.allSettled(uploads);
|
1502
1821
|
const errors = res.filter((r) => r.status === "rejected");
|
1503
1822
|
if (errors.length) {
|
1504
|
-
throw this.logger.Error().Any(
|
1505
|
-
"errors",
|
1506
|
-
errors.map((e) => e.reason)
|
1507
|
-
).Msg("error uploading").AsError();
|
1508
|
-
errors[0].reason;
|
1823
|
+
throw this.logger.Error().Any("errors", errors).Msg("error uploading").AsError();
|
1509
1824
|
}
|
1510
1825
|
if (operations.length) {
|
1511
1826
|
const lastOp = operations[operations.length - 1];
|
@@ -1520,29 +1835,11 @@ var RemoteWAL = class extends VersionedStore {
|
|
1520
1835
|
})();
|
1521
1836
|
await rmlp;
|
1522
1837
|
}
|
1523
|
-
async start() {
|
1524
|
-
const res = await this.gateway.start(this.url);
|
1525
|
-
if (res.isErr()) {
|
1526
|
-
return res;
|
1527
|
-
}
|
1528
|
-
const ver = guardVersion(this.url);
|
1529
|
-
if (ver.isErr()) {
|
1530
|
-
await this.close();
|
1531
|
-
return ver;
|
1532
|
-
}
|
1533
|
-
const ready = await exception2Result(() => this.ready());
|
1534
|
-
this._onStarted.forEach((fn) => fn());
|
1535
|
-
if (ready.isErr()) {
|
1536
|
-
await this.close();
|
1537
|
-
return ready;
|
1538
|
-
}
|
1539
|
-
return ready;
|
1540
|
-
}
|
1541
1838
|
async load() {
|
1542
1839
|
this.logger.Debug().Msg("loading");
|
1543
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
1840
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
1544
1841
|
if (filepath.isErr()) {
|
1545
|
-
throw this.logger.Error().Err(filepath.Err()).
|
1842
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
1546
1843
|
}
|
1547
1844
|
const bytes = await this.gateway.get(filepath.Ok());
|
1548
1845
|
if (bytes.isErr()) {
|
@@ -1552,15 +1849,15 @@ var RemoteWAL = class extends VersionedStore {
|
|
1552
1849
|
throw this.logger.Error().Err(bytes.Err()).Msg("error get").AsError();
|
1553
1850
|
}
|
1554
1851
|
try {
|
1555
|
-
return bytes && parse(
|
1852
|
+
return bytes && parse(this.sthis.txt.decode(bytes.Ok()));
|
1556
1853
|
} catch (e) {
|
1557
1854
|
throw this.logger.Error().Err(e).Msg("error parse").AsError();
|
1558
1855
|
}
|
1559
1856
|
}
|
1560
1857
|
async save(state) {
|
1561
|
-
const filepath = await this.gateway.buildUrl(this.url, "main");
|
1858
|
+
const filepath = await this.gateway.buildUrl(this.url(), "main");
|
1562
1859
|
if (filepath.isErr()) {
|
1563
|
-
throw this.logger.Error().Err(filepath.Err()).
|
1860
|
+
throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError();
|
1564
1861
|
}
|
1565
1862
|
let encoded;
|
1566
1863
|
try {
|
@@ -1568,55 +1865,315 @@ var RemoteWAL = class extends VersionedStore {
|
|
1568
1865
|
} catch (e) {
|
1569
1866
|
throw this.logger.Error().Err(e).Any("state", state).Msg("error format").AsError();
|
1570
1867
|
}
|
1571
|
-
const res = await this.gateway.put(filepath.Ok(),
|
1868
|
+
const res = await this.gateway.put(filepath.Ok(), this.sthis.txt.encode(encoded));
|
1572
1869
|
if (res.isErr()) {
|
1573
1870
|
throw this.logger.Error().Err(res.Err()).Str("filePath", filepath.Ok().toString()).Msg("error saving").AsError();
|
1574
1871
|
}
|
1575
1872
|
}
|
1576
1873
|
async close() {
|
1577
|
-
await this.gateway.close(this.url);
|
1874
|
+
await this.gateway.close(this.url());
|
1578
1875
|
this._onClosed.forEach((fn) => fn());
|
1579
|
-
return
|
1876
|
+
return Result4.Ok(void 0);
|
1580
1877
|
}
|
1581
1878
|
destroy() {
|
1582
|
-
return this.gateway.destroy(this.url);
|
1879
|
+
return this.gateway.destroy(this.url());
|
1583
1880
|
}
|
1584
1881
|
};
|
1585
1882
|
|
1586
|
-
// src/blockstore/store-
|
1587
|
-
|
1588
|
-
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1592
|
-
|
1593
|
-
|
1883
|
+
// src/blockstore/store-meta.ts
|
1884
|
+
import { format as format2, parse as parse2 } from "@ipld/dag-json";
|
1885
|
+
import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock";
|
1886
|
+
import { CID as CID2 } from "multiformats";
|
1887
|
+
import { Result as Result5 } from "@adviser/cement";
|
1888
|
+
|
1889
|
+
// src/runtime/index.ts
|
1890
|
+
var runtime_exports = {};
|
1891
|
+
__export(runtime_exports, {
|
1892
|
+
FILESTORE_VERSION: () => FILESTORE_VERSION,
|
1893
|
+
INDEXDB_VERSION: () => INDEXDB_VERSION,
|
1894
|
+
files: () => files_exports,
|
1895
|
+
getFileName: () => getFileName,
|
1896
|
+
getFileSystem: () => getFileSystem,
|
1897
|
+
getPath: () => getPath,
|
1898
|
+
kb: () => key_bag_exports,
|
1899
|
+
kc: () => keyed_crypto_exports,
|
1900
|
+
mf: () => wait_pr_multiformats_exports,
|
1901
|
+
runtimeFn: () => runtimeFn2,
|
1902
|
+
toArrayBuffer: () => toArrayBuffer
|
1903
|
+
});
|
1904
|
+
|
1905
|
+
// src/runtime/wait-pr-multiformats/index.ts
|
1906
|
+
var wait_pr_multiformats_exports = {};
|
1907
|
+
__export(wait_pr_multiformats_exports, {
|
1908
|
+
block: () => block_exports,
|
1909
|
+
codec: () => codec_interface_exports
|
1910
|
+
});
|
1911
|
+
|
1912
|
+
// src/runtime/wait-pr-multiformats/codec-interface.ts
|
1913
|
+
var codec_interface_exports = {};
|
1914
|
+
|
1915
|
+
// src/runtime/index.ts
|
1916
|
+
import { runtimeFn as runtimeFn2 } from "@adviser/cement";
|
1917
|
+
|
1918
|
+
// src/blockstore/store-meta.ts
|
1919
|
+
async function decodeGatewayMetaBytesToDbMeta(sthis, byteHeads) {
|
1920
|
+
const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads));
|
1921
|
+
return Promise.all(
|
1922
|
+
crdtEntries.map(async (crdtEntry) => {
|
1923
|
+
const eventBlock = await decodeEventBlock(decodeFromBase64(crdtEntry.data));
|
1924
|
+
const dbMeta = parse2(sthis.txt.decode(eventBlock.value.data.dbMeta));
|
1925
|
+
return {
|
1926
|
+
eventCid: eventBlock.cid,
|
1927
|
+
parents: crdtEntry.parents,
|
1928
|
+
dbMeta
|
1929
|
+
};
|
1930
|
+
})
|
1931
|
+
);
|
1932
|
+
}
|
1933
|
+
async function setCryptoKeyFromGatewayMetaPayload(uri, sthis, data) {
|
1934
|
+
try {
|
1935
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Setting crypto key from gateway meta payload");
|
1936
|
+
const keyInfo = await decodeGatewayMetaBytesToDbMeta(sthis, data);
|
1937
|
+
if (keyInfo.length) {
|
1938
|
+
const dbMeta = keyInfo[0].dbMeta;
|
1939
|
+
if (dbMeta.key) {
|
1940
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1941
|
+
const keyName = getStoreKeyName(uri);
|
1942
|
+
const res = await kb.setNamedKey(keyName, dbMeta.key);
|
1943
|
+
if (res.isErr()) {
|
1944
|
+
sthis.logger.Debug().Str("keyName", keyName).Str("dbMeta.key", dbMeta.key).Msg("Failed to set named key");
|
1945
|
+
throw res.Err();
|
1946
|
+
}
|
1947
|
+
}
|
1948
|
+
sthis.logger.Debug().Str("dbMeta.key", dbMeta.key).Str("uri", uri.toString()).Msg("Set crypto key from gateway meta payload");
|
1949
|
+
return Result5.Ok(dbMeta);
|
1950
|
+
}
|
1951
|
+
sthis.logger.Debug().Str("data", new TextDecoder().decode(data)).Msg("No crypto in gateway meta payload");
|
1952
|
+
return Result5.Ok(void 0);
|
1953
|
+
} catch (error) {
|
1954
|
+
sthis.logger.Debug().Err(error).Msg("Failed to set crypto key from gateway meta payload");
|
1955
|
+
return Result5.Err(error);
|
1594
1956
|
}
|
1595
1957
|
}
|
1596
|
-
function
|
1597
|
-
if (pathOrUrl instanceof URL) return ensureIsIndex(pathOrUrl, isIndex);
|
1958
|
+
async function addCryptoKeyToGatewayMetaPayload(uri, sthis, body) {
|
1598
1959
|
try {
|
1599
|
-
|
1600
|
-
|
1601
|
-
|
1602
|
-
const
|
1603
|
-
|
1960
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Adding crypto key to gateway meta payload");
|
1961
|
+
const keyName = getStoreKeyName(uri);
|
1962
|
+
const kb = await key_bag_exports.getKeyBag(sthis);
|
1963
|
+
const res = await kb.getNamedExtractableKey(keyName, true);
|
1964
|
+
if (res.isErr()) {
|
1965
|
+
sthis.logger.Error().Str("keyName", keyName).Msg("Failed to get named extractable key");
|
1966
|
+
throw res.Err();
|
1967
|
+
}
|
1968
|
+
const keyData = await res.Ok().extract();
|
1969
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(sthis, body);
|
1970
|
+
const { dbMeta, parents } = dbMetas[0];
|
1971
|
+
const parentLinks = parents.map((p) => CID2.parse(p));
|
1972
|
+
dbMeta.key = keyData.keyStr;
|
1973
|
+
const events = await Promise.all([dbMeta].map((dbMeta2) => createDbMetaEventBlock(sthis, dbMeta2, parentLinks)));
|
1974
|
+
const encoded = await encodeEventsWithParents(sthis, events, parentLinks);
|
1975
|
+
sthis.logger.Debug().Str("uri", uri.toString()).Msg("Added crypto key to gateway meta payload");
|
1976
|
+
return Result5.Ok(encoded);
|
1977
|
+
} catch (error) {
|
1978
|
+
sthis.logger.Error().Err(error).Msg("Failed to add crypto key to gateway meta payload");
|
1979
|
+
return Result5.Err(error);
|
1980
|
+
}
|
1981
|
+
}
|
1982
|
+
function getStoreKeyName(url) {
|
1983
|
+
const storeKeyName = [url.getParam("localName") || url.getParam("name")];
|
1984
|
+
const idx = url.getParam("index");
|
1985
|
+
if (idx) {
|
1986
|
+
storeKeyName.push(idx);
|
1987
|
+
}
|
1988
|
+
storeKeyName.push("data");
|
1989
|
+
return `@${storeKeyName.join(":")}@`;
|
1990
|
+
}
|
1991
|
+
async function createDbMetaEventBlock(sthis, dbMeta, parents) {
|
1992
|
+
const event = await EventBlock.create(
|
1993
|
+
{
|
1994
|
+
dbMeta: sthis.txt.encode(format2(dbMeta))
|
1995
|
+
},
|
1996
|
+
parents
|
1997
|
+
);
|
1998
|
+
return event;
|
1999
|
+
}
|
2000
|
+
async function encodeEventsWithParents(sthis, events, parents) {
|
2001
|
+
const crdtEntries = events.map((event) => {
|
2002
|
+
const base64String = encodeToBase64(event.bytes);
|
2003
|
+
return {
|
2004
|
+
cid: event.cid.toString(),
|
2005
|
+
data: base64String,
|
2006
|
+
parents: parents.map((p) => p.toString())
|
2007
|
+
};
|
2008
|
+
});
|
2009
|
+
return sthis.txt.encode(JSON.stringify(crdtEntries));
|
2010
|
+
}
|
2011
|
+
var MetaStoreImpl = class extends BaseStoreImpl {
|
2012
|
+
constructor(sthis, name, url, opts, remote) {
|
2013
|
+
super(
|
2014
|
+
name,
|
2015
|
+
url,
|
2016
|
+
{
|
2017
|
+
...opts
|
2018
|
+
},
|
2019
|
+
sthis,
|
2020
|
+
ensureLogger(sthis, "MetaStoreImpl")
|
2021
|
+
);
|
2022
|
+
this.storeType = "meta";
|
2023
|
+
this.subscribers = /* @__PURE__ */ new Map();
|
2024
|
+
this.parents = [];
|
2025
|
+
if (remote && opts.gateway.subscribe) {
|
2026
|
+
this.onStarted(async () => {
|
2027
|
+
this.logger.Debug().Str("url", this.url().toString()).Msg("Subscribing to the gateway");
|
2028
|
+
opts.gateway.subscribe?.(this.url(), async (message) => {
|
2029
|
+
this.logger.Debug().Msg("Received message from gateway");
|
2030
|
+
const dbMetas = await decodeGatewayMetaBytesToDbMeta(this.sthis, message);
|
2031
|
+
await Promise.all(
|
2032
|
+
dbMetas.map((dbMeta) => this.loader?.taskManager?.handleEvent(dbMeta.eventCid, dbMeta.parents, dbMeta.dbMeta))
|
2033
|
+
);
|
2034
|
+
});
|
2035
|
+
});
|
2036
|
+
}
|
2037
|
+
}
|
2038
|
+
async handleByteHeads(byteHeads) {
|
2039
|
+
return await decodeGatewayMetaBytesToDbMeta(this.sthis, byteHeads);
|
2040
|
+
}
|
2041
|
+
async load() {
|
2042
|
+
const branch = "main";
|
2043
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2044
|
+
if (url.isErr()) {
|
2045
|
+
throw this.logger.Error().Result("buildUrl", url).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2046
|
+
}
|
2047
|
+
const bytes = await this.gateway.get(url.Ok());
|
2048
|
+
if (bytes.isErr()) {
|
2049
|
+
if (isNotFoundError(bytes)) {
|
2050
|
+
return void 0;
|
2051
|
+
}
|
2052
|
+
throw this.logger.Error().Url(url.Ok()).Result("bytes:", bytes).Msg("gateway get").AsError();
|
2053
|
+
}
|
2054
|
+
const dbMetas = await this.handleByteHeads(bytes.Ok());
|
2055
|
+
await this.loader?.handleDbMetasFromStore(dbMetas.map((m) => m.dbMeta));
|
2056
|
+
const cids = dbMetas.map((m) => m.eventCid);
|
2057
|
+
const uniqueParentsMap = new Map([...this.parents, ...cids].map((p) => [p.toString(), p]));
|
2058
|
+
this.parents = Array.from(uniqueParentsMap.values());
|
2059
|
+
return dbMetas.map((m) => m.dbMeta);
|
2060
|
+
}
|
2061
|
+
async save(meta, branch) {
|
2062
|
+
branch = branch || "main";
|
2063
|
+
this.logger.Debug().Str("branch", branch).Any("meta", meta).Msg("saving meta");
|
2064
|
+
const event = await createDbMetaEventBlock(this.sthis, meta, this.parents);
|
2065
|
+
const bytes = await encodeEventsWithParents(this.sthis, [event], this.parents);
|
2066
|
+
const url = await this.gateway.buildUrl(this.url(), branch);
|
2067
|
+
if (url.isErr()) {
|
2068
|
+
throw this.logger.Error().Err(url.Err()).Str("branch", branch).Msg("got error from gateway.buildUrl").AsError();
|
2069
|
+
}
|
2070
|
+
const res = await this.gateway.put(url.Ok(), bytes);
|
2071
|
+
if (res.isErr()) {
|
2072
|
+
throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError();
|
2073
|
+
}
|
2074
|
+
await this.loader?.handleDbMetasFromStore([meta]);
|
2075
|
+
this.parents = [event.cid];
|
2076
|
+
return res;
|
2077
|
+
}
|
2078
|
+
async close() {
|
2079
|
+
await this.gateway.close(this.url());
|
2080
|
+
this._onClosed.forEach((fn) => fn());
|
2081
|
+
return Result5.Ok(void 0);
|
2082
|
+
}
|
2083
|
+
async destroy() {
|
2084
|
+
return this.gateway.destroy(this.url());
|
2085
|
+
}
|
2086
|
+
};
|
2087
|
+
function encodeToBase64(bytes) {
|
2088
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2089
|
+
let base64 = "";
|
2090
|
+
let i;
|
2091
|
+
for (i = 0; i < bytes.length - 2; i += 3) {
|
2092
|
+
base64 += chars[bytes[i] >> 2];
|
2093
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2094
|
+
base64 += chars[(bytes[i + 1] & 15) << 2 | bytes[i + 2] >> 6];
|
2095
|
+
base64 += chars[bytes[i + 2] & 63];
|
2096
|
+
}
|
2097
|
+
if (i < bytes.length) {
|
2098
|
+
base64 += chars[bytes[i] >> 2];
|
2099
|
+
if (i === bytes.length - 1) {
|
2100
|
+
base64 += chars[(bytes[i] & 3) << 4];
|
2101
|
+
base64 += "==";
|
2102
|
+
} else {
|
2103
|
+
base64 += chars[(bytes[i] & 3) << 4 | bytes[i + 1] >> 4];
|
2104
|
+
base64 += chars[(bytes[i + 1] & 15) << 2];
|
2105
|
+
base64 += "=";
|
2106
|
+
}
|
1604
2107
|
}
|
2108
|
+
return base64;
|
2109
|
+
}
|
2110
|
+
function decodeFromBase64(base64) {
|
2111
|
+
const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
2112
|
+
const bytes = new Uint8Array(base64.length * 3 / 4);
|
2113
|
+
let i;
|
2114
|
+
let j = 0;
|
2115
|
+
for (i = 0; i < base64.length; i += 4) {
|
2116
|
+
const a = chars.indexOf(base64[i]);
|
2117
|
+
const b = chars.indexOf(base64[i + 1]);
|
2118
|
+
const c = chars.indexOf(base64[i + 2]);
|
2119
|
+
const d = chars.indexOf(base64[i + 3]);
|
2120
|
+
bytes[j++] = a << 2 | b >> 4;
|
2121
|
+
if (base64[i + 2] !== "=") {
|
2122
|
+
bytes[j++] = (b & 15) << 4 | c >> 2;
|
2123
|
+
}
|
2124
|
+
if (base64[i + 3] !== "=") {
|
2125
|
+
bytes[j++] = (c & 3) << 6 | d;
|
2126
|
+
}
|
2127
|
+
}
|
2128
|
+
return bytes.slice(0, j);
|
2129
|
+
}
|
2130
|
+
|
2131
|
+
// src/blockstore/store-factory.ts
|
2132
|
+
function ensureIsIndex(url, isIndex) {
|
2133
|
+
if (isIndex) {
|
2134
|
+
return url.build().setParam("index", isIndex).URI();
|
2135
|
+
}
|
2136
|
+
return url.build().delParam("index").URI();
|
1605
2137
|
}
|
1606
|
-
var storeFactory = /* @__PURE__ */ new Map();
|
1607
2138
|
function ensureName(name, url) {
|
1608
|
-
if (!url.
|
1609
|
-
url.
|
2139
|
+
if (!url.hasParam("name")) {
|
2140
|
+
return url.build().setParam("name", name).URI();
|
1610
2141
|
}
|
2142
|
+
return url;
|
1611
2143
|
}
|
2144
|
+
var storeFactory = /* @__PURE__ */ new Map();
|
1612
2145
|
function buildURL(optURL, loader) {
|
1613
2146
|
const storeOpts = loader.ebOpts.store;
|
1614
2147
|
const obuItem = Array.from(storeFactory.values()).find((items) => items.overrideBaseURL);
|
1615
2148
|
let obuUrl;
|
1616
2149
|
if (obuItem && obuItem.overrideBaseURL) {
|
1617
|
-
obuUrl =
|
2150
|
+
obuUrl = URI5.from(obuItem.overrideBaseURL);
|
1618
2151
|
}
|
1619
|
-
|
2152
|
+
const ret = ensureIsIndex(
|
2153
|
+
URI5.from(optURL || obuUrl || dataDir(loader.sthis, loader.name, storeOpts.stores?.base)),
|
2154
|
+
storeOpts.isIndex
|
2155
|
+
);
|
2156
|
+
return ret;
|
2157
|
+
}
|
2158
|
+
var onceGateway = new KeyedResolvOnce2();
|
2159
|
+
async function getGatewayFromURL(url, sthis) {
|
2160
|
+
return onceGateway.get(url.toString()).once(async () => {
|
2161
|
+
const item = storeFactory.get(url.protocol);
|
2162
|
+
if (item) {
|
2163
|
+
const ret = {
|
2164
|
+
gateway: await item.gateway(sthis),
|
2165
|
+
test: await item.test(sthis)
|
2166
|
+
};
|
2167
|
+
const res = await ret.gateway.start(url);
|
2168
|
+
if (res.isErr()) {
|
2169
|
+
sthis.logger.Error().Result("start", res).Msg("start failed");
|
2170
|
+
return void 0;
|
2171
|
+
}
|
2172
|
+
return ret;
|
2173
|
+
}
|
2174
|
+
sthis.logger.Warn().Url(url).Msg("unsupported protocol");
|
2175
|
+
return void 0;
|
2176
|
+
});
|
1620
2177
|
}
|
1621
2178
|
function registerStoreProtocol(item) {
|
1622
2179
|
let protocol = item.protocol;
|
@@ -1625,8 +2182,7 @@ function registerStoreProtocol(item) {
|
|
1625
2182
|
}
|
1626
2183
|
if (storeFactory.has(protocol)) {
|
1627
2184
|
if (!item.overrideBaseURL && storeFactory.get(protocol) !== item) {
|
1628
|
-
|
1629
|
-
logger.Warn().Msg(`protocol ${protocol} already registered`);
|
2185
|
+
throw new Error(`we need a logger here`);
|
1630
2186
|
return () => {
|
1631
2187
|
};
|
1632
2188
|
}
|
@@ -1641,106 +2197,92 @@ function registerStoreProtocol(item) {
|
|
1641
2197
|
storeFactory.delete(protocol);
|
1642
2198
|
};
|
1643
2199
|
}
|
1644
|
-
|
1645
|
-
const item = storeFactory.get(url.protocol);
|
1646
|
-
if (!item) {
|
1647
|
-
throw logger.Error().Url(url).Str("protocol", url.protocol).Any("keys", Array(storeFactory.keys())).Msg(`unsupported protocol`).AsError();
|
1648
|
-
}
|
1649
|
-
logger.Debug().Str("protocol", url.protocol).Msg("run");
|
1650
|
-
return run(item);
|
1651
|
-
}
|
1652
|
-
var onceLoadDataGateway = new KeyedResolvOnce();
|
1653
|
-
function loadDataGateway(url, logger) {
|
1654
|
-
return onceLoadDataGateway.get(url.protocol).once(async () => {
|
1655
|
-
return runStoreFactory(url, logger, async (item) => item.data(logger));
|
1656
|
-
});
|
1657
|
-
}
|
1658
|
-
var onceDataStoreFactory = new KeyedResolvOnce();
|
2200
|
+
var onceDataStoreFactory = new KeyedResolvOnce2();
|
1659
2201
|
async function dataStoreFactory(loader) {
|
1660
|
-
const url = buildURL(loader.ebOpts.store.stores?.data, loader);
|
1661
|
-
|
1662
|
-
const logger = ensureLogger(loader.logger, "dataStoreFactory", { url: url.toString() });
|
1663
|
-
url.searchParams.set("store", "data");
|
2202
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI();
|
2203
|
+
const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() });
|
1664
2204
|
return onceDataStoreFactory.get(url.toString()).once(async () => {
|
1665
|
-
const gateway = await
|
1666
|
-
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
2205
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2206
|
+
if (!gateway) {
|
2207
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2208
|
+
}
|
2209
|
+
const store = new DataStoreImpl(sthis, loader.name, url, {
|
2210
|
+
gateway: gateway.gateway,
|
2211
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2212
|
+
...loader.ebOpts.keyBag
|
2213
|
+
})
|
2214
|
+
});
|
1672
2215
|
return store;
|
1673
2216
|
});
|
1674
2217
|
}
|
1675
|
-
var
|
1676
|
-
function loadMetaGateway(url, logger) {
|
1677
|
-
return onceLoadMetaGateway.get(url.protocol).once(async () => {
|
1678
|
-
return runStoreFactory(url, logger, async (item) => item.meta(logger));
|
1679
|
-
});
|
1680
|
-
}
|
1681
|
-
var onceMetaStoreFactory = new KeyedResolvOnce();
|
2218
|
+
var onceMetaStoreFactory = new KeyedResolvOnce2();
|
1682
2219
|
async function metaStoreFactory(loader) {
|
1683
|
-
const url = buildURL(loader.ebOpts.store.stores?.meta, loader);
|
1684
|
-
|
1685
|
-
const logger = ensureLogger(loader.logger, "metaStoreFactory", { url: () => url.toString() });
|
1686
|
-
url.searchParams.set("store", "meta");
|
2220
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI();
|
2221
|
+
const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() });
|
1687
2222
|
return onceMetaStoreFactory.get(url.toString()).once(async () => {
|
1688
|
-
logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
1689
|
-
const gateway = await
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
2223
|
+
sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch");
|
2224
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2225
|
+
if (!gateway) {
|
2226
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2227
|
+
}
|
2228
|
+
const store = new MetaStoreImpl(loader.sthis, loader.name, url, {
|
2229
|
+
gateway: gateway.gateway,
|
2230
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2231
|
+
...loader.ebOpts.keyBag
|
2232
|
+
})
|
2233
|
+
});
|
1695
2234
|
return store;
|
1696
2235
|
});
|
1697
2236
|
}
|
1698
|
-
var
|
1699
|
-
function loadWalGateway(url, logger) {
|
1700
|
-
return onceWalGateway.get(url.protocol).once(async () => {
|
1701
|
-
return runStoreFactory(url, logger, async (item) => item.wal(logger));
|
1702
|
-
});
|
1703
|
-
}
|
1704
|
-
var onceRemoteWalFactory = new KeyedResolvOnce();
|
2237
|
+
var onceRemoteWalFactory = new KeyedResolvOnce2();
|
1705
2238
|
async function remoteWalFactory(loader) {
|
1706
|
-
const url = buildURL(loader.ebOpts.store.stores?.
|
1707
|
-
|
1708
|
-
const logger = ensureLogger(loader.logger, "remoteWalFactory", { url: url.toString() });
|
1709
|
-
url.searchParams.set("store", "wal");
|
2239
|
+
const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.wal, loader)).build().setParam("store", "wal").URI();
|
2240
|
+
const sthis = ensureSuperLog(loader.sthis, "remoteWalFactory", { url: url.toString() });
|
1710
2241
|
return onceRemoteWalFactory.get(url.toString()).once(async () => {
|
1711
|
-
const gateway = await
|
1712
|
-
|
1713
|
-
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1717
|
-
|
2242
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2243
|
+
if (!gateway) {
|
2244
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2245
|
+
}
|
2246
|
+
sthis.logger.Debug().Str("prepared", url.toString()).Msg("produced");
|
2247
|
+
const store = new WALStoreImpl(loader, url, {
|
2248
|
+
gateway: gateway.gateway,
|
2249
|
+
keybag: () => getKeyBag(loader.sthis, {
|
2250
|
+
...loader.ebOpts.keyBag
|
2251
|
+
})
|
2252
|
+
});
|
1718
2253
|
return store;
|
1719
2254
|
});
|
1720
2255
|
}
|
1721
|
-
async function testStoreFactory(url,
|
1722
|
-
|
1723
|
-
|
1724
|
-
|
1725
|
-
|
1726
|
-
|
1727
|
-
|
1728
|
-
|
2256
|
+
async function testStoreFactory(url, sthis) {
|
2257
|
+
sthis = ensureSuperLog(sthis, "testStoreFactory");
|
2258
|
+
const gateway = await getGatewayFromURL(url, sthis);
|
2259
|
+
if (!gateway) {
|
2260
|
+
throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError();
|
2261
|
+
}
|
2262
|
+
return gateway.test;
|
2263
|
+
}
|
2264
|
+
async function ensureStart(store, logger) {
|
2265
|
+
const ret = await store.start();
|
2266
|
+
if (ret.isErr()) {
|
2267
|
+
throw logger.Error().Result("start", ret).Msg("start failed").AsError();
|
2268
|
+
}
|
2269
|
+
logger.Debug().Url(ret.Ok(), "prepared").Msg("produced");
|
2270
|
+
return store;
|
1729
2271
|
}
|
1730
|
-
function toStoreRuntime(opts,
|
1731
|
-
const logger = ensureLogger(
|
2272
|
+
function toStoreRuntime(opts, sthis) {
|
2273
|
+
const logger = ensureLogger(sthis, "toStoreRuntime", {});
|
1732
2274
|
return {
|
1733
|
-
makeMetaStore: (loader) => {
|
2275
|
+
makeMetaStore: async (loader) => {
|
1734
2276
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeMetaStore).Msg("makeMetaStore");
|
1735
|
-
return (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader);
|
2277
|
+
return ensureStart(await (loader.ebOpts.store.makeMetaStore || metaStoreFactory)(loader), logger);
|
1736
2278
|
},
|
1737
|
-
makeDataStore: (loader) => {
|
2279
|
+
makeDataStore: async (loader) => {
|
1738
2280
|
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeDataStore).Msg("makeDataStore");
|
1739
|
-
return (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader);
|
2281
|
+
return ensureStart(await (loader.ebOpts.store.makeDataStore || dataStoreFactory)(loader), logger);
|
1740
2282
|
},
|
1741
|
-
|
1742
|
-
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.
|
1743
|
-
return (loader.ebOpts.store.
|
2283
|
+
makeWALStore: async (loader) => {
|
2284
|
+
logger.Debug().Str("fromOpts", "" + !!loader.ebOpts.store.makeWALStore).Msg("makeRemoteWAL");
|
2285
|
+
return ensureStart(await (loader.ebOpts.store.makeWALStore || remoteWalFactory)(loader), logger);
|
1744
2286
|
},
|
1745
2287
|
encodeFile: opts.encodeFile || encodeFile,
|
1746
2288
|
decodeFile: opts.decodeFile || decodeFile
|
@@ -1748,43 +2290,125 @@ function toStoreRuntime(opts, ilogger) {
|
|
1748
2290
|
}
|
1749
2291
|
registerStoreProtocol({
|
1750
2292
|
protocol: "file:",
|
1751
|
-
|
1752
|
-
const {
|
1753
|
-
return new
|
2293
|
+
gateway: async (sthis) => {
|
2294
|
+
const { FileGateway } = await import("./gateway-VWWKLHUI.js");
|
2295
|
+
return new FileGateway(sthis);
|
1754
2296
|
},
|
1755
|
-
|
1756
|
-
const {
|
1757
|
-
return new
|
1758
|
-
},
|
1759
|
-
wal: async (logger) => {
|
1760
|
-
const { FileWALGateway } = await import("./store-file-VJ6BI4II.js");
|
1761
|
-
return new FileWALGateway(logger);
|
1762
|
-
},
|
1763
|
-
test: async (logger) => {
|
1764
|
-
const { FileTestStore } = await import("./store-file-VJ6BI4II.js");
|
1765
|
-
return new FileTestStore(logger);
|
2297
|
+
test: async (sthis) => {
|
2298
|
+
const { FileTestStore } = await import("./gateway-VWWKLHUI.js");
|
2299
|
+
return new FileTestStore(sthis);
|
1766
2300
|
}
|
1767
2301
|
});
|
1768
2302
|
registerStoreProtocol({
|
1769
2303
|
protocol: "indexdb:",
|
1770
|
-
|
1771
|
-
const {
|
1772
|
-
return new
|
1773
|
-
},
|
1774
|
-
meta: async (logger) => {
|
1775
|
-
const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
|
1776
|
-
return new IndexDBMetaGateway(logger);
|
1777
|
-
},
|
1778
|
-
wal: async (logger) => {
|
1779
|
-
const { IndexDBMetaGateway } = await import("./store-indexdb-WLRSICCB.js");
|
1780
|
-
return new IndexDBMetaGateway(logger);
|
2304
|
+
gateway: async (sthis) => {
|
2305
|
+
const { IndexDBGateway } = await import("./gateway-7OM6OSYK.js");
|
2306
|
+
return new IndexDBGateway(sthis);
|
1781
2307
|
},
|
1782
|
-
test: async (
|
1783
|
-
const { IndexDBTestStore } = await import("./
|
1784
|
-
return new IndexDBTestStore(
|
2308
|
+
test: async (sthis) => {
|
2309
|
+
const { IndexDBTestStore } = await import("./gateway-7OM6OSYK.js");
|
2310
|
+
return new IndexDBTestStore(sthis);
|
1785
2311
|
}
|
1786
2312
|
});
|
1787
2313
|
|
2314
|
+
// src/blockstore/store-remote.ts
|
2315
|
+
async function RemoteDataStore(sthis, name, url, opts) {
|
2316
|
+
const ds = new DataStoreImpl(sthis, name, url, opts);
|
2317
|
+
await ds.start();
|
2318
|
+
return ds;
|
2319
|
+
}
|
2320
|
+
async function RemoteMetaStore(sthis, name, url, opts) {
|
2321
|
+
const ms = new MetaStoreImpl(sthis, name, url, opts, true);
|
2322
|
+
await ms.start();
|
2323
|
+
return ms;
|
2324
|
+
}
|
2325
|
+
|
2326
|
+
// src/blockstore/connection-base.ts
|
2327
|
+
var ConnectionBase = class {
|
2328
|
+
constructor(url, logger) {
|
2329
|
+
this.loaded = Promise.resolve();
|
2330
|
+
this.logger = logger;
|
2331
|
+
this.url = url;
|
2332
|
+
}
|
2333
|
+
async refresh() {
|
2334
|
+
await throwFalsy(throwFalsy(this.loader).remoteMetaStore).load();
|
2335
|
+
await (await throwFalsy(this.loader).WALStore()).process();
|
2336
|
+
}
|
2337
|
+
async connect_X({ loader }) {
|
2338
|
+
if (!loader) throw this.logger.Error().Msg("loader is required").AsError();
|
2339
|
+
await this.connectMeta_X({ loader });
|
2340
|
+
await this.connectStorage_X({ loader });
|
2341
|
+
}
|
2342
|
+
async connectMeta_X({ loader }) {
|
2343
|
+
if (!loader) throw this.logger.Error().Msg("connectMeta_X: loader is required").AsError();
|
2344
|
+
this.loader = loader;
|
2345
|
+
await this.onConnect();
|
2346
|
+
const metaUrl = this.url.build().defParam("store", "meta").URI();
|
2347
|
+
const gateway = await getGatewayFromURL(metaUrl, this.loader.sthis);
|
2348
|
+
if (!gateway) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: gateway is required").AsError();
|
2349
|
+
const dbName = metaUrl.getParam("name");
|
2350
|
+
if (!dbName) throw this.logger.Error().Url(metaUrl).Msg("connectMeta_X: name is required").AsError();
|
2351
|
+
const remote = await RemoteMetaStore(loader.sthis, dbName, metaUrl, {
|
2352
|
+
gateway: gateway.gateway,
|
2353
|
+
keybag: () => getKeyBag(loader.sthis, loader.ebOpts.keyBag),
|
2354
|
+
loader
|
2355
|
+
});
|
2356
|
+
this.loader.remoteMetaStore = remote;
|
2357
|
+
this.loaded = this.loader.ready().then(async () => {
|
2358
|
+
remote.load().then(async () => {
|
2359
|
+
(await throwFalsy(this.loader).WALStore()).process();
|
2360
|
+
});
|
2361
|
+
});
|
2362
|
+
}
|
2363
|
+
async connectStorage_X({ loader }) {
|
2364
|
+
if (!loader) throw this.logger.Error().Msg("connectStorage_X: loader is required").AsError();
|
2365
|
+
this.loader = loader;
|
2366
|
+
const dataUrl = this.url.build().defParam("store", "data").URI();
|
2367
|
+
const gateway = await getGatewayFromURL(dataUrl, this.loader.sthis);
|
2368
|
+
if (!gateway) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: gateway is required").AsError();
|
2369
|
+
const name = dataUrl.getParam("name");
|
2370
|
+
if (!name) throw this.logger.Error().Url(dataUrl).Msg("connectStorage_X: name is required").AsError;
|
2371
|
+
loader.remoteCarStore = await RemoteDataStore(loader.sthis, name, this.url, {
|
2372
|
+
gateway: gateway.gateway,
|
2373
|
+
keybag: () => getKeyBag(loader.sthis, this.loader?.ebOpts.keyBag)
|
2374
|
+
});
|
2375
|
+
loader.remoteFileStore = loader.remoteCarStore;
|
2376
|
+
}
|
2377
|
+
// move this stuff to connect
|
2378
|
+
// async getDashboardURL(compact = true) {
|
2379
|
+
// const baseUrl = 'https://dashboard.fireproof.storage/'
|
2380
|
+
// if (!this.loader?.remoteCarStore) return new URL('/howto', baseUrl)
|
2381
|
+
// // if (compact) {
|
2382
|
+
// // await this.compact()
|
2383
|
+
// // }
|
2384
|
+
// const currents = await this.loader?.metaStore?.load()
|
2385
|
+
// if (!currents) throw new Error("Can't sync empty database: save data first")
|
2386
|
+
// if (currents.length > 1)
|
2387
|
+
// throw new Error("Can't sync database with split heads: make an update first")
|
2388
|
+
// const current = currents[0]
|
2389
|
+
// const params = {
|
2390
|
+
// car: current.car.toString()
|
2391
|
+
// }
|
2392
|
+
// if (current.key) {
|
2393
|
+
// // @ts-ignore
|
2394
|
+
// params.key = current.key.toString()
|
2395
|
+
// }
|
2396
|
+
// // @ts-ignore
|
2397
|
+
// if (this.name) {
|
2398
|
+
// // @ts-ignore
|
2399
|
+
// params.name = this.name
|
2400
|
+
// }
|
2401
|
+
// const url = new URL('/import#' + new URLSearchParams(params).toString(), baseUrl)
|
2402
|
+
// console.log('Import to dashboard: ' + url.toString())
|
2403
|
+
// return url
|
2404
|
+
// }
|
2405
|
+
// openDashboard() {
|
2406
|
+
// void this.getDashboardURL().then(url => {
|
2407
|
+
// if (url) window.open(url.toString(), '_blank')
|
2408
|
+
// })
|
2409
|
+
// }
|
2410
|
+
};
|
2411
|
+
|
1788
2412
|
// src/crdt-helpers.ts
|
1789
2413
|
function time(tag) {
|
1790
2414
|
}
|
@@ -1833,7 +2457,7 @@ async function writeDocContent(store, blocks, update, logger) {
|
|
1833
2457
|
await processFiles(store, blocks, update.value, logger);
|
1834
2458
|
value = { doc: update.value };
|
1835
2459
|
}
|
1836
|
-
const block = await
|
2460
|
+
const block = await encode({ value, hasher: hasher5, codec });
|
1837
2461
|
blocks.putSync(block.cid, block.bytes);
|
1838
2462
|
return block.cid;
|
1839
2463
|
}
|
@@ -1842,10 +2466,16 @@ async function processFiles(store, blocks, doc, logger) {
|
|
1842
2466
|
await processFileset(logger, store, blocks, doc._files);
|
1843
2467
|
}
|
1844
2468
|
if (doc._publicFiles) {
|
1845
|
-
await processFileset(
|
2469
|
+
await processFileset(
|
2470
|
+
logger,
|
2471
|
+
store,
|
2472
|
+
blocks,
|
2473
|
+
doc._publicFiles
|
2474
|
+
/*, true*/
|
2475
|
+
);
|
1846
2476
|
}
|
1847
2477
|
}
|
1848
|
-
async function processFileset(logger, store, blocks, files
|
2478
|
+
async function processFileset(logger, store, blocks, files) {
|
1849
2479
|
const dbBlockstore = blocks.parent;
|
1850
2480
|
if (!dbBlockstore.loader) throw logger.Error().Msg("Missing loader, database name is required").AsError();
|
1851
2481
|
const t = new CarTransaction(dbBlockstore);
|
@@ -1867,9 +2497,10 @@ async function processFileset(logger, store, blocks, files, publicFiles = false)
|
|
1867
2497
|
}
|
1868
2498
|
}
|
1869
2499
|
if (didPut.length) {
|
1870
|
-
const car = await dbBlockstore.loader.commitFiles(
|
1871
|
-
|
1872
|
-
|
2500
|
+
const car = await dbBlockstore.loader.commitFiles(
|
2501
|
+
t,
|
2502
|
+
{ files }
|
2503
|
+
);
|
1873
2504
|
if (car) {
|
1874
2505
|
for (const name of didPut) {
|
1875
2506
|
files[name] = { car, ...files[name] };
|
@@ -1903,7 +2534,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
1903
2534
|
fileMeta.file = async () => await blocks.ebOpts.storeRuntime.decodeFile(
|
1904
2535
|
{
|
1905
2536
|
get: async (cid) => {
|
1906
|
-
return await blocks.getFile(throwFalsy(fileMeta.car), cid
|
2537
|
+
return await blocks.getFile(throwFalsy(fileMeta.car), cid);
|
1907
2538
|
}
|
1908
2539
|
},
|
1909
2540
|
fileMeta.cid,
|
@@ -1917,7 +2548,7 @@ function readFileset(blocks, files, isPublic = false) {
|
|
1917
2548
|
async function getValueFromLink(blocks, link, logger) {
|
1918
2549
|
const block = await blocks.get(link);
|
1919
2550
|
if (!block) throw logger.Error().Str("link", link.toString()).Msg(`Missing linked block`).AsError();
|
1920
|
-
const { value } = await
|
2551
|
+
const { value } = await decode({ bytes: block.bytes, hasher: hasher5, codec });
|
1921
2552
|
const cvalue = {
|
1922
2553
|
...value,
|
1923
2554
|
cid: link
|
@@ -1926,17 +2557,21 @@ async function getValueFromLink(blocks, link, logger) {
|
|
1926
2557
|
return cvalue;
|
1927
2558
|
}
|
1928
2559
|
var DirtyEventFetcher = class extends EventFetcher {
|
2560
|
+
constructor(logger, blocks) {
|
2561
|
+
super(blocks);
|
2562
|
+
this.logger = logger;
|
2563
|
+
}
|
1929
2564
|
async get(link) {
|
1930
2565
|
try {
|
1931
2566
|
return super.get(link);
|
1932
2567
|
} catch (e) {
|
1933
|
-
|
2568
|
+
this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
|
1934
2569
|
return { value: void 0 };
|
1935
2570
|
}
|
1936
2571
|
}
|
1937
2572
|
};
|
1938
2573
|
async function clockChangesSince(blocks, head, since, opts, logger) {
|
1939
|
-
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(blocks) : new EventFetcher(blocks);
|
2574
|
+
const eventsFetcher = opts.dirty ? new DirtyEventFetcher(logger, blocks) : new EventFetcher(blocks);
|
1940
2575
|
const keys = /* @__PURE__ */ new Set();
|
1941
2576
|
const updates = await gatherUpdates(
|
1942
2577
|
blocks,
|
@@ -2032,20 +2667,19 @@ async function doCompact(blockLog, head, logger) {
|
|
2032
2667
|
isCompacting = false;
|
2033
2668
|
}
|
2034
2669
|
async function getBlock(blocks, cidString) {
|
2035
|
-
const block = await blocks.get(
|
2670
|
+
const block = await blocks.get(parse3(cidString));
|
2036
2671
|
if (!block) throw new Error(`Missing block ${cidString}`);
|
2037
|
-
const { cid, value } = await
|
2038
|
-
return new
|
2672
|
+
const { cid, value } = await decode({ bytes: block.bytes, codec, hasher: hasher5 });
|
2673
|
+
return new Block({ cid, value, bytes: block.bytes });
|
2039
2674
|
}
|
2040
2675
|
|
2041
2676
|
// src/indexer-helpers.ts
|
2042
|
-
import {
|
2043
|
-
import
|
2044
|
-
import * as codec3 from "@ipld/dag-cbor";
|
2677
|
+
import { sha256 as hasher6 } from "multiformats/hashes/sha2";
|
2678
|
+
import * as codec2 from "@ipld/dag-cbor";
|
2045
2679
|
import charwise from "charwise";
|
2046
2680
|
import * as DbIndex from "prolly-trees/db-index";
|
2047
|
-
import { bf
|
2048
|
-
import { nocache as
|
2681
|
+
import { bf, simpleCompare } from "prolly-trees/utils";
|
2682
|
+
import { nocache as cache } from "prolly-trees/cache";
|
2049
2683
|
var IndexTree = class {
|
2050
2684
|
};
|
2051
2685
|
function refCompare(aRef, bRef) {
|
@@ -2061,8 +2695,8 @@ function compare(a, b) {
|
|
2061
2695
|
if (comp !== 0) return comp;
|
2062
2696
|
return refCompare(aRef, bRef);
|
2063
2697
|
}
|
2064
|
-
var byKeyOpts = { cache
|
2065
|
-
var byIdOpts = { cache
|
2698
|
+
var byKeyOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare };
|
2699
|
+
var byIdOpts = { cache, chunker: bf(30), codec: codec2, hasher: hasher6, compare: simpleCompare };
|
2066
2700
|
function indexEntriesForChanges(changes, mapFn) {
|
2067
2701
|
const indexEntries = [];
|
2068
2702
|
changes.forEach(({ id: key, value, del }) => {
|
@@ -2090,7 +2724,7 @@ function makeProllyGetBlock(blocks) {
|
|
2090
2724
|
const block = await blocks.get(address);
|
2091
2725
|
if (!block) throw new Error(`Missing block ${address.toString()}`);
|
2092
2726
|
const { cid, bytes } = block;
|
2093
|
-
return
|
2727
|
+
return create({ cid, bytes, hasher: hasher6, codec: codec2 });
|
2094
2728
|
};
|
2095
2729
|
}
|
2096
2730
|
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
|
@@ -2161,25 +2795,25 @@ function encodeKey(key) {
|
|
2161
2795
|
}
|
2162
2796
|
|
2163
2797
|
// src/indexer.ts
|
2164
|
-
function index({ _crdt }, name, mapFn, meta) {
|
2798
|
+
function index(sthis, { _crdt }, name, mapFn, meta) {
|
2165
2799
|
if (mapFn && meta) throw _crdt.logger.Error().Msg("cannot provide both mapFn and meta").AsError();
|
2166
2800
|
if (mapFn && mapFn.constructor.name !== "Function") throw _crdt.logger.Error().Msg("mapFn must be a function").AsError();
|
2167
2801
|
if (_crdt.indexers.has(name)) {
|
2168
2802
|
const idx = _crdt.indexers.get(name);
|
2169
2803
|
idx.applyMapFn(name, mapFn, meta);
|
2170
2804
|
} else {
|
2171
|
-
const idx = new Index(_crdt, name, mapFn, meta);
|
2805
|
+
const idx = new Index(sthis, _crdt, name, mapFn, meta);
|
2172
2806
|
_crdt.indexers.set(name, idx);
|
2173
2807
|
}
|
2174
2808
|
return _crdt.indexers.get(name);
|
2175
2809
|
}
|
2176
2810
|
var Index = class {
|
2177
|
-
constructor(crdt, name, mapFn, meta) {
|
2811
|
+
constructor(sthis, crdt, name, mapFn, meta) {
|
2178
2812
|
this.mapFnString = "";
|
2179
2813
|
this.byKey = new IndexTree();
|
2180
2814
|
this.byId = new IndexTree();
|
2181
2815
|
this.includeDocsDefault = false;
|
2182
|
-
this.logger = ensureLogger(
|
2816
|
+
this.logger = ensureLogger(sthis, "Index");
|
2183
2817
|
this.blockstore = crdt.indexBlockstore;
|
2184
2818
|
this.crdt = crdt;
|
2185
2819
|
this.applyMapFn(name, mapFn, meta);
|
@@ -2234,7 +2868,7 @@ var Index = class {
|
|
2234
2868
|
}
|
2235
2869
|
if (this.mapFnString) {
|
2236
2870
|
if (this.mapFnString !== mapFn.toString()) {
|
2237
|
-
throw this.logger.Error().Msg("cannot apply different mapFn app").AsError();
|
2871
|
+
throw this.logger.Error().Str("mapFnString", this.mapFnString).Str("mapFn", mapFn.toString()).Msg("cannot apply different mapFn app").AsError();
|
2238
2872
|
}
|
2239
2873
|
} else {
|
2240
2874
|
this.mapFnString = mapFn.toString();
|
@@ -2369,7 +3003,7 @@ var Index = class {
|
|
2369
3003
|
// src/crdt-clock.ts
|
2370
3004
|
import { advance } from "@web3-storage/pail/clock";
|
2371
3005
|
import { root as root2 } from "@web3-storage/pail/crdt";
|
2372
|
-
import { ResolveOnce as
|
3006
|
+
import { ResolveOnce as ResolveOnce4 } from "@adviser/cement";
|
2373
3007
|
|
2374
3008
|
// src/apply-head-queue.ts
|
2375
3009
|
function applyHeadQueue(worker, logger) {
|
@@ -2426,9 +3060,9 @@ var CRDTClock = class {
|
|
2426
3060
|
this.zoomers = /* @__PURE__ */ new Set();
|
2427
3061
|
this.watchers = /* @__PURE__ */ new Set();
|
2428
3062
|
this.emptyWatchers = /* @__PURE__ */ new Set();
|
2429
|
-
this._ready = new
|
3063
|
+
this._ready = new ResolveOnce4();
|
2430
3064
|
this.blockstore = blockstore;
|
2431
|
-
this.logger = ensureLogger(blockstore.
|
3065
|
+
this.logger = ensureLogger(blockstore.sthis, "CRDTClock");
|
2432
3066
|
this.applyHeadQueue = applyHeadQueue(this.int_applyHead.bind(this), this.logger);
|
2433
3067
|
}
|
2434
3068
|
async ready() {
|
@@ -2501,7 +3135,7 @@ var CRDTClock = class {
|
|
2501
3135
|
}
|
2502
3136
|
return { head: advancedHead };
|
2503
3137
|
},
|
2504
|
-
{ noLoader }
|
3138
|
+
{ noLoader, add: false }
|
2505
3139
|
);
|
2506
3140
|
this.setHead(meta.head);
|
2507
3141
|
}
|
@@ -2535,13 +3169,14 @@ async function advanceBlocks(logger, newHead, tblocks, head) {
|
|
2535
3169
|
|
2536
3170
|
// src/crdt.ts
|
2537
3171
|
var CRDT = class {
|
2538
|
-
constructor(name, opts = {}) {
|
2539
|
-
this.onceReady = new ResolveOnce4();
|
3172
|
+
constructor(sthis, name, opts = {}) {
|
2540
3173
|
this.indexers = /* @__PURE__ */ new Map();
|
3174
|
+
this.onceReady = new ResolveOnce5();
|
3175
|
+
this.sthis = sthis;
|
2541
3176
|
this.name = name;
|
2542
|
-
this.logger = ensureLogger(
|
3177
|
+
this.logger = ensureLogger(sthis, "CRDT");
|
2543
3178
|
this.opts = opts;
|
2544
|
-
this.blockstore = blockstoreFactory({
|
3179
|
+
this.blockstore = blockstoreFactory(sthis, {
|
2545
3180
|
name,
|
2546
3181
|
applyMeta: async (meta) => {
|
2547
3182
|
const crdtMeta = meta;
|
@@ -2553,22 +3188,20 @@ var CRDT = class {
|
|
2553
3188
|
return { head: this.clock.head };
|
2554
3189
|
},
|
2555
3190
|
autoCompact: this.opts.autoCompact || 100,
|
2556
|
-
crypto: this.opts.crypto,
|
2557
3191
|
store: { ...this.opts.store, isIndex: void 0 },
|
2558
3192
|
public: this.opts.public,
|
2559
3193
|
meta: this.opts.meta,
|
2560
3194
|
threshold: this.opts.threshold
|
2561
3195
|
});
|
2562
|
-
this.indexBlockstore = blockstoreFactory({
|
3196
|
+
this.indexBlockstore = blockstoreFactory(sthis, {
|
2563
3197
|
name,
|
2564
3198
|
applyMeta: async (meta) => {
|
2565
3199
|
const idxCarMeta = meta;
|
2566
3200
|
if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
|
2567
3201
|
for (const [name2, idx] of Object.entries(idxCarMeta.indexes)) {
|
2568
|
-
index({ _crdt: this }, name2, void 0, idx);
|
3202
|
+
index(this.sthis, { _crdt: this }, name2, void 0, idx);
|
2569
3203
|
}
|
2570
3204
|
},
|
2571
|
-
crypto: this.opts.crypto,
|
2572
3205
|
store: { ...this.opts.store, isIndex: this.opts.store?.isIndex || "idx" },
|
2573
3206
|
public: this.opts.public
|
2574
3207
|
});
|
@@ -2579,17 +3212,6 @@ var CRDT = class {
|
|
2579
3212
|
}
|
2580
3213
|
});
|
2581
3214
|
}
|
2582
|
-
async ready() {
|
2583
|
-
return this.onceReady.once(async () => {
|
2584
|
-
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
2585
|
-
});
|
2586
|
-
}
|
2587
|
-
async close() {
|
2588
|
-
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
2589
|
-
}
|
2590
|
-
async destroy() {
|
2591
|
-
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
2592
|
-
}
|
2593
3215
|
async bulk(updates) {
|
2594
3216
|
await this.ready();
|
2595
3217
|
const prevHead = [...this.clock.head];
|
@@ -2610,6 +3232,22 @@ var CRDT = class {
|
|
2610
3232
|
await this.clock.applyHead(done.meta.head, prevHead, updates);
|
2611
3233
|
return done.meta;
|
2612
3234
|
}
|
3235
|
+
async ready() {
|
3236
|
+
return this.onceReady.once(async () => {
|
3237
|
+
try {
|
3238
|
+
await Promise.all([this.blockstore.ready(), this.indexBlockstore.ready(), this.clock.ready()]);
|
3239
|
+
} catch (e) {
|
3240
|
+
const ee = e;
|
3241
|
+
throw this.logger.Error().Err(e).Msg(`CRDT is not ready: ${ee.stack}`).AsError();
|
3242
|
+
}
|
3243
|
+
});
|
3244
|
+
}
|
3245
|
+
async close() {
|
3246
|
+
await Promise.all([this.blockstore.close(), this.indexBlockstore.close(), this.clock.close()]);
|
3247
|
+
}
|
3248
|
+
async destroy() {
|
3249
|
+
await Promise.all([this.blockstore.destroy(), this.indexBlockstore.destroy()]);
|
3250
|
+
}
|
2613
3251
|
// if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
|
2614
3252
|
async allDocs() {
|
2615
3253
|
await this.ready();
|
@@ -2654,11 +3292,12 @@ var Database = class {
|
|
2654
3292
|
this._listening = false;
|
2655
3293
|
this._listeners = /* @__PURE__ */ new Set();
|
2656
3294
|
this._noupdate_listeners = /* @__PURE__ */ new Set();
|
2657
|
-
this._ready = new
|
3295
|
+
this._ready = new ResolveOnce6();
|
2658
3296
|
this.name = name;
|
2659
3297
|
this.opts = opts || this.opts;
|
2660
|
-
this.
|
2661
|
-
this.
|
3298
|
+
this.sthis = ensureSuperThis(this.opts);
|
3299
|
+
this.logger = ensureLogger(this.sthis, "Database");
|
3300
|
+
this._crdt = new CRDT(this.sthis, name, this.opts);
|
2662
3301
|
this.blockstore = this._crdt.blockstore;
|
2663
3302
|
this._writeQueue = writeQueue(async (updates) => {
|
2664
3303
|
return await this._crdt.bulk(updates);
|
@@ -2682,7 +3321,7 @@ var Database = class {
|
|
2682
3321
|
}
|
2683
3322
|
async ready() {
|
2684
3323
|
return this._ready.once(async () => {
|
2685
|
-
await
|
3324
|
+
await this.sthis.start();
|
2686
3325
|
await this._crdt.ready();
|
2687
3326
|
await this.blockstore.ready();
|
2688
3327
|
});
|
@@ -2702,7 +3341,7 @@ var Database = class {
|
|
2702
3341
|
await this.ready();
|
2703
3342
|
this.logger.Debug().Str("id", doc._id).Msg("put");
|
2704
3343
|
const { _id, ...value } = doc;
|
2705
|
-
const docId = _id ||
|
3344
|
+
const docId = _id || this.sthis.nextId().str;
|
2706
3345
|
const result = await this._writeQueue.push({
|
2707
3346
|
id: docId,
|
2708
3347
|
value: {
|
@@ -2767,7 +3406,7 @@ var Database = class {
|
|
2767
3406
|
await this.ready();
|
2768
3407
|
this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
|
2769
3408
|
const _crdt = this._crdt;
|
2770
|
-
const idx = typeof field === "string" ? index({ _crdt }, field) : index({ _crdt }, makeName(field.toString()), field);
|
3409
|
+
const idx = typeof field === "string" ? index(this.sthis, { _crdt }, field) : index(this.sthis, { _crdt }, makeName(field.toString()), field);
|
2771
3410
|
return await idx.query(opts);
|
2772
3411
|
}
|
2773
3412
|
async compact() {
|
@@ -2804,12 +3443,7 @@ function fireproof(name, opts) {
|
|
2804
3443
|
const key = JSON.stringify(
|
2805
3444
|
toSortedArray({
|
2806
3445
|
name,
|
2807
|
-
stores: toSortedArray(opts?.store?.stores)
|
2808
|
-
makeMetaStore: !!opts?.store?.makeMetaStore,
|
2809
|
-
makeDataStore: !!opts?.store?.makeDataStore,
|
2810
|
-
makeRemoteWAL: !!opts?.store?.makeRemoteWAL,
|
2811
|
-
encodeFile: !!opts?.store?.encodeFile,
|
2812
|
-
decodeFile: !!opts?.store?.decodeFile
|
3446
|
+
stores: toSortedArray(opts?.store?.stores)
|
2813
3447
|
})
|
2814
3448
|
);
|
2815
3449
|
let db = Database.databases.get(key);
|
@@ -2824,7 +3458,10 @@ function makeName(fnString) {
|
|
2824
3458
|
let found = null;
|
2825
3459
|
const matches = Array.from(fnString.matchAll(regex), (match) => match[1].trim());
|
2826
3460
|
if (matches.length === 0) {
|
2827
|
-
found = /=>\s*(
|
3461
|
+
found = /=>\s*{?\s*([^{}]+)\s*}?/.exec(fnString);
|
3462
|
+
if (found && found[1].includes("return")) {
|
3463
|
+
found = null;
|
3464
|
+
}
|
2828
3465
|
}
|
2829
3466
|
if (!found) {
|
2830
3467
|
return fnString;
|
@@ -2835,18 +3472,22 @@ function makeName(fnString) {
|
|
2835
3472
|
|
2836
3473
|
// src/version.ts
|
2837
3474
|
var PACKAGE_VERSION = Object.keys({
|
2838
|
-
"0.19.
|
3475
|
+
"0.19.11-dev-dryrun": "xxxx"
|
2839
3476
|
})[0];
|
2840
3477
|
export {
|
2841
3478
|
CRDT,
|
2842
3479
|
Database,
|
2843
3480
|
Index,
|
3481
|
+
NotFoundError,
|
2844
3482
|
PACKAGE_VERSION,
|
2845
3483
|
Result,
|
3484
|
+
UInt8ArrayEqual,
|
2846
3485
|
blockstore_exports as blockstore,
|
2847
3486
|
blockstore_exports as bs,
|
3487
|
+
dataDir,
|
2848
3488
|
ensureLogger,
|
2849
|
-
|
3489
|
+
ensureSuperLog,
|
3490
|
+
ensureSuperThis,
|
2850
3491
|
exceptionWrapper,
|
2851
3492
|
falsyToUndef,
|
2852
3493
|
fireproof,
|
@@ -2855,6 +3496,7 @@ export {
|
|
2855
3496
|
getStore,
|
2856
3497
|
index,
|
2857
3498
|
isFalsy,
|
3499
|
+
isNotFoundError,
|
2858
3500
|
runtime_exports as rt,
|
2859
3501
|
runtime_exports as runtime,
|
2860
3502
|
throwFalsy
|