@fireproof/core 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -22
- package/dist/src/fireproof.d.ts +45 -15
- package/dist/src/fireproof.js +97 -27
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +97 -28
- package/dist/src/fireproof.mjs.map +1 -1
- package/package.json +2 -2
- package/src/database.js +31 -13
- package/src/db-index.js +19 -9
- package/src/fireproof.js +45 -5
- package/src/prolly.js +2 -1
- package/src/valet.js +3 -2
- package/dist/blockstore.js +0 -242
- package/dist/clock.js +0 -355
- package/dist/crypto.js +0 -59
- package/dist/database.js +0 -308
- package/dist/db-index.js +0 -314
- package/dist/fireproof.js +0 -83
- package/dist/hooks/use-fireproof.js +0 -100
- package/dist/listener.js +0 -110
- package/dist/main.js +0 -2
- package/dist/main.js.LICENSE.txt +0 -17
- package/dist/prolly.js +0 -316
- package/dist/sha1.js +0 -74
- package/dist/src/blockstore.js +0 -242
- package/dist/src/clock.js +0 -355
- package/dist/src/crypto.js +0 -59
- package/dist/src/database.js +0 -312
- package/dist/src/db-index.js +0 -314
- package/dist/src/index.d.ts +0 -321
- package/dist/src/index.js +0 -38936
- package/dist/src/index.js.map +0 -1
- package/dist/src/index.mjs +0 -38931
- package/dist/src/index.mjs.map +0 -1
- package/dist/src/listener.js +0 -108
- package/dist/src/prolly.js +0 -319
- package/dist/src/sha1.js +0 -74
- package/dist/src/utils.js +0 -16
- package/dist/src/valet.js +0 -262
- package/dist/test/block.js +0 -57
- package/dist/test/clock.test.js +0 -556
- package/dist/test/db-index.test.js +0 -231
- package/dist/test/fireproof.test.js +0 -444
- package/dist/test/fulltext.test.js +0 -61
- package/dist/test/helpers.js +0 -39
- package/dist/test/hydrator.test.js +0 -142
- package/dist/test/listener.test.js +0 -103
- package/dist/test/prolly.test.js +0 -162
- package/dist/test/proofs.test.js +0 -45
- package/dist/test/reproduce-fixture-bug.test.js +0 -57
- package/dist/test/valet.test.js +0 -56
- package/dist/utils.js +0 -16
- package/dist/valet.js +0 -262
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@fireproof/core",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.5.0",
|
4
4
|
"description": "Realtime database for IPFS",
|
5
5
|
"main": "dist/src/fireproof.js",
|
6
6
|
"module": "dist/src/fireproof.mjs",
|
@@ -11,7 +11,7 @@
|
|
11
11
|
"keygen": "node scripts/keygen.js",
|
12
12
|
"test": "standard && npm run test:unencrypted && npm run test:mocha",
|
13
13
|
"test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
|
14
|
-
"test:mocha": "mocha
|
14
|
+
"test:mocha": "mocha test/*.test.js",
|
15
15
|
"test:watch": "npm run test:mocha -- -w --parallel",
|
16
16
|
"coverage": "c8 -r html -r text npm test",
|
17
17
|
"prepublishOnly": "cp ../../README.md . && npm run build",
|
package/src/database.js
CHANGED
@@ -117,7 +117,7 @@ export class Database {
|
|
117
117
|
// console.log('change rows', this.instanceId, rows)
|
118
118
|
} else {
|
119
119
|
const allResp = await getAll(this.blocks, this.clock)
|
120
|
-
rows = allResp.result.map(({ key, value }) =>
|
120
|
+
rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
|
121
121
|
dataCIDs = allResp.cids
|
122
122
|
// console.log('dbdoc rows', this.instanceId, rows)
|
123
123
|
}
|
@@ -130,7 +130,9 @@ export class Database {
|
|
130
130
|
|
131
131
|
async allDocuments () {
|
132
132
|
const allResp = await getAll(this.blocks, this.clock)
|
133
|
-
const rows = allResp.result
|
133
|
+
const rows = allResp.result
|
134
|
+
.map(({ key, value }) => decodeEvent({ key, value }))
|
135
|
+
.map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
|
134
136
|
return {
|
135
137
|
rows,
|
136
138
|
clock: this.clockToJSON(),
|
@@ -138,6 +140,15 @@ export class Database {
|
|
138
140
|
}
|
139
141
|
}
|
140
142
|
|
143
|
+
async allCIDs () {
|
144
|
+
const allResp = await getAll(this.blocks, this.clock)
|
145
|
+
const cids = await cidsToProof(allResp.cids)
|
146
|
+
const clockCids = await cidsToProof(allResp.clockCIDs)
|
147
|
+
// console.log('allcids', cids, clockCids)
|
148
|
+
// todo we need to put the clock head as the last block in the encrypted car
|
149
|
+
return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
|
150
|
+
}
|
151
|
+
|
141
152
|
/**
|
142
153
|
* Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
|
143
154
|
*
|
@@ -150,7 +161,7 @@ export class Database {
|
|
150
161
|
async runValidation (doc) {
|
151
162
|
if (this.config && this.config.validateChange) {
|
152
163
|
const oldDoc = await this.get(doc._id)
|
153
|
-
.then(
|
164
|
+
.then(doc => doc)
|
154
165
|
.catch(() => ({}))
|
155
166
|
this.config.validateChange(doc, oldDoc, this.authCtx)
|
156
167
|
}
|
@@ -185,12 +196,13 @@ export class Database {
|
|
185
196
|
return doc
|
186
197
|
}
|
187
198
|
/**
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
199
|
+
* @typedef {Object} Document
|
200
|
+
* @property {string} _id - The ID of the document (required)
|
201
|
+
* @property {string} [_proof] - The proof of the document (optional)
|
202
|
+
* @property {string} [_clock] - The clock of the document (optional)
|
203
|
+
* @property {any} [key: string] - Index signature notation to allow any other unknown fields
|
204
|
+
* * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
|
205
|
+
*/
|
194
206
|
|
195
207
|
/**
|
196
208
|
* Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
|
@@ -245,17 +257,17 @@ export class Database {
|
|
245
257
|
throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
|
246
258
|
}
|
247
259
|
}
|
260
|
+
const prevClock = [...this.clock]
|
248
261
|
const result = await doTransaction(
|
249
262
|
'putToProllyTree',
|
250
263
|
this.blocks,
|
251
|
-
async
|
264
|
+
async blocks => await put(blocks, this.clock, event)
|
252
265
|
)
|
253
266
|
if (!result) {
|
254
267
|
console.error('failed', event)
|
255
268
|
throw new Error('failed to put at storage layer')
|
256
269
|
}
|
257
|
-
|
258
|
-
this.clock = result.head // do we want to do this as a finally block
|
270
|
+
this.applyClock(prevClock, result.head)
|
259
271
|
await this.notifyListeners([decodedEvent]) // this type is odd
|
260
272
|
return {
|
261
273
|
id: decodedEvent.key,
|
@@ -265,6 +277,12 @@ export class Database {
|
|
265
277
|
// todo should include additions (or split clock)
|
266
278
|
}
|
267
279
|
|
280
|
+
applyClock (prevClock, newClock) {
|
281
|
+
// console.log('applyClock', prevClock, newClock, this.clock)
|
282
|
+
const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
|
283
|
+
this.clock = removedprevCIDs.concat(newClock)
|
284
|
+
}
|
285
|
+
|
268
286
|
// /**
|
269
287
|
// * Advances the clock to the specified event and updates the root CID
|
270
288
|
// * Will be used by replication
|
@@ -323,7 +341,7 @@ export class Database {
|
|
323
341
|
export async function cidsToProof (cids) {
|
324
342
|
if (!cids || !cids.all) return []
|
325
343
|
const all = await cids.all()
|
326
|
-
return [...all].map(
|
344
|
+
return [...all].map(cid => cid.toString())
|
327
345
|
}
|
328
346
|
|
329
347
|
function decodeEvent (event) {
|
package/src/db-index.js
CHANGED
@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
|
|
72
72
|
changes.forEach(({ key, value, del }) => {
|
73
73
|
if (del || !value) return
|
74
74
|
mapFn(makeDoc({ key, value }), (k, v) => {
|
75
|
-
if (typeof
|
75
|
+
if (typeof k === 'undefined') return
|
76
76
|
indexEntries.push({
|
77
77
|
key: [charwise.encode(k), key],
|
78
|
-
value: v
|
78
|
+
value: v || null
|
79
79
|
})
|
80
80
|
})
|
81
81
|
})
|
@@ -93,7 +93,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
|
|
93
93
|
*
|
94
94
|
*/
|
95
95
|
export class DbIndex {
|
96
|
-
constructor (database, mapFn, clock, opts = {}) {
|
96
|
+
constructor (database, name, mapFn, clock = null, opts = {}) {
|
97
97
|
this.database = database
|
98
98
|
if (!database.indexBlocks) {
|
99
99
|
database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
|
@@ -109,7 +109,7 @@ export class DbIndex {
|
|
109
109
|
this.mapFn = mapFn
|
110
110
|
this.mapFnString = mapFn.toString()
|
111
111
|
}
|
112
|
-
this.name =
|
112
|
+
this.name = name || this.makeName()
|
113
113
|
this.indexById = { root: null, cid: null }
|
114
114
|
this.indexByKey = { root: null, cid: null }
|
115
115
|
this.dbHead = null
|
@@ -159,7 +159,7 @@ export class DbIndex {
|
|
159
159
|
|
160
160
|
static fromJSON (database, { code, clock, name }) {
|
161
161
|
// console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
|
162
|
-
return new DbIndex(database, code, clock
|
162
|
+
return new DbIndex(database, name, code, clock)
|
163
163
|
}
|
164
164
|
|
165
165
|
/**
|
@@ -176,7 +176,7 @@ export class DbIndex {
|
|
176
176
|
* @memberof DbIndex
|
177
177
|
* @instance
|
178
178
|
*/
|
179
|
-
async query (query, update = true) {
|
179
|
+
async query (query = {}, update = true) {
|
180
180
|
// const callId = Math.random().toString(36).substring(2, 7)
|
181
181
|
// todo pass a root to query a snapshot
|
182
182
|
// console.time(callId + '.updateIndex')
|
@@ -203,7 +203,12 @@ export class DbIndex {
|
|
203
203
|
async updateIndex (blocks) {
|
204
204
|
// todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
|
205
205
|
// what would it do in a world where all indexes provide a database snapshot to query?
|
206
|
-
if (this.updateIndexPromise)
|
206
|
+
if (this.updateIndexPromise) {
|
207
|
+
return this.updateIndexPromise.then(() => {
|
208
|
+
this.updateIndexPromise = null
|
209
|
+
return this.updateIndex(blocks)
|
210
|
+
})
|
211
|
+
}
|
207
212
|
this.updateIndexPromise = this.innerUpdateIndex(blocks)
|
208
213
|
this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
|
209
214
|
return this.updateIndexPromise
|
@@ -232,7 +237,7 @@ export class DbIndex {
|
|
232
237
|
this.dbHead = result.clock
|
233
238
|
return
|
234
239
|
}
|
235
|
-
await doTransaction('updateIndex', inBlocks, async (blocks) => {
|
240
|
+
const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
|
236
241
|
let oldIndexEntries = []
|
237
242
|
let removeByIdIndexEntries = []
|
238
243
|
await loadIndex(blocks, this.indexById, idIndexOpts)
|
@@ -254,6 +259,7 @@ export class DbIndex {
|
|
254
259
|
this.database.notifyExternal('dbIndex')
|
255
260
|
// console.timeEnd(callTag + '.doTransactionupdateIndex')
|
256
261
|
// console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
|
262
|
+
return didT
|
257
263
|
}
|
258
264
|
}
|
259
265
|
|
@@ -296,7 +302,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
|
|
296
302
|
async function loadIndex (blocks, index, indexOpts) {
|
297
303
|
if (!index.root) {
|
298
304
|
const cid = index.cid
|
299
|
-
if (!cid)
|
305
|
+
if (!cid) {
|
306
|
+
// console.log('no cid', index)
|
307
|
+
// throw new Error('cannot load index')
|
308
|
+
return null
|
309
|
+
}
|
300
310
|
const { getBlock } = makeGetBlock(blocks)
|
301
311
|
index.root = await load({ cid, get: getBlock, ...indexOpts })
|
302
312
|
}
|
package/src/fireproof.js
CHANGED
@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
|
|
6
6
|
import { CID } from 'multiformats'
|
7
7
|
import { TransactionBlockstore } from './blockstore.js'
|
8
8
|
import { localGet } from './utils.js'
|
9
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
|
9
10
|
|
10
|
-
export { Index, Listener }
|
11
|
+
export { Index, Listener, Database }
|
11
12
|
|
12
|
-
const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
|
13
|
+
const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
|
13
14
|
|
14
15
|
export class Fireproof {
|
15
16
|
/**
|
@@ -40,7 +41,11 @@ export class Fireproof {
|
|
40
41
|
static fromJSON (json, database) {
|
41
42
|
database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
|
42
43
|
if (json.indexes) {
|
43
|
-
for (const {
|
44
|
+
for (const {
|
45
|
+
name,
|
46
|
+
code,
|
47
|
+
clock: { byId, byKey, db }
|
48
|
+
} of json.indexes) {
|
44
49
|
Index.fromJSON(database, {
|
45
50
|
clock: {
|
46
51
|
byId: byId ? parseCID(byId) : null,
|
@@ -67,14 +72,14 @@ export class Fireproof {
|
|
67
72
|
})
|
68
73
|
}
|
69
74
|
const snappedDb = this.fromJSON(definition, withBlocks)
|
70
|
-
;
|
75
|
+
;[...database.indexes.values()].forEach(index => {
|
71
76
|
snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
|
72
77
|
})
|
73
78
|
return snappedDb
|
74
79
|
}
|
75
80
|
|
76
81
|
static async zoom (database, clock) {
|
77
|
-
;
|
82
|
+
;[...database.indexes.values()].forEach(index => {
|
78
83
|
index.indexById = { root: null, cid: null }
|
79
84
|
index.indexByKey = { root: null, cid: null }
|
80
85
|
index.dbHead = null
|
@@ -83,4 +88,39 @@ export class Fireproof {
|
|
83
88
|
await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
|
84
89
|
return database
|
85
90
|
}
|
91
|
+
|
92
|
+
// get all the cids
|
93
|
+
// tell valet to make a file
|
94
|
+
static async makeCar (database, key) {
|
95
|
+
const allCIDs = await database.allCIDs()
|
96
|
+
const blocks = database.blocks
|
97
|
+
|
98
|
+
const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
|
99
|
+
if (typeof key === 'undefined') {
|
100
|
+
key = blocks.valet?.getKeyMaterial()
|
101
|
+
}
|
102
|
+
if (key) {
|
103
|
+
return blocksToEncryptedCarBlock(
|
104
|
+
rootCid,
|
105
|
+
{
|
106
|
+
entries: () => allCIDs.map(cid => ({ cid })),
|
107
|
+
get: async cid => await blocks.get(cid)
|
108
|
+
},
|
109
|
+
key
|
110
|
+
)
|
111
|
+
} else {
|
112
|
+
const carBlocks = await Promise.all(
|
113
|
+
allCIDs.map(async c => {
|
114
|
+
const b = await blocks.get(c)
|
115
|
+
// console.log('block', b)
|
116
|
+
if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
|
117
|
+
// if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
|
118
|
+
return b
|
119
|
+
})
|
120
|
+
)
|
121
|
+
return blocksToCarBlock(rootCid, {
|
122
|
+
entries: () => carBlocks
|
123
|
+
})
|
124
|
+
}
|
125
|
+
}
|
86
126
|
}
|
package/src/prolly.js
CHANGED
@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
|
|
270
270
|
*/
|
271
271
|
export async function eventsSince (blocks, head, since) {
|
272
272
|
if (!head.length) {
|
273
|
-
throw new Error('no head')
|
273
|
+
// throw new Error('no head')
|
274
|
+
return { clockCIDs: [], result: [] }
|
274
275
|
}
|
275
276
|
// @ts-ignore
|
276
277
|
const sinceHead = [...since, ...head] // ?
|
package/src/valet.js
CHANGED
@@ -200,7 +200,7 @@ export class Valet {
|
|
200
200
|
}
|
201
201
|
}
|
202
202
|
|
203
|
-
const blocksToCarBlock = async (lastCid, blocks) => {
|
203
|
+
export const blocksToCarBlock = async (lastCid, blocks) => {
|
204
204
|
let size = 0
|
205
205
|
const headerSize = CBW.headerLength({ roots: [lastCid] })
|
206
206
|
size += headerSize
|
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
|
|
208
208
|
blocks = Array.from(blocks.entries())
|
209
209
|
}
|
210
210
|
for (const { cid, bytes } of blocks) {
|
211
|
+
// console.log(cid, bytes)
|
211
212
|
size += CBW.blockLength({ cid, bytes })
|
212
213
|
}
|
213
214
|
const buffer = new Uint8Array(size)
|
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
|
|
222
223
|
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
223
224
|
}
|
224
225
|
|
225
|
-
const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
226
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
226
227
|
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
227
228
|
const encryptedBlocks = []
|
228
229
|
const theCids = []
|
package/dist/blockstore.js
DELETED
@@ -1,242 +0,0 @@
|
|
1
|
-
import { parse } from 'multiformats/link';
|
2
|
-
import { CID } from 'multiformats';
|
3
|
-
import { Valet } from './valet.js';
|
4
|
-
// const sleep = ms => new Promise(r => setTimeout(r, ms))
|
5
|
-
const husherMap = new Map();
|
6
|
-
const husher = (id, workFn) => {
|
7
|
-
if (!husherMap.has(id)) {
|
8
|
-
husherMap.set(id, workFn().finally(() => setTimeout(() => husherMap.delete(id), 100)));
|
9
|
-
}
|
10
|
-
return husherMap.get(id);
|
11
|
-
};
|
12
|
-
/**
|
13
|
-
* @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
|
14
|
-
*/
|
15
|
-
/**
|
16
|
-
* @typedef {Object} AnyBlock
|
17
|
-
* @property {import('./link').AnyLink} cid - The CID of the block
|
18
|
-
* @property {Uint8Array} bytes - The block's data
|
19
|
-
*
|
20
|
-
* @typedef {Object} Blockstore
|
21
|
-
* @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
|
22
|
-
* @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
|
23
|
-
*
|
24
|
-
* A blockstore that caches writes to a transaction and only persists them when committed.
|
25
|
-
*/
|
26
|
-
export class TransactionBlockstore {
|
27
|
-
/** @type {Map<string, Uint8Array>} */
|
28
|
-
committedBlocks = new Map();
|
29
|
-
valet = null;
|
30
|
-
instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4);
|
31
|
-
inflightTransactions = new Set();
|
32
|
-
constructor(name, encryptionKey) {
|
33
|
-
if (name) {
|
34
|
-
this.valet = new Valet(name, encryptionKey);
|
35
|
-
}
|
36
|
-
this.remoteBlockFunction = null;
|
37
|
-
}
|
38
|
-
/**
|
39
|
-
* Get a block from the store.
|
40
|
-
*
|
41
|
-
* @param {import('./link').AnyLink} cid
|
42
|
-
* @returns {Promise<AnyBlock | undefined>}
|
43
|
-
*/
|
44
|
-
async get(cid) {
|
45
|
-
const key = cid.toString();
|
46
|
-
// it is safe to read from the in-flight transactions becauase they are immutable
|
47
|
-
const bytes = await Promise.any([this.transactionsGet(key), this.committedGet(key)]).catch(e => {
|
48
|
-
// console.log('networkGet', cid.toString(), e)
|
49
|
-
return this.networkGet(key);
|
50
|
-
});
|
51
|
-
if (!bytes)
|
52
|
-
throw new Error('Missing block: ' + key);
|
53
|
-
return { cid, bytes };
|
54
|
-
}
|
55
|
-
// this iterates over the in-flight transactions
|
56
|
-
// and returns the first matching block it finds
|
57
|
-
async transactionsGet(key) {
|
58
|
-
for (const transaction of this.inflightTransactions) {
|
59
|
-
const got = await transaction.get(key);
|
60
|
-
if (got && got.bytes)
|
61
|
-
return got.bytes;
|
62
|
-
}
|
63
|
-
throw new Error('Missing block: ' + key);
|
64
|
-
}
|
65
|
-
async committedGet(key) {
|
66
|
-
const old = this.committedBlocks.get(key);
|
67
|
-
if (old)
|
68
|
-
return old;
|
69
|
-
if (!this.valet)
|
70
|
-
throw new Error('Missing block: ' + key);
|
71
|
-
const got = await this.valet.getBlock(key);
|
72
|
-
// console.log('committedGet: ' + key)
|
73
|
-
this.committedBlocks.set(key, got);
|
74
|
-
return got;
|
75
|
-
}
|
76
|
-
async clearCommittedCache() {
|
77
|
-
this.committedBlocks.clear();
|
78
|
-
}
|
79
|
-
async networkGet(key) {
|
80
|
-
if (this.remoteBlockFunction) {
|
81
|
-
// todo why is this on valet?
|
82
|
-
const value = await husher(key, async () => await this.remoteBlockFunction(key));
|
83
|
-
if (value) {
|
84
|
-
// console.log('networkGot: ' + key, value.length)
|
85
|
-
doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
|
86
|
-
await innerBlockstore.put(CID.parse(key), value);
|
87
|
-
});
|
88
|
-
return value;
|
89
|
-
}
|
90
|
-
}
|
91
|
-
else {
|
92
|
-
return false;
|
93
|
-
}
|
94
|
-
}
|
95
|
-
/**
|
96
|
-
* Add a block to the store. Usually bound to a transaction by a closure.
|
97
|
-
* It sets the lastCid property to the CID of the block that was put.
|
98
|
-
* This is used by the transaction as the head of the car when written to the valet.
|
99
|
-
* We don't have to worry about which transaction we are when we are here because
|
100
|
-
* we are the transactionBlockstore.
|
101
|
-
*
|
102
|
-
* @param {import('./link').AnyLink} cid
|
103
|
-
* @param {Uint8Array} bytes
|
104
|
-
*/
|
105
|
-
put(cid, bytes) {
|
106
|
-
throw new Error('use a transaction to put');
|
107
|
-
}
|
108
|
-
/**
|
109
|
-
* Iterate over all blocks in the store.
|
110
|
-
*
|
111
|
-
* @yields {AnyBlock}
|
112
|
-
* @returns {AsyncGenerator<AnyBlock>}
|
113
|
-
*/
|
114
|
-
// * entries () {
|
115
|
-
// // needs transaction blocks?
|
116
|
-
// // for (const [str, bytes] of this.blocks) {
|
117
|
-
// // yield { cid: parse(str), bytes }
|
118
|
-
// // }
|
119
|
-
// for (const [str, bytes] of this.committedBlocks) {
|
120
|
-
// yield { cid: parse(str), bytes }
|
121
|
-
// }
|
122
|
-
// }
|
123
|
-
/**
|
124
|
-
* Begin a transaction. Ensures the uncommited blocks are empty at the begining.
|
125
|
-
* Returns the blocks to read and write during the transaction.
|
126
|
-
* @returns {InnerBlockstore}
|
127
|
-
* @memberof TransactionBlockstore
|
128
|
-
*/
|
129
|
-
begin(label = '') {
|
130
|
-
const innerTransactionBlockstore = new InnerBlockstore(label, this);
|
131
|
-
this.inflightTransactions.add(innerTransactionBlockstore);
|
132
|
-
return innerTransactionBlockstore;
|
133
|
-
}
|
134
|
-
/**
|
135
|
-
* Commit the transaction. Writes the blocks to the store.
|
136
|
-
* @returns {Promise<void>}
|
137
|
-
* @memberof TransactionBlockstore
|
138
|
-
*/
|
139
|
-
async commit(innerBlockstore) {
|
140
|
-
await this.doCommit(innerBlockstore);
|
141
|
-
}
|
142
|
-
// first get the transaction blockstore from the map of transaction blockstores
|
143
|
-
// then copy it to committedBlocks
|
144
|
-
// then write the transaction blockstore to a car
|
145
|
-
// then write the car to the valet
|
146
|
-
// then remove the transaction blockstore from the map of transaction blockstores
|
147
|
-
doCommit = async (innerBlockstore) => {
|
148
|
-
const cids = new Set();
|
149
|
-
for (const { cid, bytes } of innerBlockstore.entries()) {
|
150
|
-
const stringCid = cid.toString(); // unnecessary string conversion, can we fix upstream?
|
151
|
-
if (this.committedBlocks.has(stringCid)) {
|
152
|
-
// console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
|
153
|
-
}
|
154
|
-
else {
|
155
|
-
this.committedBlocks.set(stringCid, bytes);
|
156
|
-
cids.add(stringCid);
|
157
|
-
}
|
158
|
-
}
|
159
|
-
if (cids.size > 0 && this.valet) {
|
160
|
-
// console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
|
161
|
-
await this.valet.writeTransaction(innerBlockstore, cids);
|
162
|
-
}
|
163
|
-
};
|
164
|
-
/**
|
165
|
-
* Retire the transaction. Clears the uncommited blocks.
|
166
|
-
* @returns {void}
|
167
|
-
* @memberof TransactionBlockstore
|
168
|
-
*/
|
169
|
-
retire(innerBlockstore) {
|
170
|
-
this.inflightTransactions.delete(innerBlockstore);
|
171
|
-
}
|
172
|
-
}
|
173
|
-
/**
|
174
|
-
* Runs a function on an inner blockstore, then persists the change to a car writer
|
175
|
-
* or other outer blockstore.
|
176
|
-
* @param {string} label
|
177
|
-
* @param {TransactionBlockstore} blockstore
|
178
|
-
* @param {(innerBlockstore: Blockstore) => Promise<any>} doFun
|
179
|
-
* @returns {Promise<any>}
|
180
|
-
* @memberof TransactionBlockstore
|
181
|
-
*/
|
182
|
-
export const doTransaction = async (label, blockstore, doFun) => {
|
183
|
-
// @ts-ignore
|
184
|
-
if (!blockstore.commit)
|
185
|
-
return await doFun(blockstore);
|
186
|
-
// @ts-ignore
|
187
|
-
const innerBlockstore = blockstore.begin(label);
|
188
|
-
try {
|
189
|
-
const result = await doFun(innerBlockstore);
|
190
|
-
// @ts-ignore
|
191
|
-
await blockstore.commit(innerBlockstore);
|
192
|
-
return result;
|
193
|
-
}
|
194
|
-
catch (e) {
|
195
|
-
console.error(`Transaction ${label} failed`, e, e.stack);
|
196
|
-
throw e;
|
197
|
-
}
|
198
|
-
finally {
|
199
|
-
// @ts-ignore
|
200
|
-
blockstore.retire(innerBlockstore);
|
201
|
-
}
|
202
|
-
};
|
203
|
-
export class InnerBlockstore {
|
204
|
-
/** @type {Map<string, Uint8Array>} */
|
205
|
-
blocks = new Map();
|
206
|
-
lastCid = null;
|
207
|
-
label = '';
|
208
|
-
parentBlockstore = null;
|
209
|
-
constructor(label, parentBlockstore) {
|
210
|
-
this.label = label;
|
211
|
-
this.parentBlockstore = parentBlockstore;
|
212
|
-
}
|
213
|
-
/**
|
214
|
-
* @param {import('./link').AnyLink} cid
|
215
|
-
* @returns {Promise<AnyBlock | undefined>}
|
216
|
-
*/
|
217
|
-
async get(cid) {
|
218
|
-
const key = cid.toString();
|
219
|
-
let bytes = this.blocks.get(key);
|
220
|
-
if (bytes) {
|
221
|
-
return { cid, bytes };
|
222
|
-
}
|
223
|
-
bytes = await this.parentBlockstore.committedGet(key);
|
224
|
-
if (bytes) {
|
225
|
-
return { cid, bytes };
|
226
|
-
}
|
227
|
-
}
|
228
|
-
/**
|
229
|
-
* @param {import('./link').AnyLink} cid
|
230
|
-
* @param {Uint8Array} bytes
|
231
|
-
*/
|
232
|
-
async put(cid, bytes) {
|
233
|
-
// console.log('put', cid)
|
234
|
-
this.blocks.set(cid.toString(), bytes);
|
235
|
-
this.lastCid = cid;
|
236
|
-
}
|
237
|
-
*entries() {
|
238
|
-
for (const [str, bytes] of this.blocks) {
|
239
|
-
yield { cid: parse(str), bytes };
|
240
|
-
}
|
241
|
-
}
|
242
|
-
}
|