@fireproof/core 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -21
- package/dist/src/fireproof.d.ts +20 -15
- package/dist/src/fireproof.js +284 -247
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +284 -248
- package/dist/src/fireproof.mjs.map +1 -1
- package/package.json +4 -4
- package/src/database.js +31 -14
- package/src/db-index.js +26 -14
- package/src/fireproof.js +45 -5
- package/src/prolly.js +2 -1
- package/src/valet.js +3 -2
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "@fireproof/core",
|
3
|
-
"version": "0.
|
4
|
-
"description": "
|
3
|
+
"version": "0.5.1",
|
4
|
+
"description": "Cloudless database for apps, the browser, and IPFS",
|
5
5
|
"main": "dist/src/fireproof.js",
|
6
6
|
"module": "dist/src/fireproof.mjs",
|
7
7
|
"typings": "dist/src/fireproof.d.ts",
|
@@ -10,8 +10,8 @@
|
|
10
10
|
"scripts": {
|
11
11
|
"keygen": "node scripts/keygen.js",
|
12
12
|
"test": "standard && npm run test:unencrypted && npm run test:mocha",
|
13
|
-
"test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
|
14
|
-
"test:mocha": "mocha
|
13
|
+
"test:unencrypted": "set NO_ENCRYPT=true && npm run test:mocha",
|
14
|
+
"test:mocha": "mocha test/*.test.js",
|
15
15
|
"test:watch": "npm run test:mocha -- -w --parallel",
|
16
16
|
"coverage": "c8 -r html -r text npm test",
|
17
17
|
"prepublishOnly": "cp ../../README.md . && npm run build",
|
package/src/database.js
CHANGED
@@ -117,7 +117,7 @@ export class Database {
|
|
117
117
|
// console.log('change rows', this.instanceId, rows)
|
118
118
|
} else {
|
119
119
|
const allResp = await getAll(this.blocks, this.clock)
|
120
|
-
rows = allResp.result.map(({ key, value }) =>
|
120
|
+
rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
|
121
121
|
dataCIDs = allResp.cids
|
122
122
|
// console.log('dbdoc rows', this.instanceId, rows)
|
123
123
|
}
|
@@ -130,7 +130,9 @@ export class Database {
|
|
130
130
|
|
131
131
|
async allDocuments () {
|
132
132
|
const allResp = await getAll(this.blocks, this.clock)
|
133
|
-
const rows = allResp.result
|
133
|
+
const rows = allResp.result
|
134
|
+
.map(({ key, value }) => decodeEvent({ key, value }))
|
135
|
+
.map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
|
134
136
|
return {
|
135
137
|
rows,
|
136
138
|
clock: this.clockToJSON(),
|
@@ -138,6 +140,15 @@ export class Database {
|
|
138
140
|
}
|
139
141
|
}
|
140
142
|
|
143
|
+
async allCIDs () {
|
144
|
+
const allResp = await getAll(this.blocks, this.clock)
|
145
|
+
const cids = await cidsToProof(allResp.cids)
|
146
|
+
const clockCids = await cidsToProof(allResp.clockCIDs)
|
147
|
+
// console.log('allcids', cids, clockCids)
|
148
|
+
// todo we need to put the clock head as the last block in the encrypted car
|
149
|
+
return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
|
150
|
+
}
|
151
|
+
|
141
152
|
/**
|
142
153
|
* Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
|
143
154
|
*
|
@@ -150,7 +161,7 @@ export class Database {
|
|
150
161
|
async runValidation (doc) {
|
151
162
|
if (this.config && this.config.validateChange) {
|
152
163
|
const oldDoc = await this.get(doc._id)
|
153
|
-
.then(
|
164
|
+
.then(doc => doc)
|
154
165
|
.catch(() => ({}))
|
155
166
|
this.config.validateChange(doc, oldDoc, this.authCtx)
|
156
167
|
}
|
@@ -185,13 +196,13 @@ export class Database {
|
|
185
196
|
return doc
|
186
197
|
}
|
187
198
|
/**
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
199
|
+
* @typedef {Object} Document
|
200
|
+
* @property {string} _id - The ID of the document (required)
|
201
|
+
* @property {string} [_proof] - The proof of the document (optional)
|
202
|
+
* @property {string} [_clock] - The clock of the document (optional)
|
203
|
+
* @property {any} [key: string] - Index signature notation to allow any other unknown fields
|
204
|
+
* * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
|
205
|
+
*/
|
195
206
|
|
196
207
|
/**
|
197
208
|
* Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
|
@@ -246,17 +257,17 @@ export class Database {
|
|
246
257
|
throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
|
247
258
|
}
|
248
259
|
}
|
260
|
+
const prevClock = [...this.clock]
|
249
261
|
const result = await doTransaction(
|
250
262
|
'putToProllyTree',
|
251
263
|
this.blocks,
|
252
|
-
async
|
264
|
+
async blocks => await put(blocks, this.clock, event)
|
253
265
|
)
|
254
266
|
if (!result) {
|
255
267
|
console.error('failed', event)
|
256
268
|
throw new Error('failed to put at storage layer')
|
257
269
|
}
|
258
|
-
|
259
|
-
this.clock = result.head // do we want to do this as a finally block
|
270
|
+
this.applyClock(prevClock, result.head)
|
260
271
|
await this.notifyListeners([decodedEvent]) // this type is odd
|
261
272
|
return {
|
262
273
|
id: decodedEvent.key,
|
@@ -266,6 +277,12 @@ export class Database {
|
|
266
277
|
// todo should include additions (or split clock)
|
267
278
|
}
|
268
279
|
|
280
|
+
applyClock (prevClock, newClock) {
|
281
|
+
// console.log('applyClock', prevClock, newClock, this.clock)
|
282
|
+
const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
|
283
|
+
this.clock = removedprevCIDs.concat(newClock)
|
284
|
+
}
|
285
|
+
|
269
286
|
// /**
|
270
287
|
// * Advances the clock to the specified event and updates the root CID
|
271
288
|
// * Will be used by replication
|
@@ -324,7 +341,7 @@ export class Database {
|
|
324
341
|
export async function cidsToProof (cids) {
|
325
342
|
if (!cids || !cids.all) return []
|
326
343
|
const all = await cids.all()
|
327
|
-
return [...all].map(
|
344
|
+
return [...all].map(cid => cid.toString())
|
328
345
|
}
|
329
346
|
|
330
347
|
function decodeEvent (event) {
|
package/src/db-index.js
CHANGED
@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
|
|
72
72
|
changes.forEach(({ key, value, del }) => {
|
73
73
|
if (del || !value) return
|
74
74
|
mapFn(makeDoc({ key, value }), (k, v) => {
|
75
|
-
if (typeof
|
75
|
+
if (typeof k === 'undefined') return
|
76
76
|
indexEntries.push({
|
77
77
|
key: [charwise.encode(k), key],
|
78
|
-
value: v
|
78
|
+
value: v || null
|
79
79
|
})
|
80
80
|
})
|
81
81
|
})
|
@@ -93,23 +93,25 @@ const indexEntriesForChanges = (changes, mapFn) => {
|
|
93
93
|
*
|
94
94
|
*/
|
95
95
|
export class DbIndex {
|
96
|
-
constructor (database, mapFn, clock, opts = {}) {
|
96
|
+
constructor (database, name, mapFn, clock = null, opts = {}) {
|
97
97
|
this.database = database
|
98
98
|
if (!database.indexBlocks) {
|
99
99
|
database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
|
100
100
|
}
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
101
|
+
if (typeof name === 'function') {
|
102
|
+
// app is using deprecated API, remove in 0.7
|
103
|
+
opts = clock
|
104
|
+
clock = mapFn
|
105
|
+
mapFn = name
|
106
|
+
name = null
|
107
|
+
}
|
106
108
|
if (typeof mapFn === 'string') {
|
107
109
|
this.mapFnString = mapFn
|
108
110
|
} else {
|
109
111
|
this.mapFn = mapFn
|
110
112
|
this.mapFnString = mapFn.toString()
|
111
113
|
}
|
112
|
-
this.name =
|
114
|
+
this.name = name || this.makeName()
|
113
115
|
this.indexById = { root: null, cid: null }
|
114
116
|
this.indexByKey = { root: null, cid: null }
|
115
117
|
this.dbHead = null
|
@@ -159,7 +161,7 @@ export class DbIndex {
|
|
159
161
|
|
160
162
|
static fromJSON (database, { code, clock, name }) {
|
161
163
|
// console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
|
162
|
-
return new DbIndex(database, code, clock
|
164
|
+
return new DbIndex(database, name, code, clock)
|
163
165
|
}
|
164
166
|
|
165
167
|
/**
|
@@ -176,7 +178,7 @@ export class DbIndex {
|
|
176
178
|
* @memberof DbIndex
|
177
179
|
* @instance
|
178
180
|
*/
|
179
|
-
async query (query, update = true) {
|
181
|
+
async query (query = {}, update = true) {
|
180
182
|
// const callId = Math.random().toString(36).substring(2, 7)
|
181
183
|
// todo pass a root to query a snapshot
|
182
184
|
// console.time(callId + '.updateIndex')
|
@@ -203,7 +205,12 @@ export class DbIndex {
|
|
203
205
|
async updateIndex (blocks) {
|
204
206
|
// todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
|
205
207
|
// what would it do in a world where all indexes provide a database snapshot to query?
|
206
|
-
if (this.updateIndexPromise)
|
208
|
+
if (this.updateIndexPromise) {
|
209
|
+
return this.updateIndexPromise.then(() => {
|
210
|
+
this.updateIndexPromise = null
|
211
|
+
return this.updateIndex(blocks)
|
212
|
+
})
|
213
|
+
}
|
207
214
|
this.updateIndexPromise = this.innerUpdateIndex(blocks)
|
208
215
|
this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
|
209
216
|
return this.updateIndexPromise
|
@@ -232,7 +239,7 @@ export class DbIndex {
|
|
232
239
|
this.dbHead = result.clock
|
233
240
|
return
|
234
241
|
}
|
235
|
-
await doTransaction('updateIndex', inBlocks, async (blocks) => {
|
242
|
+
const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
|
236
243
|
let oldIndexEntries = []
|
237
244
|
let removeByIdIndexEntries = []
|
238
245
|
await loadIndex(blocks, this.indexById, idIndexOpts)
|
@@ -254,6 +261,7 @@ export class DbIndex {
|
|
254
261
|
this.database.notifyExternal('dbIndex')
|
255
262
|
// console.timeEnd(callTag + '.doTransactionupdateIndex')
|
256
263
|
// console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
|
264
|
+
return didT
|
257
265
|
}
|
258
266
|
}
|
259
267
|
|
@@ -296,7 +304,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
|
|
296
304
|
async function loadIndex (blocks, index, indexOpts) {
|
297
305
|
if (!index.root) {
|
298
306
|
const cid = index.cid
|
299
|
-
if (!cid)
|
307
|
+
if (!cid) {
|
308
|
+
// console.log('no cid', index)
|
309
|
+
// throw new Error('cannot load index')
|
310
|
+
return null
|
311
|
+
}
|
300
312
|
const { getBlock } = makeGetBlock(blocks)
|
301
313
|
index.root = await load({ cid, get: getBlock, ...indexOpts })
|
302
314
|
}
|
package/src/fireproof.js
CHANGED
@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
|
|
6
6
|
import { CID } from 'multiformats'
|
7
7
|
import { TransactionBlockstore } from './blockstore.js'
|
8
8
|
import { localGet } from './utils.js'
|
9
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
|
9
10
|
|
10
|
-
export { Index, Listener }
|
11
|
+
export { Index, Listener, Database }
|
11
12
|
|
12
|
-
const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
|
13
|
+
const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
|
13
14
|
|
14
15
|
export class Fireproof {
|
15
16
|
/**
|
@@ -40,7 +41,11 @@ export class Fireproof {
|
|
40
41
|
static fromJSON (json, database) {
|
41
42
|
database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
|
42
43
|
if (json.indexes) {
|
43
|
-
for (const {
|
44
|
+
for (const {
|
45
|
+
name,
|
46
|
+
code,
|
47
|
+
clock: { byId, byKey, db }
|
48
|
+
} of json.indexes) {
|
44
49
|
Index.fromJSON(database, {
|
45
50
|
clock: {
|
46
51
|
byId: byId ? parseCID(byId) : null,
|
@@ -67,14 +72,14 @@ export class Fireproof {
|
|
67
72
|
})
|
68
73
|
}
|
69
74
|
const snappedDb = this.fromJSON(definition, withBlocks)
|
70
|
-
;
|
75
|
+
;[...database.indexes.values()].forEach(index => {
|
71
76
|
snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
|
72
77
|
})
|
73
78
|
return snappedDb
|
74
79
|
}
|
75
80
|
|
76
81
|
static async zoom (database, clock) {
|
77
|
-
;
|
82
|
+
;[...database.indexes.values()].forEach(index => {
|
78
83
|
index.indexById = { root: null, cid: null }
|
79
84
|
index.indexByKey = { root: null, cid: null }
|
80
85
|
index.dbHead = null
|
@@ -83,4 +88,39 @@ export class Fireproof {
|
|
83
88
|
await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
|
84
89
|
return database
|
85
90
|
}
|
91
|
+
|
92
|
+
// get all the cids
|
93
|
+
// tell valet to make a file
|
94
|
+
static async makeCar (database, key) {
|
95
|
+
const allCIDs = await database.allCIDs()
|
96
|
+
const blocks = database.blocks
|
97
|
+
|
98
|
+
const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
|
99
|
+
if (typeof key === 'undefined') {
|
100
|
+
key = blocks.valet?.getKeyMaterial()
|
101
|
+
}
|
102
|
+
if (key) {
|
103
|
+
return blocksToEncryptedCarBlock(
|
104
|
+
rootCid,
|
105
|
+
{
|
106
|
+
entries: () => allCIDs.map(cid => ({ cid })),
|
107
|
+
get: async cid => await blocks.get(cid)
|
108
|
+
},
|
109
|
+
key
|
110
|
+
)
|
111
|
+
} else {
|
112
|
+
const carBlocks = await Promise.all(
|
113
|
+
allCIDs.map(async c => {
|
114
|
+
const b = await blocks.get(c)
|
115
|
+
// console.log('block', b)
|
116
|
+
if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
|
117
|
+
// if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
|
118
|
+
return b
|
119
|
+
})
|
120
|
+
)
|
121
|
+
return blocksToCarBlock(rootCid, {
|
122
|
+
entries: () => carBlocks
|
123
|
+
})
|
124
|
+
}
|
125
|
+
}
|
86
126
|
}
|
package/src/prolly.js
CHANGED
@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
|
|
270
270
|
*/
|
271
271
|
export async function eventsSince (blocks, head, since) {
|
272
272
|
if (!head.length) {
|
273
|
-
throw new Error('no head')
|
273
|
+
// throw new Error('no head')
|
274
|
+
return { clockCIDs: [], result: [] }
|
274
275
|
}
|
275
276
|
// @ts-ignore
|
276
277
|
const sinceHead = [...since, ...head] // ?
|
package/src/valet.js
CHANGED
@@ -200,7 +200,7 @@ export class Valet {
|
|
200
200
|
}
|
201
201
|
}
|
202
202
|
|
203
|
-
const blocksToCarBlock = async (lastCid, blocks) => {
|
203
|
+
export const blocksToCarBlock = async (lastCid, blocks) => {
|
204
204
|
let size = 0
|
205
205
|
const headerSize = CBW.headerLength({ roots: [lastCid] })
|
206
206
|
size += headerSize
|
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
|
|
208
208
|
blocks = Array.from(blocks.entries())
|
209
209
|
}
|
210
210
|
for (const { cid, bytes } of blocks) {
|
211
|
+
// console.log(cid, bytes)
|
211
212
|
size += CBW.blockLength({ cid, bytes })
|
212
213
|
}
|
213
214
|
const buffer = new Uint8Array(size)
|
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
|
|
222
223
|
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
223
224
|
}
|
224
225
|
|
225
|
-
const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
226
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
|
226
227
|
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
227
228
|
const encryptedBlocks = []
|
228
229
|
const theCids = []
|