@fireproof/core 0.5.8 → 0.5.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hooks/use-fireproof.js +149 -0
- package/dist/src/blockstore.js +259 -0
- package/dist/src/clock.js +385 -0
- package/dist/src/crypto.js +59 -0
- package/dist/src/database.js +379 -0
- package/dist/src/db-index.js +387 -0
- package/dist/src/fireproof.d.ts +25 -5
- package/dist/src/fireproof.js +131 -52
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +131 -52
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/src/link.js +1 -0
- package/dist/src/listener.js +112 -0
- package/dist/src/prolly.js +360 -0
- package/dist/src/sha1.js +73 -0
- package/dist/src/sync.js +198 -0
- package/dist/src/utils.js +16 -0
- package/dist/src/valet.js +291 -0
- package/hooks/use-fireproof.js +77 -14
- package/package.json +2 -1
- package/src/database.js +15 -4
- package/src/db-index.js +107 -44
- package/src/fireproof.js +2 -2
- package/src/listener.js +3 -1
- package/src/sync.js +5 -1
package/src/database.js
CHANGED
@@ -8,6 +8,7 @@ import { CID } from 'multiformats'
|
|
8
8
|
// TypeScript Types
|
9
9
|
// eslint-disable-next-line no-unused-vars
|
10
10
|
// import { CID } from 'multiformats/dist/types/src/cid.js'
|
11
|
+
|
11
12
|
// eslint-disable-next-line no-unused-vars
|
12
13
|
class Proof {}
|
13
14
|
export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
|
@@ -89,9 +90,9 @@ export class Database {
|
|
89
90
|
}
|
90
91
|
|
91
92
|
// used be indexes etc to notify database listeners of new availability
|
92
|
-
async notifyExternal (source = 'unknown') {
|
93
|
-
|
94
|
-
}
|
93
|
+
// async notifyExternal (source = 'unknown') {
|
94
|
+
// // await this.notifyListeners({ _external: source, _clock: this.clockToJSON() })
|
95
|
+
// }
|
95
96
|
|
96
97
|
/**
|
97
98
|
* Returns the changes made to the Fireproof instance since the specified event.
|
@@ -351,13 +352,23 @@ export class Database {
|
|
351
352
|
* @returns {Function} - A function that can be called to unregister the listener.
|
352
353
|
* @memberof Fireproof
|
353
354
|
*/
|
354
|
-
|
355
|
+
subscribe (listener) {
|
355
356
|
this.listeners.add(listener)
|
356
357
|
return () => {
|
357
358
|
this.listeners.delete(listener)
|
358
359
|
}
|
359
360
|
}
|
360
361
|
|
362
|
+
/**
|
363
|
+
* @deprecated 0.7.0 - renamed subscribe(listener)
|
364
|
+
* @param {Function} listener - The listener to be called when the clock is updated.
|
365
|
+
* @returns {Function} - A function that can be called to unregister the listener.
|
366
|
+
* @memberof Fireproof
|
367
|
+
*/
|
368
|
+
registerListener (listener) {
|
369
|
+
return this.subscribe(listener)
|
370
|
+
}
|
371
|
+
|
361
372
|
async notifyListeners (changes) {
|
362
373
|
// await sleep(10)
|
363
374
|
await this.maybeSaveClock()
|
package/src/db-index.js
CHANGED
@@ -70,14 +70,23 @@ const makeDoc = ({ key, value }) => ({ _id: key, ...value })
|
|
70
70
|
const indexEntriesForChanges = (changes, mapFn) => {
|
71
71
|
const indexEntries = []
|
72
72
|
changes.forEach(({ key, value, del }) => {
|
73
|
+
// key is _id, value is the document
|
73
74
|
if (del || !value) return
|
74
|
-
|
75
|
+
let mapCalled = false
|
76
|
+
const mapReturn = mapFn(makeDoc({ key, value }), (k, v) => {
|
77
|
+
mapCalled = true
|
75
78
|
if (typeof k === 'undefined') return
|
76
79
|
indexEntries.push({
|
77
80
|
key: [charwise.encode(k), key],
|
78
81
|
value: v || null
|
79
82
|
})
|
80
83
|
})
|
84
|
+
if (!mapCalled && mapReturn) {
|
85
|
+
indexEntries.push({
|
86
|
+
key: [charwise.encode(mapReturn), key],
|
87
|
+
value: null
|
88
|
+
})
|
89
|
+
}
|
81
90
|
})
|
82
91
|
return indexEntries
|
83
92
|
}
|
@@ -99,7 +108,10 @@ export class DbIndex {
|
|
99
108
|
constructor (database, name, mapFn, clock = null, opts = {}) {
|
100
109
|
this.database = database
|
101
110
|
if (!database.indexBlocks) {
|
102
|
-
database.indexBlocks = new TransactionBlockstore(
|
111
|
+
database.indexBlocks = new TransactionBlockstore(
|
112
|
+
database?.name + '.indexes',
|
113
|
+
database.blocks.valet?.getKeyMaterial()
|
114
|
+
)
|
103
115
|
}
|
104
116
|
if (typeof name === 'function') {
|
105
117
|
// app is using deprecated API, remove in 0.7
|
@@ -108,13 +120,8 @@ export class DbIndex {
|
|
108
120
|
mapFn = name
|
109
121
|
name = null
|
110
122
|
}
|
111
|
-
|
112
|
-
|
113
|
-
} else {
|
114
|
-
this.mapFn = mapFn
|
115
|
-
this.mapFnString = mapFn.toString()
|
116
|
-
}
|
117
|
-
this.name = name || this.makeName()
|
123
|
+
this.applyMapFn(mapFn, name)
|
124
|
+
|
118
125
|
this.indexById = { root: null, cid: null }
|
119
126
|
this.indexByKey = { root: null, cid: null }
|
120
127
|
this.dbHead = null
|
@@ -125,13 +132,34 @@ export class DbIndex {
|
|
125
132
|
}
|
126
133
|
this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
|
127
134
|
this.updateIndexPromise = null
|
128
|
-
if (!opts.temporary) {
|
135
|
+
if (!opts.temporary) {
|
136
|
+
DbIndex.registerWithDatabase(this, this.database)
|
137
|
+
}
|
138
|
+
}
|
139
|
+
|
140
|
+
applyMapFn (mapFn, name) {
|
141
|
+
if (typeof mapFn === 'string') {
|
142
|
+
this.mapFnString = mapFn
|
143
|
+
} else {
|
144
|
+
this.mapFn = mapFn
|
145
|
+
this.mapFnString = mapFn.toString()
|
146
|
+
}
|
147
|
+
this.name = name || this.makeName()
|
129
148
|
}
|
130
149
|
|
131
150
|
makeName () {
|
132
151
|
const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g
|
133
|
-
|
134
|
-
|
152
|
+
let matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim())
|
153
|
+
if (matches.length === 0) {
|
154
|
+
matches = /=>\s*(.*)/.exec(this.mapFnString)
|
155
|
+
}
|
156
|
+
if (matches.length === 0) {
|
157
|
+
return this.mapFnString
|
158
|
+
} else {
|
159
|
+
// it's a consise arrow function, match everythign after the arrow
|
160
|
+
this.includeDocsDefault = true
|
161
|
+
return matches[1]
|
162
|
+
}
|
135
163
|
}
|
136
164
|
|
137
165
|
static registerWithDatabase (inIndex, database) {
|
@@ -141,7 +169,8 @@ export class DbIndex {
|
|
141
169
|
// merge our inIndex code with the inIndex clock or vice versa
|
142
170
|
const existingIndex = database.indexes.get(inIndex.mapFnString)
|
143
171
|
// keep the code instance, discard the clock instance
|
144
|
-
if (existingIndex.mapFn) {
|
172
|
+
if (existingIndex.mapFn) {
|
173
|
+
// this one also has other config
|
145
174
|
existingIndex.dbHead = inIndex.dbHead
|
146
175
|
existingIndex.indexById.cid = inIndex.indexById.cid
|
147
176
|
existingIndex.indexByKey.cid = inIndex.indexByKey.cid
|
@@ -185,7 +214,7 @@ export class DbIndex {
|
|
185
214
|
/**
|
186
215
|
* Query object can have {range}
|
187
216
|
* @param {DbQuery} query - the query range to use
|
188
|
-
* @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any}>}>}
|
217
|
+
* @returns {Promise<{proof: {}, rows: Array<{id: string, key: string, value: any, doc?: any}>}>}
|
189
218
|
* @memberof DbIndex
|
190
219
|
* @instance
|
191
220
|
*/
|
@@ -193,20 +222,63 @@ export class DbIndex {
|
|
193
222
|
// const callId = Math.random().toString(36).substring(2, 7)
|
194
223
|
// todo pass a root to query a snapshot
|
195
224
|
// console.time(callId + '.updateIndex')
|
196
|
-
update && await this.updateIndex(this.database.indexBlocks)
|
225
|
+
update && (await this.updateIndex(this.database.indexBlocks))
|
197
226
|
// console.timeEnd(callId + '.updateIndex')
|
198
227
|
// console.time(callId + '.doIndexQuery')
|
199
228
|
// console.log('query', query)
|
200
|
-
const response = await doIndexQuery(
|
229
|
+
const response = await this.doIndexQuery(query)
|
201
230
|
// console.timeEnd(callId + '.doIndexQuery')
|
202
231
|
return {
|
203
232
|
proof: { index: await cidsToProof(response.cids) },
|
204
|
-
rows: response.result.map(({ id, key, row }) => {
|
205
|
-
return
|
233
|
+
rows: response.result.map(({ id, key, row, doc }) => {
|
234
|
+
return { id, key: charwise.decode(key), value: row, doc }
|
206
235
|
})
|
207
236
|
}
|
208
237
|
}
|
209
238
|
|
239
|
+
/**
|
240
|
+
*
|
241
|
+
* @param {any} resp
|
242
|
+
* @param {any} query
|
243
|
+
* @returns
|
244
|
+
*/
|
245
|
+
async applyQuery (resp, query) {
|
246
|
+
if (query.descending) {
|
247
|
+
resp.result = resp.result.reverse()
|
248
|
+
}
|
249
|
+
if (query.limit) {
|
250
|
+
resp.result = resp.result.slice(0, query.limit)
|
251
|
+
}
|
252
|
+
if (query.includeDocs) {
|
253
|
+
resp.result = await Promise.all(
|
254
|
+
resp.result.map(async row => {
|
255
|
+
const doc = await this.database.get(row.id)
|
256
|
+
return { ...row, doc }
|
257
|
+
})
|
258
|
+
)
|
259
|
+
}
|
260
|
+
return resp
|
261
|
+
}
|
262
|
+
|
263
|
+
async doIndexQuery (query = {}) {
|
264
|
+
await loadIndex(this.database.indexBlocks, this.indexByKey, dbIndexOpts)
|
265
|
+
if (!this.indexByKey.root) return { result: [] }
|
266
|
+
if (query.includeDocs === undefined) query.includeDocs = this.includeDocsDefault
|
267
|
+
if (query.range) {
|
268
|
+
const encodedRange = query.range.map(key => charwise.encode(key))
|
269
|
+
return await this.applyQuery(await this.indexByKey.root.range(...encodedRange), query)
|
270
|
+
} else if (query.key) {
|
271
|
+
const encodedKey = charwise.encode(query.key)
|
272
|
+
return await this.applyQuery(this.indexByKey.root.get(encodedKey), query)
|
273
|
+
} else {
|
274
|
+
const { result, ...all } = await this.indexByKey.root.getAllEntries()
|
275
|
+
return await this.applyQuery(
|
276
|
+
{ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all },
|
277
|
+
query
|
278
|
+
)
|
279
|
+
}
|
280
|
+
}
|
281
|
+
|
210
282
|
/**
|
211
283
|
* Update the DbIndex with the latest changes
|
212
284
|
* @private
|
@@ -223,7 +295,9 @@ export class DbIndex {
|
|
223
295
|
})
|
224
296
|
}
|
225
297
|
this.updateIndexPromise = this.innerUpdateIndex(blocks)
|
226
|
-
this.updateIndexPromise.finally(() => {
|
298
|
+
this.updateIndexPromise.finally(() => {
|
299
|
+
this.updateIndexPromise = null
|
300
|
+
})
|
227
301
|
return this.updateIndexPromise
|
228
302
|
}
|
229
303
|
|
@@ -250,7 +324,7 @@ export class DbIndex {
|
|
250
324
|
this.dbHead = result.clock
|
251
325
|
return
|
252
326
|
}
|
253
|
-
const didT = await doTransaction('updateIndex', inBlocks, async
|
327
|
+
const didT = await doTransaction('updateIndex', inBlocks, async blocks => {
|
254
328
|
let oldIndexEntries = []
|
255
329
|
let removeByIdIndexEntries = []
|
256
330
|
await loadIndex(blocks, this.indexById, idIndexOpts)
|
@@ -258,19 +332,28 @@ export class DbIndex {
|
|
258
332
|
// console.log('head', this.dbHead, this.indexById)
|
259
333
|
if (this.indexById.root) {
|
260
334
|
const oldChangeEntries = await this.indexById.root.getMany(result.rows.map(({ key }) => key))
|
261
|
-
oldIndexEntries = oldChangeEntries.result.map(
|
335
|
+
oldIndexEntries = oldChangeEntries.result.map(key => ({ key, del: true }))
|
262
336
|
removeByIdIndexEntries = oldIndexEntries.map(({ key }) => ({ key: key[1], del: true }))
|
263
337
|
}
|
264
338
|
if (!this.mapFn) {
|
265
|
-
throw new Error(
|
339
|
+
throw new Error(
|
340
|
+
'No live map function installed for index, cannot update. Make sure your index definition runs before any queries.' +
|
341
|
+
(this.mapFnString ? ' Your code should match the stored map function source:\n' + this.mapFnString : '')
|
342
|
+
)
|
266
343
|
}
|
267
344
|
const indexEntries = indexEntriesForChanges(result.rows, this.mapFn)
|
268
345
|
const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }))
|
269
|
-
this.indexById = await bulkIndex(
|
346
|
+
this.indexById = await bulkIndex(
|
347
|
+
blocks,
|
348
|
+
this.indexById,
|
349
|
+
removeByIdIndexEntries.concat(byIdIndexEntries),
|
350
|
+
idIndexOpts
|
351
|
+
)
|
270
352
|
this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
|
271
353
|
this.dbHead = result.clock
|
272
354
|
})
|
273
|
-
|
355
|
+
// todo index subscriptions
|
356
|
+
// this.database.notifyExternal('dbIndex')
|
274
357
|
// console.timeEnd(callTag + '.doTransactionupdateIndex')
|
275
358
|
// console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
|
276
359
|
return didT
|
@@ -331,23 +414,3 @@ async function loadIndex (blocks, index, indexOpts) {
|
|
331
414
|
}
|
332
415
|
return index.root
|
333
416
|
}
|
334
|
-
|
335
|
-
async function applyLimit (results, limit) {
|
336
|
-
results.result = results.result.slice(0, limit)
|
337
|
-
return results
|
338
|
-
}
|
339
|
-
|
340
|
-
async function doIndexQuery (blocks, indexByKey, query = {}) {
|
341
|
-
await loadIndex(blocks, indexByKey, dbIndexOpts)
|
342
|
-
if (!indexByKey.root) return { result: [] }
|
343
|
-
if (query.range) {
|
344
|
-
const encodedRange = query.range.map((key) => charwise.encode(key))
|
345
|
-
return applyLimit(await indexByKey.root.range(...encodedRange), query.limit)
|
346
|
-
} else if (query.key) {
|
347
|
-
const encodedKey = charwise.encode(query.key)
|
348
|
-
return indexByKey.root.get(encodedKey)
|
349
|
-
} else {
|
350
|
-
const { result, ...all } = await indexByKey.root.getAllEntries()
|
351
|
-
return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit)
|
352
|
-
}
|
353
|
-
}
|
package/src/fireproof.js
CHANGED
@@ -24,7 +24,7 @@ export class Fireproof {
|
|
24
24
|
if (existing) {
|
25
25
|
const existingConfig = JSON.parse(existing)
|
26
26
|
const fp = new Database(new TransactionBlockstore(name, existingConfig.key), [], opts)
|
27
|
-
return
|
27
|
+
return Fireproof.fromJSON(existingConfig, fp)
|
28
28
|
} else {
|
29
29
|
const instanceKey = randomBytes(32).toString('hex') // pass null to disable encryption
|
30
30
|
return new Database(new TransactionBlockstore(name, instanceKey), [], opts)
|
@@ -67,7 +67,7 @@ export class Fireproof {
|
|
67
67
|
index.clock.db = null
|
68
68
|
})
|
69
69
|
}
|
70
|
-
const snappedDb =
|
70
|
+
const snappedDb = Fireproof.fromJSON(definition, withBlocks)
|
71
71
|
;[...database.indexes.values()].forEach(index => {
|
72
72
|
snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
|
73
73
|
})
|
package/src/listener.js
CHANGED
@@ -8,7 +8,9 @@
|
|
8
8
|
* @param {Function} routingFn - The routing function to apply to each entry in the database.
|
9
9
|
*/
|
10
10
|
// import { ChangeEvent } from './db-index'
|
11
|
-
|
11
|
+
/**
|
12
|
+
* @deprecated since version 0.7.0
|
13
|
+
*/
|
12
14
|
export class Listener {
|
13
15
|
subcribers = new Map()
|
14
16
|
doStopListening = null
|
package/src/sync.js
CHANGED
@@ -97,6 +97,10 @@ export class Sync {
|
|
97
97
|
// get the roots parents
|
98
98
|
const parents = await Promise.all(roots.map(async (cid) => {
|
99
99
|
const rbl = await reader.get(cid)
|
100
|
+
if (!rbl) {
|
101
|
+
console.log('missing root block', cid.toString(), reader)
|
102
|
+
throw new Error('missing root block')
|
103
|
+
}
|
100
104
|
const block = await decodeEventBlock(rbl.bytes)
|
101
105
|
return block.value.parents
|
102
106
|
}))
|
@@ -113,7 +117,7 @@ export class Sync {
|
|
113
117
|
} else if (message.clock) {
|
114
118
|
const reqCidDiff = message
|
115
119
|
// this might be a CID diff
|
116
|
-
|
120
|
+
console.log('got diff', reqCidDiff)
|
117
121
|
const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids)
|
118
122
|
if (!carBlock) {
|
119
123
|
// we are full synced
|