@fireproof/core 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fireproof/core",
3
- "version": "0.4.1",
3
+ "version": "0.5.0",
4
4
  "description": "Realtime database for IPFS",
5
5
  "main": "dist/src/fireproof.js",
6
6
  "module": "dist/src/fireproof.mjs",
@@ -11,7 +11,7 @@
11
11
  "keygen": "node scripts/keygen.js",
12
12
  "test": "standard && npm run test:unencrypted && npm run test:mocha",
13
13
  "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
14
- "test:mocha": "mocha --reporter list test/*.test.js",
14
+ "test:mocha": "mocha test/*.test.js",
15
15
  "test:watch": "npm run test:mocha -- -w --parallel",
16
16
  "coverage": "c8 -r html -r text npm test",
17
17
  "prepublishOnly": "cp ../../README.md . && npm run build",
package/src/database.js CHANGED
@@ -117,7 +117,7 @@ export class Database {
117
117
  // console.log('change rows', this.instanceId, rows)
118
118
  } else {
119
119
  const allResp = await getAll(this.blocks, this.clock)
120
- rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
120
+ rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
121
121
  dataCIDs = allResp.cids
122
122
  // console.log('dbdoc rows', this.instanceId, rows)
123
123
  }
@@ -130,7 +130,9 @@ export class Database {
130
130
 
131
131
  async allDocuments () {
132
132
  const allResp = await getAll(this.blocks, this.clock)
133
- const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
133
+ const rows = allResp.result
134
+ .map(({ key, value }) => decodeEvent({ key, value }))
135
+ .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
134
136
  return {
135
137
  rows,
136
138
  clock: this.clockToJSON(),
@@ -138,6 +140,15 @@ export class Database {
138
140
  }
139
141
  }
140
142
 
143
+ async allCIDs () {
144
+ const allResp = await getAll(this.blocks, this.clock)
145
+ const cids = await cidsToProof(allResp.cids)
146
+ const clockCids = await cidsToProof(allResp.clockCIDs)
147
+ // console.log('allcids', cids, clockCids)
148
+ // todo we need to put the clock head as the last block in the encrypted car
149
+ return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
150
+ }
151
+
141
152
  /**
142
153
  * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
143
154
  *
@@ -150,7 +161,7 @@ export class Database {
150
161
  async runValidation (doc) {
151
162
  if (this.config && this.config.validateChange) {
152
163
  const oldDoc = await this.get(doc._id)
153
- .then((doc) => doc)
164
+ .then(doc => doc)
154
165
  .catch(() => ({}))
155
166
  this.config.validateChange(doc, oldDoc, this.authCtx)
156
167
  }
@@ -185,13 +196,13 @@ export class Database {
185
196
  return doc
186
197
  }
187
198
  /**
188
- * @typedef {Object} Document
189
- * @property {string} _id - The ID of the document (required)
190
- * @property {string} [_proof] - The proof of the document (optional)
191
- * @property {string} [_clock] - The clock of the document (optional)
192
- * @property {any} [key: string] - Index signature notation to allow any other unknown fields
193
- * * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
194
- */
199
+ * @typedef {Object} Document
200
+ * @property {string} _id - The ID of the document (required)
201
+ * @property {string} [_proof] - The proof of the document (optional)
202
+ * @property {string} [_clock] - The clock of the document (optional)
203
+ * @property {any} [key: string] - Index signature notation to allow any other unknown fields
204
+ * * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
205
+ */
195
206
 
196
207
  /**
197
208
  * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
@@ -246,17 +257,17 @@ export class Database {
246
257
  throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
247
258
  }
248
259
  }
260
+ const prevClock = [...this.clock]
249
261
  const result = await doTransaction(
250
262
  'putToProllyTree',
251
263
  this.blocks,
252
- async (blocks) => await put(blocks, this.clock, event)
264
+ async blocks => await put(blocks, this.clock, event)
253
265
  )
254
266
  if (!result) {
255
267
  console.error('failed', event)
256
268
  throw new Error('failed to put at storage layer')
257
269
  }
258
- // console.log('new clock head', this.instanceId, result.head.toString())
259
- this.clock = result.head // do we want to do this as a finally block
270
+ this.applyClock(prevClock, result.head)
260
271
  await this.notifyListeners([decodedEvent]) // this type is odd
261
272
  return {
262
273
  id: decodedEvent.key,
@@ -266,6 +277,12 @@ export class Database {
266
277
  // todo should include additions (or split clock)
267
278
  }
268
279
 
280
+ applyClock (prevClock, newClock) {
281
+ // console.log('applyClock', prevClock, newClock, this.clock)
282
+ const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
283
+ this.clock = removedprevCIDs.concat(newClock)
284
+ }
285
+
269
286
  // /**
270
287
  // * Advances the clock to the specified event and updates the root CID
271
288
  // * Will be used by replication
@@ -324,7 +341,7 @@ export class Database {
324
341
  export async function cidsToProof (cids) {
325
342
  if (!cids || !cids.all) return []
326
343
  const all = await cids.all()
327
- return [...all].map((cid) => cid.toString())
344
+ return [...all].map(cid => cid.toString())
328
345
  }
329
346
 
330
347
  function decodeEvent (event) {
package/src/db-index.js CHANGED
@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
72
72
  changes.forEach(({ key, value, del }) => {
73
73
  if (del || !value) return
74
74
  mapFn(makeDoc({ key, value }), (k, v) => {
75
- if (typeof v === 'undefined' || typeof k === 'undefined') return
75
+ if (typeof k === 'undefined') return
76
76
  indexEntries.push({
77
77
  key: [charwise.encode(k), key],
78
- value: v
78
+ value: v || null
79
79
  })
80
80
  })
81
81
  })
@@ -93,7 +93,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
93
93
  *
94
94
  */
95
95
  export class DbIndex {
96
- constructor (database, mapFn, clock, opts = {}) {
96
+ constructor (database, name, mapFn, clock = null, opts = {}) {
97
97
  this.database = database
98
98
  if (!database.indexBlocks) {
99
99
  database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
@@ -109,7 +109,7 @@ export class DbIndex {
109
109
  this.mapFn = mapFn
110
110
  this.mapFnString = mapFn.toString()
111
111
  }
112
- this.name = opts.name || this.makeName()
112
+ this.name = name || this.makeName()
113
113
  this.indexById = { root: null, cid: null }
114
114
  this.indexByKey = { root: null, cid: null }
115
115
  this.dbHead = null
@@ -159,7 +159,7 @@ export class DbIndex {
159
159
 
160
160
  static fromJSON (database, { code, clock, name }) {
161
161
  // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
162
- return new DbIndex(database, code, clock, { name })
162
+ return new DbIndex(database, name, code, clock)
163
163
  }
164
164
 
165
165
  /**
@@ -176,7 +176,7 @@ export class DbIndex {
176
176
  * @memberof DbIndex
177
177
  * @instance
178
178
  */
179
- async query (query, update = true) {
179
+ async query (query = {}, update = true) {
180
180
  // const callId = Math.random().toString(36).substring(2, 7)
181
181
  // todo pass a root to query a snapshot
182
182
  // console.time(callId + '.updateIndex')
@@ -203,7 +203,12 @@ export class DbIndex {
203
203
  async updateIndex (blocks) {
204
204
  // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
205
205
  // what would it do in a world where all indexes provide a database snapshot to query?
206
- if (this.updateIndexPromise) return this.updateIndexPromise
206
+ if (this.updateIndexPromise) {
207
+ return this.updateIndexPromise.then(() => {
208
+ this.updateIndexPromise = null
209
+ return this.updateIndex(blocks)
210
+ })
211
+ }
207
212
  this.updateIndexPromise = this.innerUpdateIndex(blocks)
208
213
  this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
209
214
  return this.updateIndexPromise
@@ -232,7 +237,7 @@ export class DbIndex {
232
237
  this.dbHead = result.clock
233
238
  return
234
239
  }
235
- await doTransaction('updateIndex', inBlocks, async (blocks) => {
240
+ const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
236
241
  let oldIndexEntries = []
237
242
  let removeByIdIndexEntries = []
238
243
  await loadIndex(blocks, this.indexById, idIndexOpts)
@@ -254,6 +259,7 @@ export class DbIndex {
254
259
  this.database.notifyExternal('dbIndex')
255
260
  // console.timeEnd(callTag + '.doTransactionupdateIndex')
256
261
  // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
262
+ return didT
257
263
  }
258
264
  }
259
265
 
@@ -296,7 +302,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
296
302
  async function loadIndex (blocks, index, indexOpts) {
297
303
  if (!index.root) {
298
304
  const cid = index.cid
299
- if (!cid) return
305
+ if (!cid) {
306
+ // console.log('no cid', index)
307
+ // throw new Error('cannot load index')
308
+ return null
309
+ }
300
310
  const { getBlock } = makeGetBlock(blocks)
301
311
  index.root = await load({ cid, get: getBlock, ...indexOpts })
302
312
  }
package/src/fireproof.js CHANGED
@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
6
6
  import { CID } from 'multiformats'
7
7
  import { TransactionBlockstore } from './blockstore.js'
8
8
  import { localGet } from './utils.js'
9
+ import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
9
10
 
10
- export { Index, Listener }
11
+ export { Index, Listener, Database }
11
12
 
12
- const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
13
+ const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
13
14
 
14
15
  export class Fireproof {
15
16
  /**
@@ -40,7 +41,11 @@ export class Fireproof {
40
41
  static fromJSON (json, database) {
41
42
  database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
42
43
  if (json.indexes) {
43
- for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
44
+ for (const {
45
+ name,
46
+ code,
47
+ clock: { byId, byKey, db }
48
+ } of json.indexes) {
44
49
  Index.fromJSON(database, {
45
50
  clock: {
46
51
  byId: byId ? parseCID(byId) : null,
@@ -67,14 +72,14 @@ export class Fireproof {
67
72
  })
68
73
  }
69
74
  const snappedDb = this.fromJSON(definition, withBlocks)
70
- ;([...database.indexes.values()]).forEach(index => {
75
+ ;[...database.indexes.values()].forEach(index => {
71
76
  snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
72
77
  })
73
78
  return snappedDb
74
79
  }
75
80
 
76
81
  static async zoom (database, clock) {
77
- ;([...database.indexes.values()]).forEach(index => {
82
+ ;[...database.indexes.values()].forEach(index => {
78
83
  index.indexById = { root: null, cid: null }
79
84
  index.indexByKey = { root: null, cid: null }
80
85
  index.dbHead = null
@@ -83,4 +88,39 @@ export class Fireproof {
83
88
  await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
84
89
  return database
85
90
  }
91
+
92
+ // get all the cids
93
+ // tell valet to make a file
94
+ static async makeCar (database, key) {
95
+ const allCIDs = await database.allCIDs()
96
+ const blocks = database.blocks
97
+
98
+ const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
99
+ if (typeof key === 'undefined') {
100
+ key = blocks.valet?.getKeyMaterial()
101
+ }
102
+ if (key) {
103
+ return blocksToEncryptedCarBlock(
104
+ rootCid,
105
+ {
106
+ entries: () => allCIDs.map(cid => ({ cid })),
107
+ get: async cid => await blocks.get(cid)
108
+ },
109
+ key
110
+ )
111
+ } else {
112
+ const carBlocks = await Promise.all(
113
+ allCIDs.map(async c => {
114
+ const b = await blocks.get(c)
115
+ // console.log('block', b)
116
+ if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
117
+ // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
118
+ return b
119
+ })
120
+ )
121
+ return blocksToCarBlock(rootCid, {
122
+ entries: () => carBlocks
123
+ })
124
+ }
125
+ }
86
126
  }
package/src/prolly.js CHANGED
@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
270
270
  */
271
271
  export async function eventsSince (blocks, head, since) {
272
272
  if (!head.length) {
273
- throw new Error('no head')
273
+ // throw new Error('no head')
274
+ return { clockCIDs: [], result: [] }
274
275
  }
275
276
  // @ts-ignore
276
277
  const sinceHead = [...since, ...head] // ?
package/src/valet.js CHANGED
@@ -200,7 +200,7 @@ export class Valet {
200
200
  }
201
201
  }
202
202
 
203
- const blocksToCarBlock = async (lastCid, blocks) => {
203
+ export const blocksToCarBlock = async (lastCid, blocks) => {
204
204
  let size = 0
205
205
  const headerSize = CBW.headerLength({ roots: [lastCid] })
206
206
  size += headerSize
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
208
208
  blocks = Array.from(blocks.entries())
209
209
  }
210
210
  for (const { cid, bytes } of blocks) {
211
+ // console.log(cid, bytes)
211
212
  size += CBW.blockLength({ cid, bytes })
212
213
  }
213
214
  const buffer = new Uint8Array(size)
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
222
223
  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
223
224
  }
224
225
 
225
- const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
226
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
226
227
  const encryptionKey = Buffer.from(keyMaterial, 'hex')
227
228
  const encryptedBlocks = []
228
229
  const theCids = []