@fireproof/core 0.4.1 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@fireproof/core",
3
- "version": "0.4.1",
4
- "description": "Realtime database for IPFS",
3
+ "version": "0.5.1",
4
+ "description": "Cloudless database for apps, the browser, and IPFS",
5
5
  "main": "dist/src/fireproof.js",
6
6
  "module": "dist/src/fireproof.mjs",
7
7
  "typings": "dist/src/fireproof.d.ts",
@@ -10,8 +10,8 @@
10
10
  "scripts": {
11
11
  "keygen": "node scripts/keygen.js",
12
12
  "test": "standard && npm run test:unencrypted && npm run test:mocha",
13
- "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
14
- "test:mocha": "mocha --reporter list test/*.test.js",
13
+ "test:unencrypted": "set NO_ENCRYPT=true && npm run test:mocha",
14
+ "test:mocha": "mocha test/*.test.js",
15
15
  "test:watch": "npm run test:mocha -- -w --parallel",
16
16
  "coverage": "c8 -r html -r text npm test",
17
17
  "prepublishOnly": "cp ../../README.md . && npm run build",
package/src/database.js CHANGED
@@ -117,7 +117,7 @@ export class Database {
117
117
  // console.log('change rows', this.instanceId, rows)
118
118
  } else {
119
119
  const allResp = await getAll(this.blocks, this.clock)
120
- rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value })))
120
+ rows = allResp.result.map(({ key, value }) => decodeEvent({ key, value }))
121
121
  dataCIDs = allResp.cids
122
122
  // console.log('dbdoc rows', this.instanceId, rows)
123
123
  }
@@ -130,7 +130,9 @@ export class Database {
130
130
 
131
131
  async allDocuments () {
132
132
  const allResp = await getAll(this.blocks, this.clock)
133
- const rows = allResp.result.map(({ key, value }) => (decodeEvent({ key, value }))).map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
133
+ const rows = allResp.result
134
+ .map(({ key, value }) => decodeEvent({ key, value }))
135
+ .map(({ key, value }) => ({ key, value: { _id: key, ...value } }))
134
136
  return {
135
137
  rows,
136
138
  clock: this.clockToJSON(),
@@ -138,6 +140,15 @@ export class Database {
138
140
  }
139
141
  }
140
142
 
143
+ async allCIDs () {
144
+ const allResp = await getAll(this.blocks, this.clock)
145
+ const cids = await cidsToProof(allResp.cids)
146
+ const clockCids = await cidsToProof(allResp.clockCIDs)
147
+ // console.log('allcids', cids, clockCids)
148
+ // todo we need to put the clock head as the last block in the encrypted car
149
+ return [...cids, ...clockCids] // need a single block version of clock head, maybe an encoded block for it
150
+ }
151
+
141
152
  /**
142
153
  * Runs validation on the specified document using the Fireproof instance's configuration. Throws an error if the document is invalid.
143
154
  *
@@ -150,7 +161,7 @@ export class Database {
150
161
  async runValidation (doc) {
151
162
  if (this.config && this.config.validateChange) {
152
163
  const oldDoc = await this.get(doc._id)
153
- .then((doc) => doc)
164
+ .then(doc => doc)
154
165
  .catch(() => ({}))
155
166
  this.config.validateChange(doc, oldDoc, this.authCtx)
156
167
  }
@@ -185,13 +196,13 @@ export class Database {
185
196
  return doc
186
197
  }
187
198
  /**
188
- * @typedef {Object} Document
189
- * @property {string} _id - The ID of the document (required)
190
- * @property {string} [_proof] - The proof of the document (optional)
191
- * @property {string} [_clock] - The clock of the document (optional)
192
- * @property {any} [key: string] - Index signature notation to allow any other unknown fields
193
- * * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
194
- */
199
+ * @typedef {Object} Document
200
+ * @property {string} _id - The ID of the document (required)
201
+ * @property {string} [_proof] - The proof of the document (optional)
202
+ * @property {string} [_clock] - The clock of the document (optional)
203
+ * @property {any} [key: string] - Index signature notation to allow any other unknown fields
204
+ * * @property {Object.<string, any>} [otherProperties] - Any other unknown properties (optional)
205
+ */
195
206
 
196
207
  /**
197
208
  * Adds a new document to the database, or updates an existing document. Returns the ID of the document and the new clock head.
@@ -246,17 +257,17 @@ export class Database {
246
257
  throw new Error('MVCC conflict, document is changed, please reload the document and try again.')
247
258
  }
248
259
  }
260
+ const prevClock = [...this.clock]
249
261
  const result = await doTransaction(
250
262
  'putToProllyTree',
251
263
  this.blocks,
252
- async (blocks) => await put(blocks, this.clock, event)
264
+ async blocks => await put(blocks, this.clock, event)
253
265
  )
254
266
  if (!result) {
255
267
  console.error('failed', event)
256
268
  throw new Error('failed to put at storage layer')
257
269
  }
258
- // console.log('new clock head', this.instanceId, result.head.toString())
259
- this.clock = result.head // do we want to do this as a finally block
270
+ this.applyClock(prevClock, result.head)
260
271
  await this.notifyListeners([decodedEvent]) // this type is odd
261
272
  return {
262
273
  id: decodedEvent.key,
@@ -266,6 +277,12 @@ export class Database {
266
277
  // todo should include additions (or split clock)
267
278
  }
268
279
 
280
+ applyClock (prevClock, newClock) {
281
+ // console.log('applyClock', prevClock, newClock, this.clock)
282
+ const removedprevCIDs = this.clock.filter(cid => prevClock.indexOf(cid) === -1)
283
+ this.clock = removedprevCIDs.concat(newClock)
284
+ }
285
+
269
286
  // /**
270
287
  // * Advances the clock to the specified event and updates the root CID
271
288
  // * Will be used by replication
@@ -324,7 +341,7 @@ export class Database {
324
341
  export async function cidsToProof (cids) {
325
342
  if (!cids || !cids.all) return []
326
343
  const all = await cids.all()
327
- return [...all].map((cid) => cid.toString())
344
+ return [...all].map(cid => cid.toString())
328
345
  }
329
346
 
330
347
  function decodeEvent (event) {
package/src/db-index.js CHANGED
@@ -72,10 +72,10 @@ const indexEntriesForChanges = (changes, mapFn) => {
72
72
  changes.forEach(({ key, value, del }) => {
73
73
  if (del || !value) return
74
74
  mapFn(makeDoc({ key, value }), (k, v) => {
75
- if (typeof v === 'undefined' || typeof k === 'undefined') return
75
+ if (typeof k === 'undefined') return
76
76
  indexEntries.push({
77
77
  key: [charwise.encode(k), key],
78
- value: v
78
+ value: v || null
79
79
  })
80
80
  })
81
81
  })
@@ -93,23 +93,25 @@ const indexEntriesForChanges = (changes, mapFn) => {
93
93
  *
94
94
  */
95
95
  export class DbIndex {
96
- constructor (database, mapFn, clock, opts = {}) {
96
+ constructor (database, name, mapFn, clock = null, opts = {}) {
97
97
  this.database = database
98
98
  if (!database.indexBlocks) {
99
99
  database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial())
100
100
  }
101
- /**
102
- * The map function to apply to each entry in the database.
103
- * @type {Function}
104
- */
105
-
101
+ if (typeof name === 'function') {
102
+ // app is using deprecated API, remove in 0.7
103
+ opts = clock
104
+ clock = mapFn
105
+ mapFn = name
106
+ name = null
107
+ }
106
108
  if (typeof mapFn === 'string') {
107
109
  this.mapFnString = mapFn
108
110
  } else {
109
111
  this.mapFn = mapFn
110
112
  this.mapFnString = mapFn.toString()
111
113
  }
112
- this.name = opts.name || this.makeName()
114
+ this.name = name || this.makeName()
113
115
  this.indexById = { root: null, cid: null }
114
116
  this.indexByKey = { root: null, cid: null }
115
117
  this.dbHead = null
@@ -159,7 +161,7 @@ export class DbIndex {
159
161
 
160
162
  static fromJSON (database, { code, clock, name }) {
161
163
  // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
162
- return new DbIndex(database, code, clock, { name })
164
+ return new DbIndex(database, name, code, clock)
163
165
  }
164
166
 
165
167
  /**
@@ -176,7 +178,7 @@ export class DbIndex {
176
178
  * @memberof DbIndex
177
179
  * @instance
178
180
  */
179
- async query (query, update = true) {
181
+ async query (query = {}, update = true) {
180
182
  // const callId = Math.random().toString(36).substring(2, 7)
181
183
  // todo pass a root to query a snapshot
182
184
  // console.time(callId + '.updateIndex')
@@ -203,7 +205,12 @@ export class DbIndex {
203
205
  async updateIndex (blocks) {
204
206
  // todo this could enqueue the request and give fresh ones to all second comers -- right now it gives out stale promises while working
205
207
  // what would it do in a world where all indexes provide a database snapshot to query?
206
- if (this.updateIndexPromise) return this.updateIndexPromise
208
+ if (this.updateIndexPromise) {
209
+ return this.updateIndexPromise.then(() => {
210
+ this.updateIndexPromise = null
211
+ return this.updateIndex(blocks)
212
+ })
213
+ }
207
214
  this.updateIndexPromise = this.innerUpdateIndex(blocks)
208
215
  this.updateIndexPromise.finally(() => { this.updateIndexPromise = null })
209
216
  return this.updateIndexPromise
@@ -232,7 +239,7 @@ export class DbIndex {
232
239
  this.dbHead = result.clock
233
240
  return
234
241
  }
235
- await doTransaction('updateIndex', inBlocks, async (blocks) => {
242
+ const didT = await doTransaction('updateIndex', inBlocks, async (blocks) => {
236
243
  let oldIndexEntries = []
237
244
  let removeByIdIndexEntries = []
238
245
  await loadIndex(blocks, this.indexById, idIndexOpts)
@@ -254,6 +261,7 @@ export class DbIndex {
254
261
  this.database.notifyExternal('dbIndex')
255
262
  // console.timeEnd(callTag + '.doTransactionupdateIndex')
256
263
  // console.log(`updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
264
+ return didT
257
265
  }
258
266
  }
259
267
 
@@ -296,7 +304,11 @@ async function bulkIndex (blocks, inIndex, indexEntries, opts) {
296
304
  async function loadIndex (blocks, index, indexOpts) {
297
305
  if (!index.root) {
298
306
  const cid = index.cid
299
- if (!cid) return
307
+ if (!cid) {
308
+ // console.log('no cid', index)
309
+ // throw new Error('cannot load index')
310
+ return null
311
+ }
300
312
  const { getBlock } = makeGetBlock(blocks)
301
313
  index.root = await load({ cid, get: getBlock, ...indexOpts })
302
314
  }
package/src/fireproof.js CHANGED
@@ -6,10 +6,11 @@ import { DbIndex as Index } from './db-index.js'
6
6
  import { CID } from 'multiformats'
7
7
  import { TransactionBlockstore } from './blockstore.js'
8
8
  import { localGet } from './utils.js'
9
+ import { blocksToCarBlock, blocksToEncryptedCarBlock } from './valet.js'
9
10
 
10
- export { Index, Listener }
11
+ export { Index, Listener, Database }
11
12
 
12
- const parseCID = cid => typeof cid === 'string' ? CID.parse(cid) : cid
13
+ const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid)
13
14
 
14
15
  export class Fireproof {
15
16
  /**
@@ -40,7 +41,11 @@ export class Fireproof {
40
41
  static fromJSON (json, database) {
41
42
  database.hydrate({ clock: json.clock.map(c => parseCID(c)), name: json.name, key: json.key })
42
43
  if (json.indexes) {
43
- for (const { name, code, clock: { byId, byKey, db } } of json.indexes) {
44
+ for (const {
45
+ name,
46
+ code,
47
+ clock: { byId, byKey, db }
48
+ } of json.indexes) {
44
49
  Index.fromJSON(database, {
45
50
  clock: {
46
51
  byId: byId ? parseCID(byId) : null,
@@ -67,14 +72,14 @@ export class Fireproof {
67
72
  })
68
73
  }
69
74
  const snappedDb = this.fromJSON(definition, withBlocks)
70
- ;([...database.indexes.values()]).forEach(index => {
75
+ ;[...database.indexes.values()].forEach(index => {
71
76
  snappedDb.indexes.get(index.mapFnString).mapFn = index.mapFn
72
77
  })
73
78
  return snappedDb
74
79
  }
75
80
 
76
81
  static async zoom (database, clock) {
77
- ;([...database.indexes.values()]).forEach(index => {
82
+ ;[...database.indexes.values()].forEach(index => {
78
83
  index.indexById = { root: null, cid: null }
79
84
  index.indexByKey = { root: null, cid: null }
80
85
  index.dbHead = null
@@ -83,4 +88,39 @@ export class Fireproof {
83
88
  await database.notifyReset() // hmm... indexes should listen to this? might be more complex than worth it. so far this is the only caller
84
89
  return database
85
90
  }
91
+
92
+ // get all the cids
93
+ // tell valet to make a file
94
+ static async makeCar (database, key) {
95
+ const allCIDs = await database.allCIDs()
96
+ const blocks = database.blocks
97
+
98
+ const rootCid = CID.parse(allCIDs[allCIDs.length - 1])
99
+ if (typeof key === 'undefined') {
100
+ key = blocks.valet?.getKeyMaterial()
101
+ }
102
+ if (key) {
103
+ return blocksToEncryptedCarBlock(
104
+ rootCid,
105
+ {
106
+ entries: () => allCIDs.map(cid => ({ cid })),
107
+ get: async cid => await blocks.get(cid)
108
+ },
109
+ key
110
+ )
111
+ } else {
112
+ const carBlocks = await Promise.all(
113
+ allCIDs.map(async c => {
114
+ const b = await blocks.get(c)
115
+ // console.log('block', b)
116
+ if (typeof b.cid === 'string') { b.cid = CID.parse(b.cid) }
117
+ // if (b.bytes.constructor.name === 'Buffer') console.log('conver vbuff')
118
+ return b
119
+ })
120
+ )
121
+ return blocksToCarBlock(rootCid, {
122
+ entries: () => carBlocks
123
+ })
124
+ }
125
+ }
86
126
  }
package/src/prolly.js CHANGED
@@ -270,7 +270,8 @@ export async function root (inBlocks, head) {
270
270
  */
271
271
  export async function eventsSince (blocks, head, since) {
272
272
  if (!head.length) {
273
- throw new Error('no head')
273
+ // throw new Error('no head')
274
+ return { clockCIDs: [], result: [] }
274
275
  }
275
276
  // @ts-ignore
276
277
  const sinceHead = [...since, ...head] // ?
package/src/valet.js CHANGED
@@ -200,7 +200,7 @@ export class Valet {
200
200
  }
201
201
  }
202
202
 
203
- const blocksToCarBlock = async (lastCid, blocks) => {
203
+ export const blocksToCarBlock = async (lastCid, blocks) => {
204
204
  let size = 0
205
205
  const headerSize = CBW.headerLength({ roots: [lastCid] })
206
206
  size += headerSize
@@ -208,6 +208,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
208
208
  blocks = Array.from(blocks.entries())
209
209
  }
210
210
  for (const { cid, bytes } of blocks) {
211
+ // console.log(cid, bytes)
211
212
  size += CBW.blockLength({ cid, bytes })
212
213
  }
213
214
  const buffer = new Uint8Array(size)
@@ -222,7 +223,7 @@ const blocksToCarBlock = async (lastCid, blocks) => {
222
223
  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
223
224
  }
224
225
 
225
- const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
226
+ export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial) => {
226
227
  const encryptionKey = Buffer.from(keyMaterial, 'hex')
227
228
  const encryptedBlocks = []
228
229
  const theCids = []