@fireproof/core 0.1.1 → 0.3.0

package/hooks/use-fireproof.ts

@@ -4,44 +4,53 @@
 import { useEffect, useState, createContext } from 'react'
 import { Fireproof, Listener, Hydrator } from '../index'
 
-
 export interface FireproofCtxValue {
   addSubscriber: (label: String, fn: Function) => void
   database: Fireproof
   ready: boolean
+  persist: () => void
 }
 export const FireproofCtx = createContext<FireproofCtxValue>({
   addSubscriber: () => {},
   database: null,
-  ready: false,
+  ready: false
 })
 
 const inboundSubscriberQueue = new Map()
-const database = Fireproof.storage()
-const listener = new Listener(database)
-let startedSetup = false;
+
+let startedSetup = false
+let database
+let listener
+const initializeDatabase = name => {
+  if (database) return
+  database = Fireproof.storage(name)
+  listener = new Listener(database)
+}
 
 /**
  * @function useFireproof
  * React hook to initialize a Fireproof database, automatically saving and loading the clock.
+ * You might need to `import { nodePolyfills } from 'vite-plugin-node-polyfills'` in your vite.config.ts
  * @param [defineDatabaseFn] Synchronous function that defines the database, run this before any async calls
  * @param [setupDatabaseFn] Asynchronous function that sets up the database, run this to load fixture data etc
  * @returns {FireproofCtxValue} { addSubscriber, database, ready }
  */
-export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Function): FireproofCtxValue {
+export function useFireproof(
+  defineDatabaseFn = (database: Fireproof) => {},
+  setupDatabaseFn = async (database: Fireproof) => {},
+  name: string
+): FireproofCtxValue {
   const [ready, setReady] = useState(false)
-  defineDatabaseFn = defineDatabaseFn || (() => {})
-  setupDatabaseFn = setupDatabaseFn || (() => {})
-  // console.log('useFireproof', database, ready)
-
+  initializeDatabase(name || 'useFireproof')
+  const localStorageKey = 'fp.' + database.name
 
   const addSubscriber = (label: String, fn: Function) => {
     inboundSubscriberQueue.set(label, fn)
   }
 
-  const listenerCallback = async () => {
-    // console.log ('listenerCallback', JSON.stringify(database))
-    localSet('fireproof', JSON.stringify(database))
+  const listenerCallback = async event => {
+      localSet(localStorageKey, JSON.stringify(database))
+    if (event._external) return
     for (const [, fn] of inboundSubscriberQueue) fn()
   }
 
@@ -51,14 +60,14 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
       if (startedSetup) return
       startedSetup = true
       defineDatabaseFn(database) // define indexes before querying them
-      const fp = localGet('fireproof')
+      console.log('Initializing database', database.name)
+      const fp = localGet(localStorageKey) // todo use db.name
       if (fp) {
-        const serialized = JSON.parse(fp)
-        // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
-        console.log("Loading previous database clock. (localStorage.removeItem('fireproof') to reset)")
-        Hydrator.fromJSON(serialized, database)
-        // await database.setClock(clock)
         try {
+          const serialized = JSON.parse(fp)
+          // console.log('serialized', JSON.stringify(serialized.indexes.map(c => c.clock)))
+          console.log(`Loading previous database clock. (localStorage.removeItem('${localStorageKey}') to reset)`)
+          await Hydrator.fromJSON(serialized, database)
           const changes = await database.changesSince()
           if (changes.rows.length < 2) {
             // console.log('Resetting database')
@@ -66,16 +75,16 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
           }
         } catch (e) {
           console.error(`Error loading previous database clock. ${fp} Resetting.`, e)
-          await database.setClock([]) // todo this should be resetClock and also reset the indexes
+          await Hydrator.zoom(database, [])
           await setupDatabaseFn(database)
-          localSet('fireproof', JSON.stringify(database))
+          localSet(localStorageKey, JSON.stringify(database))
         }
       } else {
         await setupDatabaseFn(database)
-        localSet('fireproof', JSON.stringify(database))
+        localSet(localStorageKey, JSON.stringify(database))
       }
       setReady(true)
-      listener.on('*', hushed('*', listenerCallback, 250))
+      listener.on('*', listenerCallback)//hushed('*', listenerCallback, 250))
     }
     doSetup()
   }, [ready])
@@ -84,6 +93,9 @@ export function useFireproof(defineDatabaseFn: Function, setupDatabaseFn: Functi
     addSubscriber,
     database,
     ready,
+    persist: () => {
+      localSet(localStorageKey, JSON.stringify(database))
+    }
   }
 }
 
@@ -91,12 +103,17 @@ const husherMap = new Map()
 const husher = (id: string, workFn: { (): Promise<any> }, ms: number) => {
   if (!husherMap.has(id)) {
     const start: number = Date.now()
-    husherMap.set(id, workFn().finally(() => 
-      setTimeout(() => husherMap.delete(id), ms - (Date.now() - start))))
+    husherMap.set(
+      id,
+      workFn().finally(() => setTimeout(() => husherMap.delete(id), ms - (Date.now() - start)))
+    )
   }
   return husherMap.get(id)
 }
-const hushed = (id: string, workFn: { (): Promise<any> }, ms: number) => () => husher(id, workFn, ms)
+const hushed =
+  (id: string, workFn: { (...args): Promise<any> }, ms: number) =>
+  (...args) =>
+    husher(id, () => workFn(...args), ms)
 
 let storageSupported = false
 try {
@@ -116,4 +133,4 @@ function localSet(key: string, value: string) {
 //   if (storageSupported) {
 //     return localStorage && localStorage.removeItem(key)
 //   }
-// }
+// }

package/package.json

@@ -1,15 +1,17 @@
 {
   "name": "@fireproof/core",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Realtime database for IPFS",
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "test": "standard && npm run test:mocha",
-    "test:mocha": "mocha test/*.test.js",
-    "test:watch": "npm run test:mocha -- -w --parallel test/*.test.js",
+    "keygen": "node scripts/keygen.js",
+    "test": "standard && npm run test:unencrypted && npm run test:mocha",
+    "test:unencrypted": "NO_ENCRYPT=true npm run test:mocha",
+    "test:mocha": "mocha --reporter list test/*.test.js",
+    "test:watch": "npm run test:mocha -- -w --parallel",
     "coverage": "c8 -r html -r text npm test",
-    "prepublishOnly" : "cp ../../README.md .",
+    "prepublishOnly": "cp ../../README.md .",
     "postpublish": "rm README.md",
     "lint": "standard",
     "lint:fix": "standard --fix"
@@ -30,6 +32,7 @@
   ],
   "license": "Apache-2.0 OR MIT",
   "dependencies": {
+    "prolly-trees": "1.0.4",
     "@ipld/car": "^5.1.0",
     "@ipld/dag-cbor": "^9.0.0",
     "archy": "^1.0.0",
@@ -37,9 +40,9 @@
     "car-transaction": "^1.0.1",
     "charwise": "^3.0.1",
     "cli-color": "^2.0.3",
+    "encrypted-block": "^0.0.3",
     "idb": "^7.1.1",
     "multiformats": "^11.0.1",
-    "prolly-trees": "1.0.3",
     "sade": "^1.8.1"
   },
   "devDependencies": {

package/scripts/keygen.js

@@ -0,0 +1,3 @@
+import { randomBytes } from 'crypto'
+
+console.log(randomBytes(32).toString('hex'))

package/src/blockstore.js

@@ -1,10 +1,5 @@
 import { parse } from 'multiformats/link'
-import * as raw from 'multiformats/codecs/raw'
-import { sha256 } from 'multiformats/hashes/sha2'
-import * as Block from 'multiformats/block'
-import * as CBW from '@ipld/car/buffer-writer'
 import { CID } from 'multiformats'
-
 import Valet from './valet.js'
 
 // const sleep = ms => new Promise(r => setTimeout(r, ms))
@@ -34,15 +29,15 @@ const husher = (id, workFn) => {
  */
 export default class TransactionBlockstore {
   /** @type {Map<string, Uint8Array>} */
-  #oldBlocks = new Map()
+  #committedBlocks = new Map()
 
   valet = null
 
   #instanceId = 'blkz.' + Math.random().toString(36).substring(2, 4)
   #inflightTransactions = new Set()
 
-  constructor (name) {
-    this.valet = new Valet(name)
+  constructor (name, encryptionKey) {
+    this.valet = new Valet(name, encryptionKey)
   }
 
   /**
@@ -54,7 +49,7 @@ export default class TransactionBlockstore {
   async get (cid) {
     const key = cid.toString()
     // it is safe to read from the in-flight transactions becauase they are immutable
-    const bytes = await Promise.any([this.#transactionsGet(key), this.commitedGet(key)]).catch((e) => {
+    const bytes = await Promise.any([this.#transactionsGet(key), this.committedGet(key)]).catch(e => {
       // console.log('networkGet', cid.toString(), e)
       return this.networkGet(key)
     })
@@ -72,18 +67,26 @@ export default class TransactionBlockstore {
     throw new Error('Missing block: ' + key)
   }
 
-  async commitedGet (key) {
-    const old = this.#oldBlocks.get(key)
+  async committedGet (key) {
+    const old = this.#committedBlocks.get(key)
     if (old) return old
-    return await this.valet.getBlock(key)
+    const got = await this.valet.getBlock(key)
+    // console.log('committedGet: ' + key)
+    this.#committedBlocks.set(key, got)
+    return got
+  }
+
+  async clearCommittedCache () {
+    this.#committedBlocks.clear()
   }
 
   async networkGet (key) {
     if (this.valet.remoteBlockFunction) {
+      // todo why is this on valet?
       const value = await husher(key, async () => await this.valet.remoteBlockFunction(key))
       if (value) {
         // console.log('networkGot: ' + key, value.length)
-        doTransaction('networkGot: ' + key, this, async (innerBlockstore) => {
+        doTransaction('networkGot: ' + key, this, async innerBlockstore => {
           await innerBlockstore.put(CID.parse(key), value)
         })
         return value
@@ -118,7 +121,7 @@ export default class TransactionBlockstore {
   //   // for (const [str, bytes] of this.#blocks) {
   //   //   yield { cid: parse(str), bytes }
   //   // }
-  //   for (const [str, bytes] of this.#oldBlocks) {
+  //   for (const [str, bytes] of this.#committedBlocks) {
   //     yield { cid: parse(str), bytes }
   //   }
   // }
@@ -145,39 +148,24 @@ export default class TransactionBlockstore {
   }
 
   // first get the transaction blockstore from the map of transaction blockstores
-  // then copy it to oldBlocks
+  // then copy it to committedBlocks
   // then write the transaction blockstore to a car
   // then write the car to the valet
   // then remove the transaction blockstore from the map of transaction blockstores
-  #doCommit = async (innerBlockstore) => {
+  #doCommit = async innerBlockstore => {
     const cids = new Set()
     for (const { cid, bytes } of innerBlockstore.entries()) {
       const stringCid = cid.toString() // unnecessary string conversion, can we fix upstream?
-      if (this.#oldBlocks.has(stringCid)) {
-        // console.log('Duplicate block: ' + stringCid)
+      if (this.#committedBlocks.has(stringCid)) {
+        // console.log('Duplicate block: ' + stringCid) // todo some of this can be avoided, cost is extra size on car files
       } else {
-        this.#oldBlocks.set(stringCid, bytes)
+        this.#committedBlocks.set(stringCid, bytes)
         cids.add(stringCid)
       }
     }
     if (cids.size > 0) {
       // console.log(innerBlockstore.label, 'committing', cids.size, 'blocks')
-      await this.#valetWriteTransaction(innerBlockstore, cids)
-    }
-  }
-
-  /**
-   * Group the blocks into a car and write it to the valet.
-   * @param {InnerBlockstore} innerBlockstore
-   * @param {Set<string>} cids
-   * @returns {Promise<void>}
-   * @memberof TransactionBlockstore
-   * @private
-   */
-  #valetWriteTransaction = async (innerBlockstore, cids) => {
-    if (innerBlockstore.lastCid) {
-      const newCar = await blocksToCarBlock(innerBlockstore.lastCid, innerBlockstore)
-      await this.valet.parkCar(newCar.cid.toString(), newCar.bytes, cids)
+      await this.valet.writeTransaction(innerBlockstore, cids)
     }
   }
 
@@ -215,25 +203,6 @@ export const doTransaction = async (label, blockstore, doFun) => {
   }
 }
 
-const blocksToCarBlock = async (lastCid, blocks) => {
-  let size = 0
-  const headerSize = CBW.headerLength({ roots: [lastCid] })
-  size += headerSize
-  for (const { cid, bytes } of blocks.entries()) {
-    size += CBW.blockLength({ cid, bytes })
-  }
-  const buffer = new Uint8Array(size)
-  const writer = await CBW.createWriter(buffer, { headerSize })
-
-  writer.addRoot(lastCid)
-
-  for (const { cid, bytes } of blocks.entries()) {
-    writer.write({ cid, bytes })
-  }
-  await writer.close()
-  return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
-}
-
 /** @implements {BlockFetcher} */
 export class InnerBlockstore {
   /** @type {Map<string, Uint8Array>} */
@@ -254,8 +223,10 @@ export class InnerBlockstore {
   async get (cid) {
     const key = cid.toString()
     let bytes = this.#blocks.get(key)
-    if (bytes) { return { cid, bytes } }
-    bytes = await this.parentBlockstore.commitedGet(key)
+    if (bytes) {
+      return { cid, bytes }
+    }
+    bytes = await this.parentBlockstore.committedGet(key)
     if (bytes) {
       return { cid, bytes }
     }

package/src/clock.js

@@ -220,7 +220,6 @@ export async function findEventsToSync (blocks, head) {
 }
 
 const asyncFilter = async (arr, predicate) =>
-
   Promise.all(arr.map(predicate)).then((results) => arr.filter((_v, index) => results[index]))
 
 export async function findCommonAncestorWithSortedEvents (events, children) {

package/src/crypto.js

@@ -0,0 +1,65 @@
+import * as codec from 'encrypted-block'
+import {
+  create,
+  load
+} from 'prolly-trees/cid-set'
+import { CID } from 'multiformats'
+import { encode, decode, create as mfCreate } from 'multiformats/block'
+import * as dagcbor from '@ipld/dag-cbor'
+import { sha256 as hasher } from 'multiformats/hashes/sha2'
+
+const createBlock = (bytes, cid) => mfCreate({ cid, bytes, hasher, codec })
+
+const encrypt = async function * ({ get, cids, hasher, key, cache, chunker, root }) {
+  const set = new Set()
+  let eroot
+  for (const string of cids) {
+    const cid = CID.parse(string)
+    const unencrypted = await get(cid)
+    const block = await encode({ ...await codec.encrypt({ ...unencrypted, key }), codec, hasher })
+    // console.log(`encrypting ${string} as ${block.cid}`)
+    yield block
+    set.add(block.cid.toString())
+    if (unencrypted.cid.equals(root)) eroot = block.cid
+  }
+  if (!eroot) throw new Error('cids does not include root')
+  const list = [...set].map(s => CID.parse(s))
+  let last
+  for await (const node of create({ list, get, cache, chunker, hasher, codec: dagcbor })) {
+    const block = await node.block
+    yield block
+    last = block
+  }
+  const head = [eroot, last.cid]
+  const block = await encode({ value: head, codec: dagcbor, hasher })
+  yield block
+}
+
+const decrypt = async function * ({ root, get, key, cache, chunker, hasher }) {
+  const o = { ...await get(root), codec: dagcbor, hasher }
+  const decodedRoot = await decode(o)
+  // console.log('decodedRoot', decodedRoot)
+  const { value: [eroot, tree] } = decodedRoot
+  const rootBlock = await get(eroot) // should I decrypt?
+  const cidset = await load({ cid: tree, get, cache, chunker, codec, hasher })
+  const { result: nodes } = await cidset.getAllEntries()
+  const unwrap = async (eblock) => {
+    const { bytes, cid } = await codec.decrypt({ ...eblock, key }).catch(e => {
+      console.log('ekey', e)
+      throw new Error('bad key: ' + key.toString('hex'))
+    })
+    const block = await createBlock(bytes, cid)
+    return block
+  }
+  const promises = []
+  for (const { cid } of nodes) {
+    if (!rootBlock.cid.equals(cid)) promises.push(get(cid).then(unwrap))
+  }
+  yield * promises
+  yield unwrap(rootBlock)
+}
+
+export {
+  encrypt,
+  decrypt
+}

package/src/db-index.js

@@ -14,21 +14,6 @@ import charwise from 'charwise'
 
 const ALWAYS_REBUILD = false // todo: make false
 
-// const arrayCompare = (a, b) => {
-//   if (Array.isArray(a) && Array.isArray(b)) {
-//     const len = Math.min(a.length, b.length)
-//     for (let i = 0; i < len; i++) {
-//       const comp = simpleCompare(a[i], b[i])
-//       if (comp !== 0) {
-//         return comp
-//       }
-//     }
-//     return simpleCompare(a.length, b.length)
-//   } else {
-//     return simpleCompare(a, b)
-//   }
-// }
-
 const compare = (a, b) => {
   const [aKey, aRef] = a
   const [bKey, bRef] = b
@@ -82,6 +67,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
   changes.forEach(({ key, value, del }) => {
     if (del || !value) return
     mapFn(makeDoc({ key, value }), (k, v) => {
+      if (typeof v === 'undefined' || typeof k === 'undefined') return
       indexEntries.push({
         key: [charwise.encode(k), key],
         value: v
@@ -102,7 +88,7 @@ const indexEntriesForChanges = (changes, mapFn) => {
  *
  */
 export default class DbIndex {
-  constructor (database, mapFn, clock) {
+  constructor (database, mapFn, clock, opts = {}) {
     // console.log('DbIndex constructor', database.constructor.name, typeof mapFn, clock)
     /**
      * The database instance to DbIndex.
@@ -110,7 +96,7 @@ export default class DbIndex {
      */
     this.database = database
     if (!database.indexBlocks) {
-      database.indexBlocks = new TransactionBlockstore(database.name + '.indexes')
+      database.indexBlocks = new TransactionBlockstore(database.name + '.indexes', database.blocks.valet.getKeyMaterial())
     }
     /**
      * The map function to apply to each entry in the database.
@@ -123,6 +109,7 @@ export default class DbIndex {
       this.mapFn = mapFn
       this.mapFnString = mapFn.toString()
     }
+    this.name = opts.name || this.makeName()
     this.indexById = { root: null, cid: null }
     this.indexByKey = { root: null, cid: null }
     this.dbHead = null
@@ -133,45 +120,46 @@ export default class DbIndex {
     }
     this.instanceId = this.database.instanceId + `.DbIndex.${Math.random().toString(36).substring(2, 7)}`
     this.updateIndexPromise = null
-    DbIndex.registerWithDatabase(this, this.database)
+    if (!opts.temporary) { DbIndex.registerWithDatabase(this, this.database) }
+  }
+
+  makeName () {
+    const regex = /\(([^,()]+,\s*[^,()]+|\[[^\]]+\],\s*[^,()]+)\)/g
+    const matches = Array.from(this.mapFnString.matchAll(regex), match => match[1].trim())
+    return matches[1]
   }
 
   static registerWithDatabase (inIndex, database) {
-    // console.log('.reg > in Index', inIndex.instanceId, { live: !!inIndex.mapFn }, inIndex.indexByKey, inIndex.mapFnString)
-    if (database.indexes.has(inIndex.mapFnString)) {
+    if (!database.indexes.has(inIndex.mapFnString)) {
+      database.indexes.set(inIndex.mapFnString, inIndex)
+    } else {
       // merge our inIndex code with the inIndex clock or vice versa
-      // keep the code instance, discard the clock instance
       const existingIndex = database.indexes.get(inIndex.mapFnString)
-      // console.log('.reg - existingIndex', existingIndex.instanceId, { live: !!inIndex.mapFn }, existingIndex.indexByKey)
+      // keep the code instance, discard the clock instance
       if (existingIndex.mapFn) { // this one also has other config
         existingIndex.dbHead = inIndex.dbHead
         existingIndex.indexById.cid = inIndex.indexById.cid
         existingIndex.indexByKey.cid = inIndex.indexByKey.cid
       } else {
-        // console.log('.reg use inIndex with existingIndex clock')
         inIndex.dbHead = existingIndex.dbHead
         inIndex.indexById.cid = existingIndex.indexById.cid
         inIndex.indexByKey.cid = existingIndex.indexByKey.cid
         database.indexes.set(inIndex.mapFnString, inIndex)
       }
-    } else {
-      // console.log('.reg - fresh')
-      database.indexes.set(inIndex.mapFnString, inIndex)
     }
-    // console.log('.reg after', JSON.stringify([...database.indexes.values()].map(i => [i.instanceId, typeof i.mapFn, i.indexByKey, i.indexById])))
   }
 
   toJSON () {
-    const indexJson = { code: this.mapFn?.toString(), clock: { db: null, byId: null, byKey: null } }
+    const indexJson = { name: this.name, code: this.mapFnString, clock: { db: null, byId: null, byKey: null } }
     indexJson.clock.db = this.dbHead?.map(cid => cid.toString())
     indexJson.clock.byId = this.indexById.cid?.toString()
     indexJson.clock.byKey = this.indexByKey.cid?.toString()
     return indexJson
   }
 
-  static fromJSON (database, { code, clock }) {
+  static fromJSON (database, { code, clock, name }) {
     // console.log('DbIndex.fromJSON', database.constructor.name, code, clock)
-    return new DbIndex(database, code, clock)
+    return new DbIndex(database, code, clock, { name })
   }
 
   /**
@@ -188,25 +176,19 @@ export default class DbIndex {
    * @memberof DbIndex
    * @instance
    */
-  async query (query) {
+  async query (query, update = true) {
     // const callId = Math.random().toString(36).substring(2, 7)
-    // if (!root) {
-    // pass a root to query a snapshot
+    // todo pass a root to query a snapshot
     // console.time(callId + '.#updateIndex')
-    await this.#updateIndex(this.database.indexBlocks)
+    update && await this.#updateIndex(this.database.indexBlocks)
     // console.timeEnd(callId + '.#updateIndex')
-
-    // }
     // console.time(callId + '.doIndexQuery')
     // console.log('query', query)
     const response = await doIndexQuery(this.database.indexBlocks, this.indexByKey, query)
     // console.timeEnd(callId + '.doIndexQuery')
-
     return {
       proof: { index: await cidsToProof(response.cids) },
-      // TODO fix this naming upstream in prolly/db-DbIndex?
       rows: response.result.map(({ id, key, row }) => {
-        // console.log('query', id, key, row)
         return ({ id, key: charwise.decode(key), value: row })
       })
     }
@@ -269,6 +251,7 @@ export default class DbIndex {
       this.indexByKey = await bulkIndex(blocks, this.indexByKey, oldIndexEntries.concat(indexEntries), dbIndexOpts)
       this.dbHead = result.clock
     })
+    this.database.notifyExternal('dbIndex')
     // console.timeEnd(callTag + '.doTransaction#updateIndex')
     // console.log(`#updateIndex ${callTag} <`, this.instanceId, this.dbHead?.toString(), this.indexByKey.cid?.toString(), this.indexById.cid?.toString())
   }
@@ -320,16 +303,22 @@ async function loadIndex (blocks, index, indexOpts) {
   return index.root
 }
 
-async function doIndexQuery (blocks, indexByKey, query) {
+async function applyLimit (results, limit) {
+  results.result = results.result.slice(0, limit)
+  return results
+}
+
+async function doIndexQuery (blocks, indexByKey, query = {}) {
   await loadIndex(blocks, indexByKey, dbIndexOpts)
-  if (!query) {
-    const { result, ...all } = await indexByKey.root.getAllEntries()
-    return { result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }
-  } else if (query.range) {
+  if (!indexByKey.root) return { result: [] }
+  if (query.range) {
     const encodedRange = query.range.map((key) => charwise.encode(key))
-    return indexByKey.root.range(...encodedRange)
+    return applyLimit(await indexByKey.root.range(...encodedRange), query.limit)
   } else if (query.key) {
     const encodedKey = charwise.encode(query.key)
     return indexByKey.root.get(encodedKey)
+  } else {
+    const { result, ...all } = await indexByKey.root.getAllEntries()
+    return applyLimit({ result: result.map(({ key: [k, id], value }) => ({ key: k, id, row: value })), ...all }, query.limit)
   }
 }