@fireproof/core 0.10.2-dev → 0.10.4-dev

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.10.2-dev",
+  "version": "0.10.4-dev",
   "description": "Immutable embedded distributed database for the web",
   "main": "dist/fireproof.cjs.js",
   "module": "dist/fireproof.esm.js",
@@ -13,6 +13,7 @@
   },
   "browser": "./dist/fireproof.browser.js",
   "files": [
+    "src",
     "dist/fireproof.*"
   ],
   "type": "module",
@@ -68,7 +69,6 @@
     "@ipld/dag-cbor": "^9.0.3",
     "async": "^3.2.4",
     "idb": "^7.1.1",
-    "multiformats": "^12.0.1",
-    "prolly-trees": "^1.0.4"
+    "multiformats": "^12.0.1"
   }
 }

package/src/crdt-helpers.ts

@@ -0,0 +1,89 @@
+import { Link } from 'multiformats'
+import { create, encode, decode } from 'multiformats/block'
+import { sha256 as hasher } from 'multiformats/hashes/sha2'
+import * as codec from '@ipld/dag-cbor'
+import { put, get, EventData } from '@alanshaw/pail/crdt'
+import { EventFetcher } from '@alanshaw/pail/clock'
+
+import { TransactionBlockstore as Blockstore, Transaction } from './transaction'
+import { DocUpdate, ClockHead, BlockFetcher, AnyLink, DocValue, BulkResult } from './types'
+
+export function makeGetBlock(blocks: BlockFetcher) {
+  return async (address: Link) => {
+    const block = await blocks.get(address)
+    if (!block) throw new Error(`Missing block ${address.toString()}`)
+    const { cid, bytes } = block
+    return create({ cid, bytes, hasher, codec })
+  }
+}
+
+export async function applyBulkUpdateToCrdt(
+  tblocks: Transaction,
+  head: ClockHead,
+  updates: DocUpdate[],
+  options?: object
+): Promise<BulkResult> {
+  for (const update of updates) {
+    const link = await makeLinkForDoc(tblocks, update)
+    const result = await put(tblocks, head, update.key, link, options)
+    for (const { cid, bytes } of [...result.additions, ...result.removals, result.event]) {
+      tblocks.putSync(cid, bytes)
+    }
+    head = result.head
+  }
+  return { head }
+}
+
+async function makeLinkForDoc(blocks: Transaction, update: DocUpdate): Promise<AnyLink> {
+  let value: DocValue
+  if (update.del) {
+    value = { del: true }
+  } else {
+    value = { doc: update.value }
+  }
+  const block = await encode({ value, hasher, codec })
+  blocks.putSync(block.cid, block.bytes)
+  return block.cid
+}
+
+export async function getValueFromCrdt(blocks: Blockstore, head: ClockHead, key: string): Promise<DocValue> {
+  const link = await get(blocks, head, key)
+  if (!link) throw new Error(`Missing key ${key}`)
+  return await getValueFromLink(blocks, link)
+}
+
+export async function getValueFromLink(blocks: Blockstore, link: AnyLink): Promise<DocValue> {
+  const block = await blocks.get(link)
+  if (!block) throw new Error(`Missing block ${link.toString()}`)
+  const { value } = (await decode({ bytes: block.bytes, hasher, codec })) as { value: DocValue }
+  return value
+}
+
+export async function clockChangesSince(
+  blocks: Blockstore,
+  _head: ClockHead,
+  _since: ClockHead
+): Promise<{ result: DocUpdate[] }> {
+  const eventsFetcher = new EventFetcher<EventData>(blocks)
+  const updates = await gatherUpdates(blocks, eventsFetcher, _head, _since)
+  return { result: updates.reverse() }
+}
+
+async function gatherUpdates(blocks: Blockstore, eventsFetcher: EventFetcher<EventData>, head: ClockHead, since: ClockHead, updates: DocUpdate[] = []): Promise<DocUpdate[]> {
+  for (const link of since) {
+    if (head.includes(link)) {
+      throw new Error('found since in head, this is good, remove this error ' + updates.length)
+      return updates
+    }
+  }
+  for (const link of head) {
+    const { value: event } = await eventsFetcher.get(link)
+    const { key, value } = event.data
+    const docValue = await getValueFromLink(blocks, value)
+    updates.push({ key, value: docValue.doc, del: docValue.del })
+    if (event.parents) {
+      updates = await gatherUpdates(blocks, eventsFetcher, event.parents, since, updates)
+    }
+  }
+  return updates
+}

package/src/crdt.ts

@@ -0,0 +1,45 @@
+import { TransactionBlockstore as Blockstore } from './transaction'
+import { DocUpdate, BulkResult, ClockHead } from './types'
+import { clockChangesSince, applyBulkUpdateToCrdt, getValueFromCrdt } from './crdt-helpers'
+
+export class CRDT {
+  name: string | null
+  ready: Promise<void>
+
+  private _blocks: Blockstore
+  private _head: ClockHead
+
+  constructor(name?: string, blocks?: Blockstore) {
+    this.name = name || null
+    this._blocks = blocks || new Blockstore(name)
+    this._head = []
+    this.ready = this._blocks.ready.then(({ head }: { head: ClockHead }) => {
+      this._head = head // todo multi head support here
+    })
+  }
+
+  async bulk(updates: DocUpdate[], options?: object): Promise<BulkResult> {
+    await this.ready
+    const tResult: BulkResult = await this._blocks.transaction(async tblocks => {
+      const { head } = await applyBulkUpdateToCrdt(tblocks, this._head, updates, options)
+      this._head = head // we want multi head support here if allowing calls to bulk in parallel
+      return { head }
+    })
+    return tResult
+  }
+
+  // async root(): Promise<any> {
+  // async eventsSince(since: EventLink<T>): Promise<{clockCIDs: CIDCounter, result: T[]}> {
+  // async getAll(rootCache: any = null): Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: T[]}> {
+
+  async get(key: string) {
+    await this.ready
+    const result = await getValueFromCrdt(this._blocks, this._head, key)
+    if (result.del) return null
+    return result
+  }
+
+  async changes(since: ClockHead) {
+    return await clockChangesSince(this._blocks, this._head, since)
+  }
+}

package/src/database.ts

@@ -0,0 +1,61 @@
+// @ts-ignore
+import cargoQueue from 'async/cargoQueue'
+import { CRDT } from './crdt'
+import { Doc, BulkResult, DocUpdate, DbResponse, ClockHead } from './types'
+
+export class Database {
+  name: string
+  config: object
+  _crdt: CRDT
+  _writeQueue: any
+  constructor(name: string, config = {}) {
+    this.name = name
+    this.config = config
+    this._crdt = new CRDT(name)
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+    this._writeQueue = cargoQueue(async (updates: DocUpdate[]) => {
+      return await this._crdt.bulk(updates)
+    })
+  }
+
+  async put(doc: Doc): Promise<DbResponse> {
+    const { _id, ...value } = doc
+    return await new Promise<DbResponse>((resolve, reject) => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      this._writeQueue.push({ key: _id, value }, function (err: Error | null, result?: BulkResult) {
+        if (err) reject(err)
+        resolve({ id: doc._id, clock: result?.head } as DbResponse)
+      })
+    })
+  }
+
+  async get(id: string): Promise<Doc> {
+    const got = await this._crdt.get(id).catch(e => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+      e.message = `Not found: ${id} - ` + e.message
+      throw e
+    })
+    if (!got) throw new Error(`Not found: ${id}`)
+    const { doc } = got
+    return { _id: id, ...doc }
+  }
+
+  async del(id: string): Promise<DbResponse> {
+    return await new Promise<DbResponse>((resolve, reject) => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      this._writeQueue.push({ key: id, del: true }, function (err: Error | null, result?: BulkResult) {
+        if (err) reject(err)
+        resolve({ id, clock: result?.head } as DbResponse)
+      })
+    })
+  }
+
+  async changes(since: ClockHead): Promise<{ rows: { key: string; value: Doc }[] }> {
+    const { result } = await this._crdt.changes(since)
+    const rows = result.map(({ key, value }) => ({
+      key,
+      value: { _id: key, ...value } as Doc
+    }))
+    return { rows }
+  }
+}

package/src/fireproof.ts

@@ -0,0 +1,6 @@
+import { Database } from './database'
+export class Fireproof {
+  static storage(name: string): Database {
+    return new Database(name)
+  }
+}

package/src/loader-helpers.ts

@@ -0,0 +1,53 @@
+import { BlockView, CID } from 'multiformats'
+import { Block, encode, decode } from 'multiformats/block'
+import { sha256 as hasher } from 'multiformats/hashes/sha2'
+import * as raw from 'multiformats/codecs/raw'
+import * as CBW from '@ipld/car/buffer-writer'
+import * as codec from '@ipld/dag-cbor'
+import { CarReader } from '@ipld/car'
+
+import { Transaction } from './transaction'
+import { AnyBlock, BulkResult, ClockHead, AnyLink } from './types'
+
+export async function makeCarFile(
+  t: Transaction,
+  { head }: BulkResult,
+  cars: AnyLink[]
+): Promise<BlockView<unknown, number, number, 1>> {
+  if (!head) throw new Error('no head')
+  // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+  const fpCarHeaderBlock = (await encode({
+    value: { fp: { head, cars } },
+    hasher,
+    codec
+  })) as AnyBlock
+  await t.put(fpCarHeaderBlock.cid, fpCarHeaderBlock.bytes)
+
+  let size = 0
+  const headerSize = CBW.headerLength({ roots: [fpCarHeaderBlock.cid as CID<unknown, number, number, 1>] })
+  size += headerSize
+  for (const { cid, bytes } of t.entries()) {
+    size += CBW.blockLength({ cid, bytes } as Block<unknown, number, number, 1>)
+  }
+  const buffer = new Uint8Array(size)
+  const writer = CBW.createWriter(buffer, { headerSize })
+
+  writer.addRoot(fpCarHeaderBlock.cid as CID<unknown, number, number, 1>)
+
+  for (const { cid, bytes } of t.entries()) {
+    writer.write({ cid, bytes } as Block<unknown, number, number, 1>)
+  }
+  writer.close()
+  return await encode({ value: writer.bytes, hasher, codec: raw })
+}
+
+export async function parseCarFile(reader: CarReader): Promise<{ head: ClockHead; cars: AnyLink[] }> {
+  const roots = await reader.getRoots()
+  const header = await reader.get(roots[0])
+  if (!header) throw new Error('missing header block')
+  const got = await decode({ bytes: header.bytes, hasher, codec })
+  const {
+    fp: { head, cars }
+  } = got.value as { fp: { head: ClockHead; cars: AnyLink[] } }
+  return { head, cars }
+}

package/src/loader.ts

@@ -0,0 +1,66 @@
+import { CarReader } from '@ipld/car'
+
+// import { CarStoreFS, HeaderStoreFS } from './store-fs'
+import { CarStoreIDB as CarStore, HeaderStoreLS as HeaderStore } from './store-browser'
+import { makeCarFile, parseCarFile } from './loader-helpers'
+import { Transaction } from './transaction'
+import { AnyBlock, AnyLink, BulkResult, ClockHead } from './types'
+import { CID } from 'multiformats'
+
+export class Loader {
+  name: string
+  headerStore: HeaderStore
+  carStore: CarStore
+  carLog: AnyLink[] = []
+  carsReaders: Map<string, CarReader> = new Map()
+  ready: Promise<{ head: ClockHead}> // todo this will be a map of headers by branch name
+  constructor(name: string) {
+    this.name = name
+    this.headerStore = new HeaderStore(name)
+    this.carStore = new CarStore(name)
+    // todo config with multiple branches
+    this.ready = this.headerStore.load('main').then(async header => {
+      if (!header) return { head: [] }
+      const car = await this.carStore.load(header.car)
+      return await this.ingestCarHead(header.car, car)
+    })
+  }
+
+  async commit(t: Transaction, done: BulkResult): Promise<AnyLink> {
+    const car = await makeCarFile(t, done, this.carLog)
+    await this.carStore.save(car)
+    this.carLog.push(car.cid)
+    await this.headerStore.save(car.cid)
+    return car.cid
+  }
+
+  async loadCar(cid: AnyLink): Promise<CarReader> {
+    if (this.carsReaders.has(cid.toString())) return this.carsReaders.get(cid.toString()) as CarReader
+    const car = await this.carStore.load(cid)
+    if (!car) throw new Error(`missing car file ${cid.toString()}`)
+    const reader = await CarReader.fromBytes(car.bytes)
+    this.carsReaders.set(cid.toString(), reader)
+    return reader
+  }
+
+  async ingestCarHead(cid: AnyLink, car: AnyBlock): Promise<{ head: ClockHead, cars: AnyLink[]}> {
+    const reader = await CarReader.fromBytes(car.bytes)
+    this.carsReaders.set(cid.toString(), reader)
+    const { head, cars } = await parseCarFile(reader)
+    await this.getMoreReaders(cars)
+    return { head, cars }
+  }
+
+  async getMoreReaders(cids: AnyLink[]) {
+    for (const cid of cids) {
+      await this.loadCar(cid)
+    }
+  }
+
+  async getBlock(cid: CID): Promise<AnyBlock | undefined> {
+    for (const [, reader] of [...this.carsReaders].reverse()) { // reverse is faster
+      const block = await reader.get(cid)
+      if (block) return block
+    }
+  }
+}

package/src/store-browser.ts

@@ -0,0 +1,78 @@
+import { openDB, IDBPDatabase } from 'idb'
+
+import { AnyBlock, AnyLink } from './types'
+import { CarStore, HeaderStore, StoredHeader } from './store'
+
+export const FORMAT = '0.9'
+
+export class CarStoreIDB extends CarStore {
+  keyId: string = 'public'
+  // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents
+  idb: IDBPDatabase<unknown> | null = null
+  name: string = 'default'
+  async withDB(dbWorkFun: (arg0: any) => any) {
+    if (!this.idb) {
+      const dbName = `fp.${FORMAT}.${this.keyId}.${this.name}.valet`
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
+      this.idb = await openDB(dbName, 1, {
+        upgrade(db): void {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+          db.createObjectStore('cars')
+        }
+      })
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return await dbWorkFun(this.idb)
+  }
+
+  async load(cid: AnyLink): Promise<AnyBlock> {
+    console.log('loading', cid.toString())
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return await this.withDB(async (db: IDBPDatabase<unknown>) => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      const tx = db.transaction(['cars'], 'readonly')
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      const bytes = (await tx.objectStore('cars').get(cid.toString())) as Uint8Array
+      if (!bytes) throw new Error(`missing idb block ${cid.toString()}`)
+      console.log('loaded', cid.toString())
+      return { cid, bytes }
+    })
+  }
+
+  async save(car: AnyBlock): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    return await this.withDB(async (db: IDBPDatabase<unknown>) => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      const tx = db.transaction(['cars'], 'readwrite')
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
+      await tx.objectStore('cars').put(car.bytes, car.cid.toString())
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access
+      return await tx.done
+    })
+  }
+}
+
+export class HeaderStoreLS extends HeaderStore {
+  keyId: string = 'public'
+  name: string = 'default'
+
+  headerKey(branch: string) {
+    return `fp.${FORMAT}.${this.keyId}.${this.name}.${branch}`
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async load(branch: string = 'main'): Promise<StoredHeader | null> {
+    try {
+      const bytes = localStorage.getItem(this.headerKey(branch))
+      return bytes ? this.parseHeader(bytes.toString()) : null
+    } catch (e) {}
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async save(carCid: AnyLink, branch: string = 'main') {
+    try {
+      const headerKey = this.headerKey(branch)
+      return localStorage.setItem(headerKey, this.makeHeader(carCid))
+    } catch (e) {}
+  }
+}

package/src/store-fs.ts

@@ -0,0 +1,51 @@
+import { join, dirname } from 'node:path'
+import { homedir } from 'node:os'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+
+import { AnyBlock, AnyLink } from './types'
+import { HeaderStore, CarStore, StoredHeader } from './store'
+
+export const FORMAT = '0.9'
+
+const encoder = new TextEncoder()
+
+export const defaultConfig = {
+  dataDir: join(homedir(), '.fireproof', 'v' + FORMAT)
+}
+
+export class HeaderStoreFS extends HeaderStore {
+  async load(branch?: string): Promise<StoredHeader|null> {
+    branch = branch || 'main'
+    const filepath = join(defaultConfig.dataDir, this.name, branch + '.json')
+    const bytes = await readFile(filepath).catch((e: Error & { code: string}) => {
+      if (e.code === 'ENOENT') return null
+      throw e
+    })
+    return bytes ? this.parseHeader(bytes.toString()) : null
+  }
+
+  async save(carCid: AnyLink, branch?: string) {
+    branch = branch || 'main'
+    const filepath = join(defaultConfig.dataDir, this.name, branch + '.json')
+    const bytes = this.makeHeader(carCid)
+    await writePathFile(filepath, encoder.encode(bytes))
+  }
+}
+
+export class CarStoreFS extends CarStore {
+  async save(car: AnyBlock): Promise<void> {
+    const filepath = join(defaultConfig.dataDir, this.name, car.cid.toString() + '.car')
+    await writePathFile(filepath, car.bytes)
+  }
+
+  async load(cid: AnyLink): Promise<AnyBlock> {
+    const filepath = join(defaultConfig.dataDir, this.name, cid.toString() + '.car')
+    const bytes = await readFile(filepath)
+    return { cid, bytes: new Uint8Array(bytes) }
+  }
+}
+
+async function writePathFile(path: string, data: Uint8Array) {
+  await mkdir(dirname(path), { recursive: true })
+  return await writeFile(path, data)
+}

package/src/store.ts

@@ -0,0 +1,32 @@
+import { parse } from 'multiformats/link'
+import { AnyLink } from './types'
+
+export class StoredHeader {
+  car: AnyLink
+  constructor(jsonHeader: { car: string }) {
+    this.car = parse(jsonHeader.car)
+  }
+}
+
+export class HeaderStore {
+  name: string
+  constructor(name: string) {
+    this.name = name
+  }
+
+  makeHeader(car: AnyLink): string {
+    return JSON.stringify({ car: car.toString() })
+  }
+
+  parseHeader(headerData: string) {
+    const header = JSON.parse(headerData) as { car: string }
+    return new StoredHeader(header)
+  }
+}
+
+export class CarStore {
+  name: string
+  constructor(name: string) {
+    this.name = name
+  }
+}

package/src/transaction.ts

@@ -0,0 +1,68 @@
+import { MemoryBlockstore } from '@alanshaw/pail/block'
+import { BlockFetcher, AnyBlock, AnyLink, BulkResult, ClockHead } from './types'
+import { Loader } from './loader'
+import { CID } from 'multiformats'
+
+/** forked from
+ * https://github.com/alanshaw/pail/blob/main/src/block.js
+ * thanks Alan
+**/
+
+export class Transaction extends MemoryBlockstore {
+  constructor(private parent: BlockFetcher) {
+    super()
+    this.parent = parent
+  }
+
+  async get(cid: AnyLink): Promise<AnyBlock | undefined> {
+    return this.parent.get(cid)
+  }
+
+  async superGet(cid: AnyLink): Promise<AnyBlock | undefined> {
+    return super.get(cid)
+  }
+}
+
+export class TransactionBlockstore implements BlockFetcher {
+  name: string | null = null
+  ready: Promise<{ head: ClockHead }> // todo this will be a map of headers by branch name
+
+  private transactions: Set<Transaction> = new Set()
+  private loader: Loader | null = null
+
+  constructor(name?: string, loader?: Loader) {
+    if (name) {
+      this.name = name
+      this.loader = loader || new Loader(name)
+      this.ready = this.loader.ready
+    } else {
+      this.ready = Promise.resolve({ head: [] })
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async put() {
+    throw new Error('use a transaction to put')
+  }
+
+  async get(cid: AnyLink): Promise<AnyBlock | undefined> {
+    for (const f of this.transactions) {
+      const v = await f.superGet(cid)
+      if (v) return v
+    }
+    if (!this.loader) return
+    return await this.loader.getBlock(cid as CID)
+  }
+
+  async transaction(fn: (t: Transaction) => Promise<BulkResult>) {
+    const t = new Transaction(this)
+    this.transactions.add(t)
+    const done: BulkResult = await fn(t)
+    if (done) { return { ...done, car: await this.commit(t, done) } }
+    return done
+  }
+
+  async commit(t: Transaction, done: BulkResult): Promise<AnyLink | undefined> {
+    return await this.loader?.commit(t, done)
+  }
+}

package/src/types.d.ts

@@ -0,0 +1,38 @@
+import { Link } from 'multiformats'
+import { EventLink } from '@alanshaw/pail/clock'
+import { EventData } from '@alanshaw/pail/crdt'
+
+export type ClockHead = EventLink<EventData>[]
+
+export type BulkResult = {
+  head: ClockHead
+  car?: AnyLink
+}
+
+type DocBody = {
+  [key: string]: any
+}
+
+export type Doc = DocBody & {
+  _id: string
+}
+
+export type DocUpdate = {
+  key: string
+  value?: DocBody
+  del?: boolean
+}
+
+export type DocValue = {
+  doc?: DocBody
+  del?: boolean
+}
+
+export type AnyLink = Link<unknown, number, number, 1 | 0>
+export type AnyBlock = { cid: AnyLink; bytes: Uint8Array }
+export type BlockFetcher = { get: (link: AnyLink) => Promise<AnyBlock | undefined> }
+
+export type DbResponse = {
+  id: string
+  clock: ClockHead
+}