@fireproof/core 0.5.16 → 0.5.18

package/README.md

@@ -3,7 +3,7 @@
 Fireproof is a realtime database for today's interactive applications. It uses immutable data and distributed protocols 
 to offer a new kind of database that:
 - can be embedded in any page or app, with a flexible data ownership model
-- scales without incurring developer costs, thanks to Filecoin
+- can be hosted on any cloud
 - uses cryptographically verifiable protocols (what plants crave)
 
 Learn more about the [concepts and architecture behind Fireproof](https://fireproof.storage/documentation/how-the-database-engine-works/), or jump to the [quick start](#quick-start) for React and server-side examples.

package/dist/src/blockstore.js

@@ -71,7 +71,7 @@ export class TransactionBlockstore {
             return old;
         if (!this.valet)
             throw new Error('Missing block: ' + key);
-        const got = await this.valet.getBlock(key);
+        const got = await this.valet.getValetBlock(key);
         this.committedBlocks.set(key, got);
         return got;
     }

package/dist/src/clock.js

@@ -223,11 +223,13 @@ export async function findEventsToSync(blocks, head) {
     // console.time(callTag + '.findCommonAncestorWithSortedEvents')
     const { ancestor, sorted } = await findCommonAncestorWithSortedEvents(events, head);
     // console.timeEnd(callTag + '.findCommonAncestorWithSortedEvents')
-    // console.log('sorted', !!ancestor, sorted.length)
+    // console.log('sorted', !!ancestor, sorted)
     // console.time(callTag + '.contains')
     const toSync = ancestor ? await asyncFilter(sorted, async (uks) => !(await contains(events, ancestor, uks.cid))) : sorted;
     // console.timeEnd(callTag + '.contains')
-    // console.log('optimize sorted', !!ancestor, sorted.length - toSync.length)
+    const sortDifference = sorted.length - toSync.length;
+    if (sortDifference / sorted.length > 0.6)
+        console.log('optimize sorted', !!ancestor, sortDifference);
     return { cids: events, events: toSync };
 }
 const asyncFilter = async (arr, predicate) => Promise.all(arr.map(predicate)).then(results => arr.filter((_v, index) => results[index]));

package/dist/src/database.js

@@ -1,6 +1,6 @@
 // @ts-nocheck
 import { visMerkleClock, visMerkleTree, vis, put, get, getAll, eventsSince } from './prolly.js';
-import { doTransaction } from './blockstore.js';
+import { doTransaction, TransactionBlockstore } from './blockstore.js';
 import charwise from 'charwise';
 import { localSet } from './utils.js';
 import { CID } from 'multiformats';
@@ -17,7 +17,6 @@ export const parseCID = cid => (typeof cid === 'string' ? CID.parse(cid) : cid);
  *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
  *  usage guides in the repository README.
  *
- * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
  * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
  * @param {object} [config] - Optional configuration options for the Fireproof instance.
  * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
@@ -28,10 +27,11 @@ export class Database {
     indexes = new Map();
     rootCache = null;
     eventsCache = new Map();
-    constructor(blocks, clock, config = {}) {
-        this.name = config.name;
+    constructor(name, clock, config = {}) {
+        this.name = name;
         this.instanceId = `fp.${this.name}.${Math.random().toString(36).substring(2, 7)}`;
-        this.blocks = blocks;
+        this.blocks = new TransactionBlockstore(name, config.key);
+        this.indexBlocks = new TransactionBlockstore(name + '.indexes', config.key);
         this.clock = clock;
         this.config = config;
     }
@@ -47,6 +47,8 @@ export class Database {
             clock: this.clockToJSON(),
             name: this.name,
             key: this.blocks.valet?.getKeyMaterial(),
+            car: this.blocks.valet?.valetRootCarCid.toString(),
+            indexCar: this.indexBlocks.valet?.valetRootCarCid?.toString(),
             indexes: [...this.indexes.values()].map(index => index.toJSON())
         };
     }
@@ -59,11 +61,14 @@ export class Database {
     clockToJSON(clock = null) {
         return (clock || this.clock).map(cid => cid.toString());
     }
-    hydrate({ clock, name, key }) {
+    hydrate({ clock, name, key, car, indexCar }) {
         this.name = name;
         this.clock = clock;
         this.blocks.valet?.setKeyMaterial(key);
-        this.indexBlocks = null;
+        this.blocks.valet?.setRootCarCid(car); // maybe
+        this.indexBlocks.valet?.setKeyMaterial(key);
+        this.indexBlocks.valet?.setRootCarCid(indexCar); // maybe
+        // this.indexBlocks = null
     }
     maybeSaveClock() {
         if (this.name && this.blocks.valet) {
@@ -98,7 +103,7 @@ export class Database {
         let rows, dataCIDs, clockCIDs;
         // if (!aClock) aClock = []
         if (aClock && aClock.length > 0) {
-            aClock = aClock.map((cid) => cid.toString());
+            aClock = aClock.map(cid => cid.toString());
             const eventKey = JSON.stringify([...this.clockToJSON(aClock), ...this.clockToJSON()]);
             let resp;
             if (this.eventsCache.has(eventKey)) {
@@ -259,6 +264,7 @@ export class Database {
      * @returns {Promise<{ proof:{}, id: string, clock: CID[] }>} - The result of adding the event to storage
      */
     async putToProllyTree(decodedEvent, clock = null) {
+        // console.log('putToProllyTree', decodedEvent)
         const event = encodeEvent(decodedEvent);
         if (clock && JSON.stringify(this.clockToJSON(clock)) !== JSON.stringify(this.clockToJSON())) {
             // console.log('this.clock', this.clockToJSON())

package/dist/src/db-index.js

@@ -11,7 +11,7 @@ import { makeGetBlock, visMerkleTree } from './prolly.js';
 import { Database, cidsToProof } from './database.js';
 import * as codec from '@ipld/dag-cbor';
 // import { create as createBlock } from 'multiformats/block'
-import { TransactionBlockstore, doTransaction } from './blockstore.js';
+import { doTransaction } from './blockstore.js';
 // @ts-ignore
 import charwise from 'charwise';
 const ALWAYS_REBUILD = false; // todo: remove
@@ -63,23 +63,23 @@ const makeDoc = ({ key, value }) => ({ _id: key, ...value });
  */
 const indexEntriesForChanges = (changes, mapFn) => {
     const indexEntries = [];
-    changes.forEach(({ key, value, del }) => {
+    changes.forEach(({ key: _id, value, del }) => {
         // key is _id, value is the document
         if (del || !value)
             return;
         let mapCalled = false;
-        const mapReturn = mapFn(makeDoc({ key, value }), (k, v) => {
+        const mapReturn = mapFn(makeDoc({ key: _id, value }), (k, v) => {
             mapCalled = true;
             if (typeof k === 'undefined')
                 return;
             indexEntries.push({
-                key: [charwise.encode(k), key],
+                key: [charwise.encode(k), _id],
                 value: v || null
             });
         });
         if (!mapCalled && mapReturn) {
             indexEntries.push({
-                key: [charwise.encode(mapReturn), key],
+                key: [charwise.encode(mapReturn), _id],
                 value: null
             });
         }
@@ -102,9 +102,6 @@ export class DbIndex {
      */
     constructor(database, name, mapFn, clock = null, opts = {}) {
         this.database = database;
-        if (!database.indexBlocks) {
-            database.indexBlocks = new TransactionBlockstore(database?.name + '.indexes', database.blocks.valet?.getKeyMaterial());
-        }
         if (typeof name === 'function') {
             // app is using deprecated API, remove in 0.7
             opts = clock || {};
@@ -249,7 +246,16 @@ export class DbIndex {
             return { result: [] };
         if (query.includeDocs === undefined)
             query.includeDocs = this.includeDocsDefault;
-        if (query.range) {
+        if (query.prefix) {
+            // ensure prefix is an array
+            if (!Array.isArray(query.prefix))
+                query.prefix = [query.prefix];
+            const start = [...query.prefix, NaN];
+            const end = [...query.prefix, Infinity];
+            const prefixRange = [start, end].map(key => charwise.encode(key));
+            return await this.applyQuery(await this.indexByKey.root.range(...prefixRange), query);
+        }
+        else if (query.range) {
             const encodedRange = query.range.map(key => charwise.encode(key));
             return await this.applyQuery(await this.indexByKey.root.range(...encodedRange), query);
         }

package/dist/src/fireproof.d.ts

@@ -1,5 +1,195 @@
 import * as multiformats from 'multiformats';
-import { CID } from 'multiformats';
+import { Link, CID } from 'multiformats';
+import * as multiformats_cid from 'multiformats/cid';
+
+type AnyLink = Link<unknown, number, number, 1|0>
+
+declare class Valet {
+    constructor(name: string, keyMaterial: any);
+    idb: any;
+    name: any;
+    uploadQueue: any;
+    alreadyEnqueued: Set<any>;
+    keyMaterial: any;
+    keyId: string;
+    valetRoot: any;
+    valetRootCid: any;
+    valetRootCarCid: any;
+    valetCidBlocks: VMemoryBlockstore;
+    instanceId: string;
+    /**
+     * Function installed by the database to upload car files
+     * @type {null|function(string, Uint8Array):Promise<void>}
+     */
+    uploadFunction: null | ((arg0: string, arg1: Uint8Array) => Promise<void>);
+    getKeyMaterial(): any;
+    setKeyMaterial(km: any): void;
+    /**
+     * Group the blocks into a car and write it to the valet.
+     * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
+     * @param {Set<string>} cids
+     * @returns {Promise<void>}
+     * @memberof Valet
+     */
+    writeTransaction(innerBlockstore: InnerBlockstore, cids: Set<string>): Promise<void>;
+    withDB: (dbWorkFun: any) => Promise<any>;
+    /**
+     * Iterate over all blocks in the store.
+     *
+     * @yields {{cid: string, value: Uint8Array}}
+     * @returns {AsyncGenerator<any, any, any>}
+     */
+    cids(): AsyncGenerator<any, any, any>;
+    setRootCarCid(cid: any): void;
+    getCarCIDForCID(cid: any): Promise<{
+        result: any;
+    }>;
+    OLDgetCarCIDForCID(cid: any): Promise<{
+        result: any;
+    }>;
+    getCombinedReader(carCid: any): Promise<{
+        root: any;
+        put: (cid: any, bytes: any) => Promise<void>;
+        get: (cid: any) => Promise<any>;
+    }>;
+    /**
+     *
+     * @param {string} carCid
+     * @param {*} value
+     */
+    parkCar(carCid: string, value: any, cids: any): Promise<void>;
+    remoteBlockFunction: any;
+    getCarReader(carCid: any): Promise<{
+        root: any;
+        get: (dataCID: any) => Promise<any>;
+    }>;
+    getValetBlock(dataCID: any): Promise<any>;
+}
+declare class VMemoryBlockstore {
+    /** @type {Map<string, Uint8Array>} */
+    blocks: Map<string, Uint8Array>;
+    instanceId: string;
+    get(cid: any): Promise<{
+        cid: any;
+        bytes: Uint8Array;
+    }>;
+    /**
+     * @param {import('../src/link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    put(cid: AnyLink, bytes: Uint8Array): Promise<void>;
+    entries(): Generator<{
+        cid: multiformats_cid.Link<unknown, number, number, multiformats_cid.Version>;
+        bytes: Uint8Array;
+    }, void, unknown>;
+}
+
+/**
+ * @typedef {{ get: (link: import('../src/link').AnyLink) => Promise<AnyBlock | undefined> }} BlockFetcher
+ */
+/**
+ * @typedef {Object} AnyBlock
+ * @property {import('./link').AnyLink} cid - The CID of the block
+ * @property {Uint8Array} bytes - The block's data
+ *
+ * @typedef {Object} Blockstore
+ * @property {function(import('./link').AnyLink): Promise<AnyBlock|undefined>} get - A function to retrieve a block by CID
+ * @property {function(import('./link').AnyLink, Uint8Array): Promise<void>} put - A function to store a block's data and CID
+ *
+ * A blockstore that caches writes to a transaction and only persists them when committed.
+*/
+declare class TransactionBlockstore {
+    constructor(name: any, encryptionKey: any);
+    /** @type {Map<string, Uint8Array>} */
+    committedBlocks: Map<string, Uint8Array>;
+    /** @type {Valet} */
+    valet: Valet;
+    instanceId: string;
+    inflightTransactions: Set<any>;
+    syncs: Set<any>;
+    remoteBlockFunction: any;
+    /**
+     * Get a block from the store.
+     *
+     * @param {import('./link').AnyLink} cid
+     * @returns {Promise<AnyBlock | undefined>}
+     */
+    get(cid: AnyLink): Promise<AnyBlock | undefined>;
+    transactionsGet(key: any): Promise<any>;
+    committedGet(key: any): Promise<any>;
+    clearCommittedCache(): Promise<void>;
+    networkGet(key: any): Promise<any>;
+    /**
+     * Add a block to the store. Usually bound to a transaction by a closure.
+     * It sets the lastCid property to the CID of the block that was put.
+     * This is used by the transaction as the head of the car when written to the valet.
+     * We don't have to worry about which transaction we are when we are here because
+     * we are the transactionBlockstore.
+     *
+     * @param {import('./link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    put(cid: AnyLink, bytes: Uint8Array): void;
+    /**
+     * Iterate over all blocks in the store.
+     *
+     * @yields {{cid: string, bytes: Uint8Array}}
+     * @returns {AsyncGenerator<any, any, any>}
+     */
+    entries(): AsyncGenerator<any, any, any>;
+    /**
+     * Begin a transaction. Ensures the uncommited blocks are empty at the begining.
+     * Returns the blocks to read and write during the transaction.
+     * @returns {InnerBlockstore}
+     * @memberof TransactionBlockstore
+     */
+    begin(label?: string): InnerBlockstore;
+    /**
+     * Commit the transaction. Writes the blocks to the store.
+     * @returns {Promise<void>}
+     * @memberof TransactionBlockstore
+     */
+    commit(innerBlockstore: any, doSync?: boolean): Promise<void>;
+    doCommit: (innerBlockstore: any) => Promise<void>;
+    /**
+     * Retire the transaction. Clears the uncommited blocks.
+     * @returns {void}
+     * @memberof TransactionBlockstore
+     */
+    retire(innerBlockstore: any): void;
+}
+declare class InnerBlockstore {
+    constructor(label: any, parentBlockstore: any);
+    /** @type {Map<string, Uint8Array>} */
+    blocks: Map<string, Uint8Array>;
+    lastCid: any;
+    label: string;
+    parentBlockstore: any;
+    /**
+     * @param {import('./link').AnyLink} cid
+     * @returns {Promise<AnyBlock | undefined>}
+     */
+    get(cid: AnyLink): Promise<AnyBlock | undefined>;
+    /**
+     * @param {import('./link').AnyLink} cid
+     * @param {Uint8Array} bytes
+     */
+    put(cid: AnyLink, bytes: Uint8Array): Promise<void>;
+    entries(): Generator<{
+        cid: multiformats.Link<unknown, number, number, multiformats.Version>;
+        bytes: Uint8Array;
+    }, void, unknown>;
+}
+type AnyBlock = {
+    /**
+     * - The CID of the block
+     */
+    cid: AnyLink;
+    /**
+     * - The block's data
+     */
+    bytes: Uint8Array;
+};
 
 /**
  * @class Fireproof
@@ -7,21 +197,21 @@ import { CID } from 'multiformats';
  *  This is the main class for saving and loading JSON and other documents with the database. You can find additional examples and
  *  usage guides in the repository README.
  *
- * @param {import('./blockstore.js').TransactionBlockstore} blocks - The block storage instance to use documents and indexes
  * @param {CID[]} clock - The Merkle clock head to use for the Fireproof instance.
  * @param {object} [config] - Optional configuration options for the Fireproof instance.
  * @param {object} [authCtx] - Optional authorization context object to use for any authentication checks.
  *
  */
 declare class Database {
-    constructor(blocks: any, clock: any, config?: {});
+    constructor(name: any, clock: any, config?: {});
     listeners: Set<any>;
     indexes: Map<any, any>;
     rootCache: any;
     eventsCache: Map<any, any>;
     name: any;
     instanceId: string;
-    blocks: any;
+    blocks: TransactionBlockstore;
+    indexBlocks: TransactionBlockstore;
     clock: any;
     config: {};
     /**
@@ -38,12 +228,13 @@ declare class Database {
      * @instance
      */
     clockToJSON(clock?: any): string[];
-    hydrate({ clock, name, key }: {
+    hydrate({ clock, name, key, car, indexCar }: {
         clock: any;
         name: any;
         key: any;
+        car: any;
+        indexCar: any;
     }): void;
-    indexBlocks: any;
     maybeSaveClock(): void;
     /**
      * Triggers a notification to all listeners
@@ -405,6 +596,7 @@ declare class Fireproof {
      * @returns {Database} - a new Fireproof instance
      */
     static storage: (name?: any, opts?: {}) => Database;
+    static fromConfig(name: any, existingConfig: any, opts?: {}): any;
     static fromJSON(json: any, database: any): any;
     static snapshot(database: any, clock: any): any;
     static zoom(database: any, clock: any): Promise<any>;