@fireproof/core 0.8.0 → 0.10.1-dev

package/dist/storage/filesystem.js

@@ -1,67 +0,0 @@
-// import { mkdir, writeFile } from 'fs/promises'
-import { join, dirname } from 'path';
-import { homedir } from 'os';
-import { Base } from './base.js';
-// import { readFileSync } from 'node:fs'
-// const { readFileSync } = require('fs')
-import fs from 'fs';
-const readFileSync = fs.readFileSync;
-export const defaultConfig = {
-    dataDir: join(homedir(), '.fireproof', 'v' + Base.format)
-};
-export class Filesystem extends Base {
-    constructor(name, config = {}) {
-        const mergedConfig = Object.assign({}, defaultConfig, config);
-        // console.log('Filesystem', name, mergedConfig, header)
-        super(name, mergedConfig);
-    }
-    async writeCars(cars) {
-        if (this.config.readonly)
-            return;
-        const writes = [];
-        for (const { cid, bytes } of cars) {
-            const carFilename = join(this.config.dataDir, this.name, `${cid.toString()}.car`);
-            // console.log('writeCars', carFilename)
-            writes.push(writeSync(carFilename, bytes));
-        }
-        await Promise.all(writes);
-    }
-    async readCar(carCid) {
-        const carFilename = join(this.config.dataDir, this.name, `${carCid.toString()}.car`);
-        const got = readFileSync(carFilename);
-        // console.log('readCar', carFilename, got.constructor.name)
-        return got;
-    }
-    loadHeader(branch = 'main') {
-        const header = loadSync(this.headerFilename(branch));
-        // console.log('fs getHeader', this.headerFilename(), header, typeof header)
-        if (!header)
-            return null;
-        return JSON.parse(header);
-    }
-    async writeHeader(branch, header) {
-        // console.log('saveHeader fs', header)
-        if (this.config.readonly)
-            return;
-        // console.log('writeHeader fs', branch, pHeader)
-        await writeSync(this.headerFilename(branch), header);
-    }
-    headerFilename(branch = 'main') {
-        // console.log('headerFilename', this.config.dataDir, this.name)
-        return join(this.config.dataDir, this.name, branch + '.json');
-    }
-}
-function loadSync(filename) {
-    try {
-        return readFileSync(filename, 'utf8').toString();
-    }
-    catch (error) {
-        // console.log('error', error)
-        return null;
-    }
-}
-async function writeSync(fullpath, stringValue) {
-    await fs.promises.mkdir(dirname(fullpath), { recursive: true });
-    // writeFileSync(fullpath, stringValue)
-    await fs.promises.writeFile(fullpath, stringValue);
-}

package/dist/storage/rest.js

@@ -1,57 +0,0 @@
-import fetch from 'cross-fetch';
-import { Base } from './base.js';
-const defaultConfig = {
-    url: 'http://localhost:4000/v' + Base.format
-};
-export class Rest extends Base {
-    constructor(name, config = {}) {
-        super(name, Object.assign({}, defaultConfig, config));
-        // console.log('Rest', name, config)
-    }
-    headerURL(branch = 'main') {
-        return `${this.config.url}/${branch}.json`;
-    }
-    async writeCars(cars) {
-        if (this.config.readonly)
-            return;
-        for (const { cid, bytes } of cars) {
-            const carURL = `${this.config.url}/${cid.toString()}.car`;
-            const response = await fetch(carURL, {
-                method: 'PUT',
-                body: bytes,
-                headers: { 'Content-Type': 'application/car' }
-            });
-            if (!response.ok)
-                throw new Error(`An error occurred: ${response.statusText}`);
-        }
-    }
-    async readCar(carCid) {
-        const carURL = `${this.config.url}/${carCid.toString()}.car`;
-        const response = await fetch(carURL);
-        if (!response.ok)
-            throw new Error(`An error occurred: ${response.statusText}`);
-        const got = await response.arrayBuffer();
-        return new Uint8Array(got);
-    }
-    async loadHeader(branch = 'main') {
-        const response = await fetch(this.headerURL(branch));
-        // console.log('rest getHeader', response.constructor.name)
-        if (!response.ok)
-            return null;
-        const got = await response.json();
-        // console.log('rest getHeader', got)
-        return got;
-    }
-    async writeHeader(branch, header) {
-        if (this.config.readonly)
-            return;
-        // console.log('writeHeader rt', branch, pHeader)
-        const response = await fetch(this.headerURL(branch), {
-            method: 'PUT',
-            body: header,
-            headers: { 'Content-Type': 'application/json' }
-        });
-        if (!response.ok)
-            throw new Error(`An error occurred: ${response.statusText}`);
-    }
-}

package/dist/storage/utils.js

@@ -1,144 +0,0 @@
-import * as CBW from '@ipld/car/buffer-writer';
-import * as raw from 'multiformats/codecs/raw';
-import { encrypt, decrypt } from '../crypto.js';
-import { parse } from 'multiformats/link';
-import { sha256 } from 'multiformats/hashes/sha2';
-import * as Block from 'multiformats/block';
-import { Buffer } from 'buffer';
-// @ts-ignore
-import { bf } from 'prolly-trees/utils';
-// @ts-ignore
-import { nocache as cache } from 'prolly-trees/cache';
-const chunker = bf(30);
-export async function getEmptyLoader() {
-    const theseWriteableBlocks = new VMemoryBlockstore();
-    return {
-        blocks: theseWriteableBlocks,
-        put: async (cid, bytes) => {
-            return await theseWriteableBlocks.put(cid, bytes);
-        },
-        get: async (cid) => {
-            const got = await theseWriteableBlocks.get(cid);
-            return got.bytes;
-        }
-    };
-}
-export class VMemoryBlockstore {
-    /** @type {Map<string, Uint8Array>} */
-    blocks = new Map();
-    instanceId = Math.random().toString(36).slice(2);
-    async get(cid) {
-        const bytes = this.blocks.get(cid.toString());
-        if (!bytes)
-            throw new Error('block not found ' + cid.toString());
-        return { cid, bytes };
-    }
-    /**
-     * @param {any} cid
-     * @param {Uint8Array} bytes
-     */
-    async put(cid, bytes) {
-        this.blocks.set(cid.toString(), bytes);
-    }
-    *entries() {
-        for (const [str, bytes] of this.blocks) {
-            yield { cid: parse(str), bytes };
-        }
-    }
-}
-export const blocksToCarBlock = async (rootCids, blocks) => {
-    // console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
-    let size = 0;
-    if (!Array.isArray(rootCids)) {
-        rootCids = [rootCids];
-    }
-    const headerSize = CBW.headerLength({ roots: rootCids });
-    size += headerSize;
-    if (!Array.isArray(blocks)) {
-        blocks = Array.from(blocks.entries());
-    }
-    for (const { cid, bytes } of blocks) {
-        // console.log(cid, bytes)
-        size += CBW.blockLength({ cid, bytes });
-    }
-    const buffer = new Uint8Array(size);
-    const writer = await CBW.createWriter(buffer, { headerSize });
-    for (const cid of rootCids) {
-        writer.addRoot(cid);
-    }
-    for (const { cid, bytes } of blocks) {
-        writer.write({ cid, bytes });
-    }
-    await writer.close();
-    return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw });
-};
-export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
-    const encryptionKey = Buffer.from(keyMaterial, 'hex');
-    const encryptedBlocks = [];
-    const theCids = cids;
-    // console.trace('blocksToEncryptedCarBlock', blocks)
-    // for (const { cid } of blocks.entries()) {
-    //   theCids.push(cid.toString())
-    // }
-    // console.log(
-    //   'encrypting',
-    //   theCids.length,
-    //   'blocks',
-    //   theCids.includes(innerBlockStoreClockRootCid.toString()),
-    //   keyMaterial
-    // )
-    // console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
-    let last;
-    for await (const block of encrypt({
-        cids: theCids,
-        get: async (cid) => {
-            // console.log('getencrypt', cid)
-            const got = blocks.get(cid);
-            // console.log('got', got)
-            return got.block ? ({ cid, bytes: got.block }) : got;
-        },
-        key: encryptionKey,
-        hasher: sha256,
-        chunker,
-        cache,
-        // codec: dagcbor, // should be crypto?
-        root: innerBlockStoreClockRootCid
-    })) {
-        encryptedBlocks.push(block);
-        last = block;
-    }
-    // console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
-    const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks);
-    return encryptedCar;
-};
-// { root, get, key, cache, chunker, hasher }
-const memoizeDecryptedCarBlocks = new Map();
-export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
-    if (memoizeDecryptedCarBlocks.has(cid.toString())) {
-        return memoizeDecryptedCarBlocks.get(cid.toString());
-    }
-    else {
-        const blocksPromise = (async () => {
-            const decryptionKey = Buffer.from(keyMaterial, 'hex');
-            // console.log('decrypting', keyMaterial, cid.toString())
-            const cids = new Set();
-            const decryptedBlocks = [];
-            for await (const block of decrypt({
-                root: cid,
-                get,
-                key: decryptionKey,
-                chunker,
-                hasher: sha256,
-                cache
-                // codec: dagcbor
-            })) {
-                // console.log('decrypted', block.cid.toString())
-                decryptedBlocks.push(block);
-                cids.add(block.cid.toString());
-            }
-            return { blocks: decryptedBlocks, cids };
-        })();
-        memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise);
-        return blocksPromise;
-    }
-};

package/dist/sync.js

@@ -1,218 +0,0 @@
-import SimplePeer from 'simple-peer';
-import { parseCID } from './database.js';
-import { decodeEventBlock } from './clock.js';
-import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js';
-import { CarReader } from '@ipld/car';
-/**
- * @typedef {import('./database.js').Database} Database
- */
-export class Sync {
-    /**
-     * @param {Database} database
-     * @param {typeof SimplePeer} [PeerClass]
-     * @memberof Sync
-     * @static
-     */
-    status = 'new';
-    constructor(database, PeerClass = SimplePeer) {
-        this.database = database;
-        this.database.blocks.syncs.add(this); // should this happen during setup?
-        this.PeerClass = PeerClass;
-        this.pushBacklog = new Promise((resolve, reject) => {
-            this.pushBacklogResolve = resolve;
-            this.pushBacklogReject = reject;
-        });
-        this.isReady = false;
-        // this.pushBacklog.then(() => {
-        //   // console.log('sync backlog resolved')
-        //   this.database.notifyReset()
-        // })
-        // this.connected = new Promise((resolve, reject) => {
-        //   this.readyResolve = resolve
-        //   this.readyReject = reject
-        // })
-    }
-    async offer() {
-        this.status = 'offering';
-        return this.setupPeer(true);
-    }
-    async accept(base64offer) {
-        const offer = JSON.parse(atob(base64offer));
-        const p = this.setupPeer(false);
-        this.peer.signal(offer);
-        this.status = 'accepting';
-        return p;
-    }
-    connect(base64accept) {
-        const accept = JSON.parse(atob(base64accept));
-        this.status = 'connecting';
-        this.peer.signal(accept);
-    }
-    async setupPeer(initiator = false) {
-        this.peer = new this.PeerClass({
-            initiator,
-            trickle: false
-        });
-        this.peer.on('connect', () => this.startSync());
-        this.peer.on('data', data => this.gotData(data));
-        const p = new Promise((resolve, reject) => {
-            this.peer.on('signal', resolve);
-            this.peer.on('error', reject);
-        });
-        return p.then(signal => btoa(JSON.stringify(signal)));
-    }
-    async backlog() {
-        return this.pushBacklog;
-    }
-    async gotData(data) {
-        // console.log('got data', data.toString())
-        let reader = null;
-        try {
-            reader = await CarReader.fromBytes(data);
-        }
-        catch (e) {
-            // console.log('not a car', data.toString())
-        }
-        if (reader) {
-            // console.log('got car')
-            this.status = 'parking car';
-            const blz = new Set();
-            for await (const block of reader.blocks()) {
-                blz.add(block);
-            }
-            const roots = await reader.getRoots();
-            // console.log(
-            //   'got car',
-            //   roots.map(c => c.toString()),
-            //   this.database.clock.map(c => c.toString())
-            // )
-            // console.log(
-            //   'got blocks!',
-            //   [...blz].map(({ cid }) => cid.toString())
-            // )
-            // @ts-ignore
-            reader.entries = reader.blocks;
-            await this.database.blocks.commit({
-                label: 'sync',
-                entries: () => [...blz],
-                get: async (cid) => await reader.get(cid),
-                lastCid: [...blz][0].cid // doesn't matter
-            }, false);
-            // first arg could be the roots parents?
-            // get the roots parents
-            const parents = await Promise.all(roots.map(async (cid) => {
-                const rbl = await reader.get(cid);
-                if (!rbl) {
-                    console.log('missing root block', cid.toString(), reader);
-                    throw new Error('missing root block');
-                }
-                const block = await decodeEventBlock(rbl.bytes);
-                return block.value.parents;
-            }));
-            this.database.applyClock(parents.flat(), roots);
-            this.database.notifyReset();
-            // console.log('after', this.database.clockToJSON())
-            this.pushBacklogResolve({ ok: true });
-        }
-        else {
-            // data is a json string, parse it
-            const message = JSON.parse(data.toString());
-            // console.log('got message', message)
-            if (message.ok) {
-                this.status = 'ok';
-                this.pushBacklogResolve({ ok: true });
-            }
-            else if (message.clock) {
-                const reqCidDiff = message;
-                // this might be a CID diff
-                // console.log('got diff', reqCidDiff)
-                const carBlock = await Sync.makeCar(this.database, null, reqCidDiff.cids);
-                if (!carBlock) {
-                    // we are full synced
-                    // console.log('we are full synced')
-                    this.status = 'full synced';
-                    this.peer.send(JSON.stringify({ ok: true }));
-                    // this.pushBacklogResolve({ ok: true })
-                }
-                else {
-                    // console.log('do send diff', carBlock.bytes.length)
-                    this.status = 'sending diff car';
-                    this.peer.send(carBlock.bytes);
-                    // console.log('sent diff car')
-                    // this.pushBacklogResolve({ ok: true })
-                }
-            }
-        }
-    }
-    destroy() {
-        this.database.blocks.syncs.delete(this);
-        this.status = 'destroyed';
-        // this.peer.destroy()  todo
-    }
-    async sendUpdate(blockstore) {
-        if (!this.peer || !this.isReady)
-            return;
-        // console.log('send update from', this.database.instanceId)
-        // todo should send updates since last sync (currently sends each transaction)
-        const newCar = await blocksToCarBlock(blockstore.lastCid, blockstore);
-        this.status = 'sending update car';
-        this.peer.send(newCar.bytes);
-    }
-    async startSync() {
-        // console.log('start sync', this.peer.initiator)
-        this.isReady = true;
-        const allCIDs = await this.database.allStoredCIDs();
-        // console.log('allCIDs', allCIDs)
-        const reqCidDiff = {
-            clock: this.database.clockToJSON(),
-            cids: allCIDs.map(cid => cid.toString())
-        };
-        // console.log('send diff', reqCidDiff)
-        this.status = 'sending cid diff';
-        this.peer.send(JSON.stringify(reqCidDiff));
-    }
-    // get all the cids
-    // tell valet to make a file
-    /**
-     * @param {import("./database.js").Database} database
-     * @param {string} key
-     */
-    static async makeCar(database, key, skip = []) {
-        const allCIDs = await database.allCIDs();
-        const blocks = database.blocks;
-        const rootCIDs = database.clock;
-        const newCIDs = [...new Set([...rootCIDs, ...allCIDs])].filter(cid => !skip.includes(cid.toString()));
-        const syncCIDs = [...new Set([...rootCIDs, ...allCIDs.filter(cid => !skip.includes(cid.toString()))])];
-        // console.log(
-        //   'makeCar',
-        //   rootCIDs.map(c => c.toString()),
-        //   syncCIDs.map(c => c.toString()),
-        //   allCIDs.map(c => c.toString())
-        // )
-        if (newCIDs.length === 0) {
-            return null;
-        }
-        if (typeof key === 'undefined') {
-            key = blocks.valet?.primary.keyMaterial;
-        }
-        if (key) {
-            return blocksToEncryptedCarBlock(rootCIDs, {
-                entries: () => syncCIDs.map(cid => ({ cid })),
-                get: async (cid) => await blocks.get(cid)
-            }, key, syncCIDs.map(c => c.toString()));
-        }
-        else {
-            const carBlocks = await Promise.all(syncCIDs.map(async (c) => {
-                const b = await blocks.get(c);
-                if (typeof b.cid === 'string') {
-                    b.cid = parseCID(b.cid);
-                }
-                return b;
-            }));
-            // console.log('carblock')
-            return blocksToCarBlock(rootCIDs, {
-                entries: () => carBlocks
-            });
-        }
-    }
-}

package/dist/utils.js

@@ -1,16 +0,0 @@
-/* global localStorage */
-let storageSupported = false;
-try {
-    storageSupported = window.localStorage && true;
-}
-catch (e) { }
-export function localGet(key) {
-    if (storageSupported) {
-        return localStorage && localStorage.getItem(key);
-    }
-}
-export function localSet(key, value) {
-    if (storageSupported) {
-        return localStorage && localStorage.setItem(key, value);
-    }
-}

package/dist/valet.js

@@ -1,102 +0,0 @@
-import { Loader } from './loader.js';
-export class Valet {
-    idb = null;
-    name = null;
-    uploadQueue = null;
-    alreadyEnqueued = new Set();
-    instanceId = Math.random().toString(36).slice(2);
-    constructor(name = 'default', config = {}) {
-        this.name = name;
-        // console.log('new Valet', name, config.primary)
-        this.primary = Loader.appropriate(name, config.primary);
-        this.secondary = config.secondary ? Loader.appropriate(name, config.secondary) : null;
-        // set up a promise listener that applies all the headers to the clock
-        // when they resolve
-        const readyP = [this.primary.ready];
-        if (this.secondary)
-            readyP.push(this.secondary.ready);
-        this.ready = Promise.all(readyP).then((blocksReady) => {
-            // console.log('blocksReady valet', this.name, blocksReady)
-            return blocksReady;
-        });
-    }
-    async saveHeader(header) {
-        // each storage needs to add its own carCidMapCarCid to the header
-        if (this.secondary) {
-            this.secondary.saveHeader(header);
-        } // todo: await?
-        return await this.primary.saveHeader(header);
-    }
-    async compact(clock) {
-        await this.primary.compact(clock);
-        if (this.secondary)
-            await this.secondary.compact(clock);
-    }
-    /**
-     * Group the blocks into a car and write it to the valet.
-     * @param {import('./blockstore.js').InnerBlockstore} innerBlockstore
-     * @param {Set<string>} cids
-     * @returns {Promise<void>}
-     * @memberof Valet
-     */
-    async writeTransaction(innerBlockstore, cids) {
-        if (innerBlockstore.lastCid) {
-            await this.primary.parkCar(innerBlockstore, cids);
-            if (this.secondary)
-                await this.secondary.parkCar(innerBlockstore, cids);
-        }
-        else {
-            throw new Error('missing lastCid for car header');
-        }
-    }
-    /**
-     * Iterate over all blocks in the store.
-     *
-     * @yields {{cid: string, value: Uint8Array}}
-     * @returns {AsyncGenerator<any, any, any>}
-     */
-    async *cids() {
-        // console.log('valet cids')
-        // todo use cidMap
-        // while (cursor) {
-        // yield { cid: cursor.key, car: cursor.value.car }
-        // cursor = await cursor.continue()
-        // }
-    }
-    remoteBlockFunction = null;
-    async getValetBlock(dataCID) {
-        // console.log('getValetBlock primary', dataCID)
-        try {
-            const { block } = await this.primary.getLoaderBlock(dataCID);
-            return block;
-        }
-        catch (e) {
-            // console.log('getValetBlock error', e)
-            if (this.secondary) {
-                // console.log('getValetBlock secondary', dataCID)
-                try {
-                    // eslint-disable-next-line
-                    const { block, reader } = await this.secondary.getLoaderBlock(dataCID);
-                    const writeableCarReader = await this.primary.getWriteableCarReader(reader);
-                    // console.log('getValetBlock secondary', dataCID, block.length)
-                    // eslint-disable-next-line
-                    const cids = new Set();
-                    for await (const { cid } of reader.entries()) {
-                        // console.log(cid, bytes)
-                        cids.add(cid.toString());
-                    }
-                    // reader.get = reader.gat // some consumers prefer get
-                    // console.log('replicating', reader.root)
-                    writeableCarReader.lastCid = reader.root.cid;
-                    writeableCarReader.head = [];
-                    await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e));
-                    // console.log('FIX THIS', did)
-                    return block;
-                }
-                catch (e) {
-                    // console.log('getValetBlock secondary error', e)
-                }
-            }
-        }
-    }
-}