@fireproof/core 0.21.0-dev-preview-6 → 0.22.0-dev-preview

package/tests/blockstore/loader.test.ts

@@ -1,297 +0,0 @@
-import * as codec from "@ipld/dag-cbor";
-import { sha256 as hasher } from "multiformats/hashes/sha2";
-import { CID } from "multiformats/cid";
-import { CRDTMeta, CarTransaction, IndexTransactionMeta, SuperThis, bs, ensureSuperThis, rt } from "@fireproof/core";
-import { simpleBlockOpts } from "../helpers.js";
-import { FPBlock, isCarBlockItemReady, isCarBlockItemStale } from "../../src/blockstore/index.js";
-import { anyBlock2FPBlock } from "../../src/blockstore/loader-helpers.js";
-
-class MyMemoryBlockStore extends bs.EncryptedBlockstore {
-  readonly memblock = new Map<string, FPBlock>();
-  loader: bs.Loader;
-  constructor(sthis: SuperThis) {
-    const ebOpts = simpleBlockOpts(sthis, "MyMemoryBlockStore"); //, "MyMemoryBlockStore");
-    // const ebOpts = {
-    //   name: "MyMemoryBlockStore",
-    // } as bs.BlockstoreOpts;
-    super(sthis, ebOpts);
-    this.loader = new bs.Loader(sthis, ebOpts);
-  }
-  ready(): Promise<void> {
-    return Promise.resolve();
-  }
-  close(): Promise<void> {
-    return this.loader.close();
-  }
-  readonly transactions = new Set<CarTransaction>();
-  // readonly lastTxMeta?: TransactionMeta;
-  readonly compacting: boolean = false;
-
-  override async put(fp: FPBlock): Promise<void> {
-    this.memblock.set(fp.cid.toString(), fp);
-  }
-
-  // transaction<M ext(fn: (t: CarTransaction) => Promise<MetaType>, opts?: { noLoader: boolean }): Promise<MetaType> {
-  //   throw new Error("Method not implemented.");
-  // }
-
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  getFile(car: bs.AnyLink, cid: bs.AnyLink, isPublic?: boolean): Promise<Uint8Array> {
-    throw new Error("Method not implemented.");
-  }
-  compact(): Promise<void> {
-    throw new Error("Method not implemented.");
-  }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  defaultCompact(blocks: bs.CompactionFetcher): Promise<bs.TransactionMeta> {
-    throw new Error("Method not implemented.");
-  }
-}
-
-describe("basic Loader simple", function () {
-  let loader: bs.Loader;
-  let block: FPBlock;
-  let t: CarTransaction;
-  const sthis = ensureSuperThis();
-
-  afterEach(async () => {
-    await loader.close();
-    await loader.destroy();
-  });
-
-  beforeEach(async () => {
-    const testDbName = "test-loader-commit";
-    await sthis.start();
-    const mockM = new MyMemoryBlockStore(sthis);
-    t = new bs.CarTransactionImpl(mockM as bs.EncryptedBlockstore);
-    loader = new bs.Loader(sthis, {
-      ...simpleBlockOpts(sthis, testDbName),
-      public: true,
-    });
-    await loader.ready();
-    block = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "world" },
-        hasher,
-        codec,
-      }),
-    );
-    await t.put(block);
-    await mockM.put(block);
-  });
-  it("should have an empty car log", function () {
-    expect(loader.carLog.length).toBe(0);
-  });
-  it("should commit", async () => {
-    const carGroup = await loader.commit(t, { head: [block.cid] });
-    expect(loader.carLog.length).toBe(1);
-    const reader = await loader.loadCar(carGroup[0], loader.attachedStores.local());
-    assert(isCarBlockItemReady(reader));
-    expect(reader).toBeTruthy();
-    const parsed = await bs.parseCarFile<CRDTMeta>(reader, loader.logger);
-    expect(parsed.cars).toBeTruthy();
-    expect(parsed.cars.length).toBe(0);
-    expect(parsed.meta).toBeTruthy();
-    expect(parsed.meta.head).toBeTruthy();
-  });
-});
-
-describe("basic Loader with two commits", function () {
-  let loader: bs.Loader;
-  let block: FPBlock;
-  let block2: FPBlock;
-  let block3: FPBlock;
-  let block4: FPBlock;
-  let t: CarTransaction;
-  let carCid: bs.CarGroup;
-  let carCid0: bs.CarGroup;
-
-  const sthis = ensureSuperThis();
-
-  afterEach(async () => {
-    await loader.close();
-    await loader.destroy();
-  });
-
-  beforeEach(async () => {
-    await sthis.start();
-    const mockM = new MyMemoryBlockStore(sthis);
-    t = new bs.CarTransactionImpl(mockM);
-    loader = new bs.Loader(sthis, {
-      ...simpleBlockOpts(sthis, "test-loader-two-commit"),
-      public: true,
-    });
-    await loader.ready();
-
-    block = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "world" },
-        hasher,
-        codec,
-      }),
-    );
-    await t.put(block);
-    carCid0 = await loader.commit(t, { head: [block.cid] });
-
-    block2 = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "universe" },
-        hasher,
-        codec,
-      }),
-    );
-    await t.put(block2);
-    carCid = await loader.commit(t, { head: [block2.cid] });
-
-    block3 = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "multiverse" },
-        hasher,
-        codec,
-      }),
-    );
-    await t.put(block3);
-
-    block4 = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "megaverse" },
-        hasher,
-        codec,
-      }),
-    );
-
-    await t.put(block4);
-  });
-
-  it("should have a car log", function () {
-    expect(loader.carLog.length).toBe(2);
-    expect(loader.carLog.asArray()[0].toString()).toBe(carCid.toString());
-    expect(loader.carLog.asArray()[1].toString()).toBe(carCid0.toString());
-  });
-
-  it("should commit", async () => {
-    const reader = await loader.loadCar(carCid[0], loader.attachedStores.local());
-    expect(reader).toBeTruthy();
-    assert(isCarBlockItemReady(reader));
-    const parsed = await bs.parseCarFile<CRDTMeta>(reader, loader.logger);
-    expect(parsed.cars).toBeTruthy();
-    expect(parsed.compact.length).toBe(0);
-    expect(parsed.cars.length).toBe(1);
-    expect(parsed.meta).toBeTruthy();
-    expect(parsed.meta.head).toBeTruthy();
-  });
-
-  it("should compact", async () => {
-    const compactCid = await loader.commit(t, { head: [block2.cid] }, { compact: true });
-    expect(loader.carLog.length).toBe(1);
-
-    const reader = await loader.loadCar(compactCid[0], loader.attachedStores.local());
-    expect(reader).toBeTruthy();
-    assert(isCarBlockItemReady(reader));
-    const parsed = await bs.parseCarFile<CRDTMeta>(reader, loader.logger);
-    expect(parsed.cars).toBeTruthy();
-    expect(parsed.compact.length).toBe(2);
-    expect(parsed.cars.length).toBe(0);
-    expect(parsed.meta).toBeTruthy();
-    expect(parsed.meta.head).toBeTruthy();
-  });
-
-  it("compact should erase old files", async () => {
-    const cs = await loader.attachedStores.local().active.car;
-    await loader.commit(t, { head: [block2.cid] }, { compact: true });
-    expect(loader.carLog.length).toBe(1);
-    await loader.commit(t, { head: [block3.cid] }, { compact: false });
-    expect(loader.carLog.length).toBe(2);
-    expect(await cs.load(carCid[0])).toBeTruthy();
-    await loader.commit(t, { head: [block3.cid] }, { compact: true });
-    expect(loader.carLog.length).toBe(1);
-    const e0 = await cs.load(carCid[0]).catch((e) => e);
-    expect(e0 instanceof Error).toBeTruthy();
-    await loader.commit(t, { head: [block4.cid] }, { compact: false });
-    expect(loader.carLog.length).toBe(2);
-
-    const e = await loader.loadCar(carCid[0], loader.attachedStores.local());
-    expect(e).toBeTruthy();
-    assert(isCarBlockItemStale(e));
-    expect(e.item.status).toBe("stale");
-    expect(e.item.statusCause.message).toMatch(/(missing car file)|(not found)/);
-  }, 10000);
-});
-
-describe("basic Loader with index commits", function () {
-  let block: FPBlock;
-  let ib: bs.EncryptedBlockstore;
-  let indexerResult: IndexTransactionMeta;
-  let cid: CID;
-  // let indexMap: Map<string, CID>;
-  const sthis = ensureSuperThis();
-
-  afterEach(async () => {
-    await ib.close();
-    await ib.destroy();
-  });
-
-  beforeEach(async () => {
-    const name = "test-loader-index" + Math.random();
-    await sthis.start();
-    // t = new CarTransaction()
-    ib = new bs.EncryptedBlockstore(sthis, simpleBlockOpts(sthis, name));
-    await ib.ready();
-    block = await anyBlock2FPBlock(
-      await rt.mf.block.encode({
-        value: { hello: "world" },
-        hasher,
-        codec,
-      }),
-    );
-    // console.log('block', block.cid)
-
-    cid = CID.parse("bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4");
-    indexerResult = {
-      indexes: {
-        hello: {
-          byId: cid,
-          byKey: cid,
-          head: [cid as CID<unknown, number, number, 1>],
-          name: "hello",
-          map: "(doc) => doc.hello",
-        },
-      },
-    };
-    // indexMap = new Map();
-  });
-
-  it("should start with an empty car log", function () {
-    expect(ib.loader).toBeTruthy();
-    expect(ib.loader.carLog.length).toBe(0);
-  });
-
-  it("should commit the index metadata", async () => {
-    const { cars: carCid } = await ib.transaction<IndexTransactionMeta>(
-      async (t) => {
-        await t.put(block);
-        return indexerResult;
-      } /* , indexMap */,
-    );
-
-    expect(carCid).toBeTruthy();
-    expect(ib.loader).toBeTruthy();
-    const carLog = ib.loader.carLog;
-
-    expect(carLog.length).toBe(1);
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const reader = await ib.loader.loadCar(carCid![0], ib.loader.attachedStores.local());
-    expect(reader).toBeTruthy();
-    assert(isCarBlockItemReady(reader));
-    const parsed = await bs.parseCarFile<IndexTransactionMeta>(reader, sthis.logger);
-    expect(parsed.cars).toBeTruthy();
-    expect(parsed.cars.length).toBe(0);
-    expect(parsed.meta).toBeTruthy();
-    expect(parsed.meta.indexes).toBeTruthy();
-    const indexes = parsed.meta.indexes;
-    expect(indexes).toBeTruthy();
-    expect(indexes.hello).toBeTruthy();
-    expect(indexes.hello.map).toBe("(doc) => doc.hello");
-    expect(indexes.hello.name).toBe("hello");
-  });
-});

package/tests/blockstore/standalone.test.ts

@@ -1,152 +0,0 @@
-import { BuildURI, runtimeFn, URI } from "@adviser/cement";
-import { ensureSuperThis, PARAM, CRDTImpl, CRDT, LedgerOpts, fireproof, bs, sleep } from "@fireproof/core";
-import { Link } from "multiformats";
-import { stripper } from "@adviser/cement/utils";
-import pLimit from "p-limit";
-
-describe("standalone", () => {
-  const sthis = ensureSuperThis();
-
-  describe("howto-loader", () => {
-    // beforeAll(async () => {
-    //   const db = fireproof("howto-loader", {
-    //     storeUrls: {
-    //       base: "file://dist/how-to-loader?store-key=insecure",
-    //     },
-    //   });
-    //   for (let cid = 0; cid < 1; cid++) {
-    //     const blk = new Array(3)
-    //       .fill(0)
-    //       .map((_, peer) => ({ _id: `test-${cid}:peer-${peer}`, value: { cid: `test-${cid}`, peer: `peer-${peer}` } }));
-    //     await db.bulk(blk);
-    //   }
-    //   // await db.compact()
-    //   await db.close();
-    // });
-
-    it("understand the writer", async () => {
-      const db = fireproof("howto-writer", {
-        storeUrls: {
-          base: "memory://how-to-writer",
-        },
-      });
-      await db.put({ _id: "test-1", value: { cid: "test-1", peer: "peer-1" } });
-      await db.close();
-    });
-
-    it.skip("understand loader", async () => {
-      const db = fireproof("howto-loader", {
-        storeUrls: {
-          base: "file://dist/how-to-loader?store-key=insecure",
-        },
-      });
-      const fn = vi.fn();
-      const loader = db.ledger.crdt.blockstore.loader as bs.Loader;
-      loader.cidCache.onSet(fn);
-      expect(fn).toHaveBeenCalledTimes(0);
-      await db.ready();
-      expect(
-        loader.cidCache.values().map((i) => {
-          const v = i.value.Ok();
-          return {
-            type: v.item.type,
-            cid: v.cid,
-            value: stripper(/(type|cid)/, v.item),
-          };
-        }),
-      ).toEqual([]);
-      // let last = 0
-      // for (let i = 0; i < 100; i++) {
-      //   if (last !== fn.mock.calls.length) {
-      //     console.log("fn", i, fn.mock.calls.length);
-      //     last = fn.mock.calls.length
-      //   }
-      //   await sleep(10);
-      // }
-      const ret = await db.allDocs();
-      expect(ret.rows.length).toBe(100);
-      expect(fn).toHaveBeenCalledTimes(109393);
-      await db.close();
-    });
-  });
-
-  describe("test-plimit", () => {
-    it("should work with plimit", async () => {
-      const limiter = pLimit(2);
-      let concurrent = 0;
-      const ret = await Promise.all(
-        Array(10)
-          .fill(0)
-          .map((_, i) => {
-            return limiter(async () => {
-              concurrent++;
-              expect(concurrent).toBeLessThanOrEqual(2);
-              await sleep(100);
-              concurrent--;
-              return i;
-            });
-          }),
-      );
-      expect(ret).toEqual(Array.from(Array(10).keys()));
-    });
-  });
-
-  describe.skip("crdt-stack", () => {
-    let crdt: CRDT;
-    beforeEach(async () => {
-      let uri: URI;
-      switch (true) {
-        case runtimeFn().isBrowser:
-          uri = BuildURI.from("indexeddb://standalone-peer-log").setParam(PARAM.NAME, "peer-log").URI();
-          break;
-        default:
-          uri = BuildURI.from("file://dist/standalone")
-            .setParam(PARAM.NAME, "peer-log")
-            .setParam(PARAM.STORE_KEY, "insecure")
-            .URI();
-          break;
-      }
-      // console.log("uri", uri.toString());
-      crdt = new CRDTImpl(sthis, {
-        name: "peer-log",
-        writeQueue: { chunkSize: 32 },
-        storeUrls: {
-          data: {
-            meta: uri.build().setParam(PARAM.STORE, "meta").setParam(PARAM.STORE_KEY, "insecure").URI(),
-            car: uri.build().setParam(PARAM.STORE, "car").setParam(PARAM.STORE_KEY, "insecure").URI(),
-            file: uri.build().setParam(PARAM.STORE, "file").setParam(PARAM.STORE_KEY, "insecure").URI(),
-            wal: uri.build().setParam(PARAM.STORE, "wal").setParam(PARAM.STORE_KEY, "insecure").URI(),
-          },
-        },
-      } as LedgerOpts);
-      await crdt.ready();
-    });
-    afterEach(async () => {
-      await crdt.close();
-    });
-
-    it.each(new Array(10).fill(0).map((_, i) => ({ cid: i })))("write:$cid", async ({ cid }) => {
-      const peers = new Array(10).fill(0).map((_, peer) => {
-        return { id: `test-${cid}:peer-${peer}`, value: { cid: `test-${cid}`, peer: `peer-${peer}` } };
-      });
-      await crdt.bulk(peers);
-    });
-    it("read", async () => {
-      for (let cid = 0; cid < 10; cid++) {
-        for (let peer = 0; peer < 10; peer++) {
-          const id = `test-${cid}:peer-${peer}`;
-          const res = (await crdt.get(id)) as unknown as { cid: Link; doc: { cid: string; peer: string } };
-          expect(res.doc).toEqual({ cid: `test-${cid}`, peer: `peer-${peer}` });
-        }
-      }
-    });
-    // for (let cid = 0; cid < 10; cid++) {
-    //   const gte = `test-${cid}`;
-    //   const ret = [];
-    //   for await (const [key, link] of entries(blk, root, { gte })) {
-    //     ret.push([key, link]);
-    //   }
-    //   expect(ret.length).toEqual(10);
-    // }
-  });
-});

package/tests/blockstore/store.test.ts

@@ -1,192 +0,0 @@
-import { CID } from "multiformats";
-import { rt, bs, NotFoundError, PARAM, ensureSuperThis } from "@fireproof/core";
-import { Result } from "@adviser/cement";
-import { mockLoader, noopUrl } from "../helpers.js";
-
-// function runtime(sthis: SuperThis) {
-//   return bs.toStoreRuntime(sthis);
-// }
-
-// async function mockLoader(sthis: SuperThis, name?: string): Promise<bs.StoreFactoryItem> {
-//   const url = noopUrl(name);
-//   return {
-//     // sthis,
-//     url: url,
-//     loader:
-//   };
-// }
-
-describe("CarStore", function () {
-  let store: bs.CarStore;
-  const sthis = ensureSuperThis();
-
-  let loader: bs.Loadable;
-
-  afterEach(async () => {
-    await store.close();
-    await store.destroy();
-  });
-
-  beforeEach(async () => {
-    loader = mockLoader(sthis);
-    await sthis.start();
-    const at = await bs.createAttachedStores(noopUrl("test"), loader);
-    store = at.stores.car;
-    await store.start(at.stores);
-  });
-
-  it("should have a name", function () {
-    expect(store.url().getParam(PARAM.NAME)).toEqual("test");
-  });
-
-  it("should save a car", async () => {
-    const car: bs.AnyBlock = {
-      cid: "cidKey" as unknown as CID,
-      bytes: new Uint8Array([55, 56, 57]),
-    };
-    await store.save(car);
-    const data = (await store.realGateway.getPlain({ loader }, store.url(), car.cid.toString())).Ok();
-    expect(sthis.txt.decode(data)).toEqual(sthis.txt.decode(car.bytes));
-  });
-});
-
-describe("CarStore with a saved car", function () {
-  let store: bs.CarStore;
-  let car: bs.AnyBlock;
-  let loader: bs.Loadable;
-
-  const sthis = ensureSuperThis();
-
-  afterEach(async () => {
-    await store.close();
-    await store.destroy();
-  });
-
-  beforeEach(async () => {
-    loader = mockLoader(sthis);
-    await sthis.start();
-
-    const at = await bs.createAttachedStores(noopUrl("test2"), loader);
-    store = at.stores.car;
-    await store.start(at.stores);
-    car = {
-      cid: "cid" as unknown as CID,
-      bytes: new Uint8Array([55, 56, 57, 80]),
-    };
-    await store.save(car);
-  });
-
-  it("should have a car", async () => {
-    const data = (await store.realGateway.getPlain({ loader }, store.url(), car.cid.toString())).Ok();
-    expect(sthis.txt.decode(data)).toEqual(sthis.txt.decode(car.bytes));
-  });
-
-  it("should load a car", async () => {
-    const loaded = await store.load(car.cid);
-    expect(loaded.cid).toEqual(car.cid);
-    expect(loaded.bytes.constructor.name).toEqual("Uint8Array");
-    expect(loaded.bytes.toString()).toEqual(car.bytes.toString());
-  });
-
-  it("should remove a car", async () => {
-    await store.remove(car.cid);
-    const { e: error } = (await store.load(car.cid).catch((e: Error) => ({ e }))) as { e: NotFoundError };
-    expect(error).toBeTruthy();
-  });
-});
-
-describe("MetaStore", function () {
-  let store: bs.MetaStore;
-  const sthis = ensureSuperThis();
-  let loader: bs.Loadable;
-
-  afterEach(async () => {
-    await store.close();
-    await store.destroy();
-  });
-
-  beforeEach(async () => {
-    loader = mockLoader(sthis);
-    await sthis.start();
-    const at = await bs.createAttachedStores(noopUrl("test"), loader);
-    store = at.stores.meta;
-    await store.start(at.stores);
-  });
-
-  it("should have a name", function () {
-    expect(store.url().getParam(PARAM.NAME)).toEqual("test");
-  });
-
-  it("should save a header", async () => {
-    const cid = CID.parse("bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4");
-    const h: bs.DbMeta = {
-      cars: [cid],
-      // key: undefined,
-    };
-    await store.save(h);
-    const file = await store.realGateway.getPlain({ loader }, store.url(), "main");
-    const blockMeta = (await rt.gw.fpDeserialize(sthis, store.url(), file)) as Result<bs.FPEnvelopeMeta>;
-    expect(blockMeta.Ok()).toBeTruthy();
-    expect(blockMeta.Ok().payload.length).toEqual(1);
-    const decodedHeader = blockMeta.Ok().payload[0].dbMeta;
-    expect(decodedHeader).toBeTruthy();
-    expect(decodedHeader.cars).toBeTruthy();
-    expect(decodedHeader.cars[0].toString()).toEqual(cid.toString());
-  });
-});
-
-describe("MetaStore with a saved header", function () {
-  let store: bs.MetaStore;
-  let cid: CID;
-  const sthis = ensureSuperThis();
-  let loader: bs.Loadable;
-
-  afterEach(async () => {
-    await store.close();
-    await store.destroy();
-  });
-
-  beforeEach(async () => {
-    loader = mockLoader(sthis);
-    await sthis.start();
-    const at = await bs.createAttachedStores(noopUrl("test3-meta"), loader);
-    store = at.stores.meta;
-    await store.start(at.stores);
-    cid = CID.parse("bafybeia4luuns6dgymy5kau5rm7r4qzrrzg6cglpzpogussprpy42cmcn4");
-    await store.save({ cars: [cid] /*, key: undefined */ });
-  });
-
-  // it("should load", async () =>{
-  //   expect(onload).toBeTruthy();
-  //   expect(onload?.length).toEqual(1);
-  //   expect(onload?.[0].cars.toString()).toEqual(cid.toString());
-  // });
-
-  it("should have a header", async () => {
-    const bytes = await store.realGateway.getPlain({ loader }, store.url(), "main");
-    const data = sthis.txt.decode(bytes.Ok());
-    expect(data).toMatch(/parents/);
-    const header = JSON.parse(data)[0];
-    expect(header).toBeDefined();
-    expect(header.parents).toBeDefined();
-    // const [blockMeta] = await store.handleByteHeads(bytes);
-
-    const blockMeta = (await rt.gw.fpDeserialize(sthis, store.url(), bytes)) as Result<bs.FPEnvelopeMeta>;
-    expect(blockMeta.Ok()).toBeTruthy();
-    expect(blockMeta.Ok().payload.length).toEqual(1);
-    const decodedHeader = blockMeta.Ok().payload[0].dbMeta;
-    expect(decodedHeader).toBeDefined();
-    expect(decodedHeader.cars).toBeDefined();
-    expect(decodedHeader.cars[0].toString()).toEqual(cid.toString());
-  });
-
-  it("should load a header", async () => {
-    const metaStream = store.stream();
-    for await (const cars of metaStream) {
-      // expect(loaded).toBeTruthy();
-      expect(cars).toBeTruthy();
-      expect(cars.map((i) => i.cars.map((i) => i.toString())).flat(2)).toEqual([cid.toString()]);
-      break;
-    }
-  });
-});