@fireproof/core 0.19.8-dev-global → 0.19.8-dev-series-1

package/node-filesystem-INX4ZTHE.js

@@ -0,0 +1,45 @@
+import {
+  toArrayBuffer
+} from "./chunk-WS3YRPIA.js";
+import "./chunk-OFGPKRCM.js";
+
+// src/runtime/gateways/file/node-filesystem.ts
+import { runtimeFn } from "@adviser/cement";
+var NodeFileSystem = class {
+  async start() {
+    this.fs = await import("node:fs/promises");
+    return this;
+  }
+  async mkdir(path, options) {
+    return this.fs?.mkdir(path, options);
+  }
+  async readdir(path, options) {
+    return this.fs?.readdir(path, options);
+  }
+  async rm(path, options) {
+    return this.fs?.rm(path, options);
+  }
+  async copyFile(source, destination) {
+    return this.fs?.copyFile(source, destination);
+  }
+  async readfile(path, options) {
+    const ret = await this.fs?.readFile(path, options);
+    return toArrayBuffer(ret);
+  }
+  stat(path) {
+    return this.fs?.stat(path);
+  }
+  async unlink(path) {
+    return this.fs?.unlink(path);
+  }
+  async writefile(path, data) {
+    if (runtimeFn().isDeno) {
+      return this.fs?.writeFile(path, data);
+    }
+    return this.fs?.writeFile(path, Buffer.from(data));
+  }
+};
+export {
+  NodeFileSystem
+};
+//# sourceMappingURL=node-filesystem-INX4ZTHE.js.map

package/node-filesystem-INX4ZTHE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/runtime/gateways/file/node-filesystem.ts"],"sourcesContent":["import type { PathLike, MakeDirectoryOptions, Stats, ObjectEncodingOptions } from \"node:fs\";\nimport type { mkdir, readdir, rm, copyFile, readFile, stat, unlink, writeFile } from \"node:fs/promises\";\nimport { toArrayBuffer } from \"./utils.js\";\nimport { SysFileSystem } from \"../../../types.js\";\nimport { runtimeFn } from \"@adviser/cement\";\n\nexport class NodeFileSystem implements SysFileSystem {\n  fs?: {\n    mkdir: typeof mkdir;\n    readdir: typeof readdir;\n    rm: typeof rm;\n    copyFile: typeof copyFile;\n    readFile: typeof readFile;\n    stat: typeof stat;\n    unlink: typeof unlink;\n    writeFile: typeof writeFile;\n  };\n\n  async start(): Promise<SysFileSystem> {\n    this.fs = await import(\"node:fs/promises\");\n    return this;\n  }\n  async mkdir(path: PathLike, options?: { recursive: boolean }): Promise<string | undefined> {\n    return this.fs?.mkdir(path, options);\n  }\n  async readdir(path: PathLike, options?: ObjectEncodingOptions): Promise<string[]> {\n    return this.fs?.readdir(path, options) as Promise<string[]>;\n  }\n  async rm(path: PathLike, options?: MakeDirectoryOptions & { recursive: boolean }): Promise<void> {\n    return this.fs?.rm(path, options);\n  }\n  async copyFile(source: PathLike, destination: PathLike): Promise<void> {\n    return this.fs?.copyFile(source, destination);\n  }\n  async readfile(path: PathLike, options?: { encoding: BufferEncoding; flag?: string }): Promise<Uint8Array> {\n    const ret = (await this.fs?.readFile(path, options)) as Buffer;\n    return toArrayBuffer(ret);\n  }\n  stat(path: PathLike): Promise<Stats> {\n    return this.fs?.stat(path) as Promise<Stats>;\n  }\n  async unlink(path: PathLike): Promise<void> {\n    return this.fs?.unlink(path);\n  }\n  async writefile(path: PathLike, data: Uint8Array | string): Promise<void> {\n    if (runtimeFn().isDeno) {\n      return this.fs?.writeFile(path, data);\n    }\n    return this.fs?.writeFile(path, Buffer.from(data));\n  }\n}\n\n// import { type NodeMap, join } from \"../../sys-container.js\";\n// import type { ObjectEncodingOptions, PathLike } from \"fs\";\n// import * as fs from \"fs/promises\";\n// import * as path from \"path\";\n// import * as os from \"os\";\n// import * as url from \"url\";\n// import { toArrayBuffer } from \"./utils.js\";\n\n// export async function createNodeSysContainer(): Promise<NodeMap> {\n//   // const nodePath = \"node:path\";\n//   // const nodeOS = \"node:os\";\n//   // const nodeURL = \"node:url\";\n//   // const nodeFS = \"node:fs\";\n//   // const fs = (await import(\"node:fs\")).promises;\n//   // const assert = \"assert\";\n//   // const path = await import(\"node:path\");\n//   return {\n//     state: \"node\",\n//     ...path,\n//     // ...(await import(\"node:os\")),\n//     // ...(await import(\"node:url\")),\n//     ...os,\n//     ...url,\n//     ...fs,\n//     join,\n//     stat: fs.stat as NodeMap[\"stat\"],\n//     readdir: fs.readdir as NodeMap[\"readdir\"],\n//     readfile: async (path: PathLike, options?: ObjectEncodingOptions): Promise<Uint8Array> => {\n//       const rs = await fs.readFile(path, options);\n//       return toArrayBuffer(rs);\n//     },\n//     writefile: fs.writeFile as NodeMap[\"writefile\"],\n//   };\n// }\n"],"mappings":";;;;;;AAIA,SAAS,iBAAiB;AAEnB,IAAM,iBAAN,MAA8C;AAAA,EAYnD,MAAM,QAAgC;AACpC,SAAK,KAAK,MAAM,OAAO,kBAAkB;AACzC,WAAO;AAAA,EACT;AAAA,EACA,MAAM,MAAM,MAAgB,SAA+D;AACzF,WAAO,KAAK,IAAI,MAAM,MAAM,OAAO;AAAA,EACrC;AAAA,EACA,MAAM,QAAQ,MAAgB,SAAoD;AAChF,WAAO,KAAK,IAAI,QAAQ,MAAM,OAAO;AAAA,EACvC;AAAA,EACA,MAAM,GAAG,MAAgB,SAAwE;AAC/F,WAAO,KAAK,IAAI,GAAG,MAAM,OAAO;AAAA,EAClC;AAAA,EACA,MAAM,SAAS,QAAkB,aAAsC;AACrE,WAAO,KAAK,IAAI,SAAS,QAAQ,WAAW;AAAA,EAC9C;AAAA,EACA,MAAM,SAAS,MAAgB,SAA4E;AACzG,UAAM,MAAO,MAAM,KAAK,IAAI,SAAS,MAAM,OAAO;AAClD,WAAO,cAAc,GAAG;AAAA,EAC1B;AAAA,EACA,KAAK,MAAgC;AACnC,WAAO,KAAK,IAAI,KAAK,IAAI;AAAA,EAC3B;AAAA,EACA,MAAM,OAAO,MAA+B;AAC1C,WAAO,KAAK,IAAI,OAAO,IAAI;AAAA,EAC7B;AAAA,EACA,MAAM,UAAU,MAAgB,MAA0C;AACxE,QAAI,UAAU,EAAE,QAAQ;AACtB,aAAO,KAAK,IAAI,UAAU,MAAM,IAAI;AAAA,IACtC;AACA,WAAO,KAAK,IAAI,UAAU,MAAM,OAAO,KAAK,IAAI,CAAC;AAAA,EACnD;AACF;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.19.8-dev-global",
+  "version": "0.19.8-dev-series-1",
   "description": "Live database for the web.",
   "type": "module",
   "module": "./index.js",
@@ -46,17 +46,21 @@
     "react": ">=18.0.0"
   },
   "dependencies": {
-    "react": "^18.3.1",
-    "@adviser/cement": "^0.2.2",
+    "@adviser/cement": "^0.2.31",
+    "multiformats": "^13.3.0",
+    "@ipld/unixfs": "^3.0.0",
+    "@ipld/car": "^5.3.2",
+    "@ipld/dag-json": "^10.2.2",
+    "@ipld/dag-cbor": "^9.2.1",
+    "@web3-storage/pail": "^0.6.0",
+    "cborg": "^4.2.4",
     "charwise": "^3.0.1",
     "prolly-trees": "^1.0.4",
     "idb": "^8.0.0",
-    "ipfs-unixfs-exporter": "^13.5.0",
-    "uuidv7": "^1.0.0",
+    "ipfs-unixfs-exporter": "^13.6.1",
+    "memfs": "^4.12.0",
     "p-limit": "^6.1.0",
-    "@web3-storage/pail": "^0.6.0",
-    "@ipld/unixfs": "^3.0.0",
-    "@ipld/car": "^5.3.2"
+    "react": "^18.3.1"
   },
   "scripts": {}
 }

package/tests/blockstore/fragment-gateway.test.ts

@@ -0,0 +1,107 @@
+import { Result, URI } from "@adviser/cement";
+import { mockSuperThis } from "../helpers";
+import { bs } from "@fireproof/core";
+
+class TraceGateway implements bs.Gateway {
+  readonly buildUrlFn = vitest.fn();
+
+  readonly fragSize: number;
+  constructor(fragSize = 0) {
+    this.fragSize = fragSize;
+  }
+
+  buildUrl(baseUrl: URI, key: string): Promise<Result<URI>> {
+    this.buildUrlFn(baseUrl, key);
+    return Promise.resolve(Result.Ok(baseUrl.build().setParam("key", key).URI()));
+  }
+  readonly startFn = vitest.fn();
+  start(baseUrl: URI): Promise<Result<URI>> {
+    this.startFn(baseUrl);
+    const burl = baseUrl.build();
+    if (this.fragSize) {
+      burl.setParam("fragSize", this.fragSize.toString());
+    }
+    return Promise.resolve(Result.Ok(burl.URI()));
+  }
+  readonly closeFn = vitest.fn();
+  close(baseUrl: URI): Promise<bs.VoidResult> {
+    this.closeFn(baseUrl);
+    return Promise.resolve(Result.Ok(undefined));
+  }
+  readonly destroyFn = vitest.fn();
+  destroy(baseUrl: URI): Promise<bs.VoidResult> {
+    this.destroyFn(baseUrl);
+    return Promise.resolve(Result.Ok(undefined));
+  }
+  readonly getFn = vitest.fn();
+  putCalls = 0;
+  async get(url: URI): Promise<Result<Uint8Array>> {
+    const idx = this.putCalls++;
+    this.getFn(url);
+    return Result.Ok(this.putFn.mock.calls[idx][1]);
+  }
+  readonly putFn = vitest.fn();
+  async put(url: URI, data: Uint8Array): Promise<Result<void>> {
+    this.putFn(url, data);
+    return Result.Ok(undefined);
+  }
+  readonly deleteFn = vitest.fn();
+  async delete(url: URI): Promise<Result<void>> {
+    this.deleteFn(url);
+    return Result.Ok(undefined);
+  }
+}
+
+describe("FragmentGateway", () => {
+  const sthis = mockSuperThis();
+  it("passthrough", async () => {
+    const innerGW = new TraceGateway();
+    const fgw = new bs.FragmentGateway(sthis, innerGW);
+    const url = URI.from("http://example.com?key=3333");
+
+    expect(await fgw.put(url, new Uint8Array([1, 2, 3, 4]))).toEqual(Result.Ok(undefined));
+    expect(innerGW.putFn).toHaveBeenCalledWith(url, new Uint8Array([1, 2, 3, 4]));
+
+    expect(await fgw.get(url)).toEqual(Result.Ok(new Uint8Array([1, 2, 3, 4])));
+    expect(innerGW.getFn).toHaveBeenCalledWith(url);
+  });
+
+  function slice(total: number, headerSize: number, fragSize: number): { len?: string; ofs: string }[] {
+    const res = [];
+    for (let ofs = 0; ofs < total; ofs += fragSize - headerSize) {
+      res.push({ len: total.toString(), ofs: ofs.toString() });
+    }
+    return res;
+  }
+
+  it("enable frag", async () => {
+    const innerGW = new TraceGateway(128);
+    const fgw = new bs.FragmentGateway(sthis, innerGW);
+    const url = (await fgw.start(URI.from("http://example.com?key=3333"))).Ok();
+    const buf = new Uint8Array(1024).fill(1).map((_, i) => i);
+
+    expect(await fgw.put(url, buf)).toEqual(Result.Ok(undefined));
+
+    const ref = slice(1024, fgw.headerSize, 128);
+
+    expect(
+      innerGW.putFn.mock.calls.map((i) => {
+        return {
+          len: i[0].getParam("len"),
+          ofs: i[0].getParam("ofs"),
+        };
+      }),
+    ).toEqual(ref);
+
+    expect((await fgw.get(url)).Ok()).toEqual(buf);
+    ref[0].len = undefined;
+    expect(
+      innerGW.getFn.mock.calls.map((i) => {
+        return {
+          len: i[0].getParam("len"),
+          ofs: i[0].getParam("ofs"),
+        };
+      }),
+    ).toEqual(ref);
+  });
+});

package/tests/blockstore/keyed-crypto.test.ts

@@ -0,0 +1,332 @@
+import { bs, rt } from "@fireproof/core";
+import { BuildURI, Logger, MockLogger, runtimeFn, toCryptoRuntime, URI } from "@adviser/cement";
+import { base58btc } from "multiformats/bases/base58";
+import { sha256 as hasher } from "multiformats/hashes/sha2";
+import * as dagCodec from "@ipld/dag-cbor";
+import { MockSuperThis, mockSuperThis } from "../helpers";
+
+describe("KeyBag", () => {
+  let url: URI;
+  let sthis: MockSuperThis;
+
+  beforeEach(async () => {
+    sthis = mockSuperThis();
+    await sthis.start();
+    if (runtimeFn().isBrowser) {
+      url = URI.from("indexdb://fp-keybag");
+    } else {
+      url = URI.merge(`file://./dist/tests/key.bag`, sthis.env.get("FP_KEYBAG_URL"));
+    }
+  });
+  it("default-path", async () => {
+    const old = sthis.env.get("FP_KEYBAG_URL");
+    sthis.env.delete("FP_KEYBAG_URL");
+    const kb = await rt.kb.getKeyBag(sthis);
+    if (runtimeFn().isBrowser) {
+      expect(kb.rt.url.toString()).toBe(`indexdb://fp-keybag`);
+    } else {
+      expect(kb.rt.url.toString()).toBe(`file://${sthis.env.get("HOME")}/.fireproof/keybag`);
+    }
+    sthis.env.set("FP_KEYBAG_URL", old);
+  });
+  it("from env", async () => {
+    const old = sthis.env.get("FP_KEYBAG_URL");
+    sthis.env.set("FP_KEYBAG_URL", url.toString());
+    const kb = await rt.kb.getKeyBag(sthis);
+    expect(kb.rt.url.toString()).toBe(url.toString());
+    sthis.env.set("FP_KEYBAG_URL", old);
+  });
+
+  it("extract keyMaterial", async () => {
+    const dkb = await rt.kb.getKeyBag(sthis);
+    const old = sthis.env.get("FP_KEYBAG_URL");
+    sthis.env.set("FP_KEYBAG_URL", BuildURI.from(dkb.rt.url).setParam("extractKey", "_deprecated_internal_api").toString());
+    const kb = await rt.kb.getKeyBag(sthis);
+    const key = kb.rt.crypto.randomBytes(kb.rt.keyLength);
+    const keyStr = base58btc.encode(key);
+    const res = await kb.setNamedKey("extract.test", keyStr);
+    expect(res.isOk()).toBeTruthy();
+    const gkb = await kb.getNamedExtractableKey("extract.test", true);
+    expect(gkb.isOk()).toBeTruthy();
+    expect(await gkb.Ok().extract()).toEqual({
+      key,
+      keyStr,
+    });
+    sthis.env.set("FP_KEYBAG_URL", old);
+    await sthis.logger.Flush();
+    expect(sthis.logCollector.Logs()).toEqual([
+      {
+        level: "warn",
+        module: "KeyBag",
+        msg: "extractKey is enabled via _deprecated_internal_api --- handle keys safely!!!",
+      },
+    ]);
+  });
+
+  it("simple add", async () => {
+    const kb = await rt.kb.getKeyBag(sthis, {
+      url: url.toString(),
+      crypto: toCryptoRuntime({
+        randomBytes: (size) => new Uint8Array(size).map((_, i) => i),
+      }),
+    });
+    const name = "setkey" + Math.random();
+    expect((await kb.getNamedKey(name, true)).isErr()).toBeTruthy();
+
+    const key = base58btc.encode(kb.rt.crypto.randomBytes(kb.rt.keyLength));
+    const res = await kb.setNamedKey(name, key);
+    expect(res.isOk()).toBeTruthy();
+    expect((await kb.getNamedKey(name, true)).Ok()).toEqual(res.Ok());
+
+    const name2 = "implicit";
+    const created = await kb.getNamedKey(name2);
+    expect(created.isOk()).toBeTruthy();
+
+    expect((await kb.getNamedKey(name2)).Ok()).toEqual(created.Ok());
+
+    let diskBag: rt.kb.KeyItem;
+    let diskBag2: rt.kb.KeyItem;
+    const provider = await kb.rt.getBag();
+    if (runtimeFn().isBrowser) {
+      const p = provider as rt.kb.KeyBagProviderIndexDB;
+      diskBag = await p._prepare().then((db) => db.get("bag", name));
+      diskBag2 = await p._prepare().then((db) => db.get("bag", name2));
+    } else {
+      const p = provider as rt.kb.KeyBagProviderFile;
+      const { sysFS } = await p._prepare(name);
+
+      diskBag = await sysFS.readfile((await p._prepare(name)).fName).then((data) => {
+        return JSON.parse(new TextDecoder().decode(data)) as rt.kb.KeyItem;
+      });
+      diskBag2 = await sysFS.readfile((await p._prepare(name2)).fName).then((data) => {
+        return JSON.parse(new TextDecoder().decode(data)) as rt.kb.KeyItem;
+      });
+    }
+    expect(await kb.toKeyWithFingerPrint(diskBag.key)).toEqual(res);
+    expect(await kb.toKeyWithFingerPrint(diskBag2.key)).toEqual(created);
+    const algo = {
+      name: "AES-GCM",
+      iv: kb.rt.crypto.randomBytes(12),
+      tagLength: 128,
+    };
+    const data = kb.rt.crypto.randomBytes(122);
+    expect(await kb.rt.crypto.encrypt(algo, res.Ok().key, data)).toEqual(await kb.rt.crypto.encrypt(algo, created.Ok().key, data));
+    expect(await kb.rt.crypto.encrypt(algo, await kb.subtleKey(diskBag.key), data)).toEqual(
+      await kb.rt.crypto.encrypt(algo, created.Ok().key, data),
+    );
+    expect(await kb.rt.crypto.encrypt(algo, await kb.subtleKey(diskBag2.key), data)).toEqual(
+      await kb.rt.crypto.encrypt(algo, created.Ok().key, data),
+    );
+  });
+});
+
+describe("KeyedCryptoStore", () => {
+  let kb: rt.kb.KeyBag;
+  let logger: Logger;
+  let baseUrl: URI;
+  const sthis = mockSuperThis();
+  beforeEach(async () => {
+    await sthis.start();
+    logger = MockLogger().logger;
+    let kbUrl: URI;
+    if (runtimeFn().isBrowser) {
+      kbUrl = URI.from("indexdb://fp-keybag");
+      baseUrl = URI.from("indexdb://fp-keyed-crypto-store");
+    } else {
+      kbUrl = URI.merge(`file://./dist/tests/key.bag`, sthis.env.get("FP_KEYBAG_URL"));
+      // baseUrl = URI.merge("file://./dist/tests/keyed-crypto-store", sthis.env.get("FP_STORAGE_URL"));
+      baseUrl = URI.from(sthis.env.get("FP_STORAGE_URL"));
+    }
+    kb = await rt.kb.getKeyBag(sthis, {
+      url: kbUrl,
+    });
+  });
+  it("no crypto", async () => {
+    const loader = {
+      sthis,
+      name: "test",
+      ebOpts: {
+        keyBag: {
+          keyRuntime: kb.rt,
+        },
+        store: {
+          stores: {
+            base: baseUrl.build().setParam("storekey", "insecure"),
+          },
+        },
+      },
+    } as unknown as bs.Loadable;
+    const strt = bs.toStoreRuntime({}, sthis);
+
+    for (const pstore of [strt.makeDataStore(loader), strt.makeMetaStore(loader), strt.makeWALStore(loader)]) {
+      const store = await pstore;
+      // await store.start();
+      const kc = await store.keyedCrypto();
+      expect(kc.constructor.name).toBe("noCrypto");
+      // expect(kc.isEncrypting).toBe(false);
+    }
+  });
+
+  it("create key", async () => {
+    const loader = {
+      sthis,
+      name: "test",
+      ebOpts: {
+        keyBag: {
+          keyRuntime: kb.rt,
+        },
+        store: {
+          stores: {
+            base: baseUrl,
+          },
+        },
+      },
+    } as unknown as bs.Loadable;
+    const strt = bs.toStoreRuntime({}, sthis);
+    for (const pstore of [strt.makeDataStore(loader), strt.makeMetaStore(loader), strt.makeWALStore(loader)]) {
+      const store = await bs.ensureStart(await pstore, logger);
+      const kc = await store.keyedCrypto();
+      expect(kc.constructor.name).toBe("keyedCrypto");
+      // expect(kc.isEncrypting).toBe(true);
+      expect(store.url().getParam("storekey")).toBe(`@test:${store.url().getParam("store")}@`);
+    }
+  });
+
+  it("key ref keybag", async () => {
+    const key = base58btc.encode(kb.rt.crypto.randomBytes(kb.rt.keyLength));
+    const genKey = await kb.setNamedKey("@heute@", key);
+    const loader = {
+      sthis,
+      name: "test",
+      ebOpts: {
+        keyBag: {
+          keyRuntime: kb.rt,
+        },
+        store: {
+          stores: {
+            base: baseUrl.build().setParam("storekey", "@heute@").URI(),
+          },
+        },
+      },
+    } as unknown as bs.Loadable;
+    const strt = bs.toStoreRuntime({}, sthis);
+    for (const pstore of [strt.makeDataStore(loader), strt.makeMetaStore(loader), strt.makeWALStore(loader)]) {
+      const store = await pstore;
+      // await store.start();
+      expect(store.url().getParam("storekey")).toBe(`@heute@`);
+      const kc = await store.keyedCrypto();
+      expect(kc.constructor.name).toBe("keyedCrypto");
+      const testData = kb.rt.crypto.randomBytes(1024);
+      const iv = kb.rt.crypto.randomBytes(12);
+      const blk = await kc._encrypt({ bytes: testData, iv });
+      expect(blk).not.toEqual(testData);
+      const fpkey = genKey.Ok();
+      expect(fpkey.fingerPrint).toEqual(await kc.fingerPrint());
+      const dec = new Uint8Array(await kc.crypto.decrypt(kc.algo(iv), fpkey.key, blk));
+      expect(dec).toEqual(testData);
+    }
+  });
+
+  it("key", async () => {
+    const key = base58btc.encode(kb.rt.crypto.randomBytes(kb.rt.keyLength));
+    const loader = {
+      sthis,
+      name: "test",
+      ebOpts: {
+        keyBag: {
+          keyRuntime: kb.rt,
+        },
+        store: {
+          stores: {
+            base: BuildURI.from(baseUrl).setParam("storekey", key),
+          },
+        },
+      },
+    } as unknown as bs.Loadable;
+    const strt = bs.toStoreRuntime({}, sthis);
+    for (const pstore of [strt.makeDataStore(loader), strt.makeMetaStore(loader), strt.makeWALStore(loader)]) {
+      const store = await pstore;
+      // await store.start();
+      expect(store.url().getParam("storekey")).toBe(key);
+      const kc = await store.keyedCrypto();
+      expect(kc.constructor.name).toBe("keyedCrypto");
+      const testData = kb.rt.crypto.randomBytes(1024);
+      const iv = kb.rt.crypto.randomBytes(12);
+      const blk = await kc._encrypt({ bytes: testData, iv });
+      expect(blk).not.toEqual(testData);
+      const dec = await kc._decrypt({ bytes: blk, iv });
+      expect(dec).toEqual(testData);
+    }
+  });
+});
+
+describe("KeyedCrypto", () => {
+  let kb: rt.kb.KeyBag;
+  let kycr: bs.KeyedCrypto;
+  let keyStr: string;
+  const sthis = mockSuperThis();
+  beforeEach(async () => {
+    let url: URI;
+    if (runtimeFn().isBrowser) {
+      url = URI.from("indexdb://fp-keybag");
+    } else {
+      url = URI.merge(`file://./dist/tests/key.bag`, sthis.env.get("FP_KEYBAG_URL"));
+    }
+    kb = await rt.kb.getKeyBag(sthis, {
+      url,
+    });
+    keyStr = base58btc.encode(kb.rt.crypto.randomBytes(kb.rt.keyLength));
+    kycr = await rt.kc.keyedCryptoFactory(URI.from(`test://bla?storekey=${keyStr}`), kb, sthis);
+  });
+  it("codec explict iv", async () => {
+    const testData = kb.rt.crypto.randomBytes(1024);
+    const iv = kb.rt.crypto.randomBytes(12);
+    const codec = kycr.codec(iv, { noIVVerify: true });
+    const blk = (await codec.encode(testData)) as Uint8Array;
+    const myDec = await rt.mf.block.decode<bs.IvKeyIdData, number, number>({ bytes: blk, hasher, codec: dagCodec });
+    expect(myDec.value.iv).toEqual(iv);
+    expect(base58btc.encode(myDec.value.keyId)).toEqual(await kycr.fingerPrint());
+    const dec = await codec.decode(blk);
+    expect(dec).toEqual(testData);
+  });
+
+  it("codec implict iv", async () => {
+    const testData = kb.rt.crypto.randomBytes(1024);
+    const codec = kycr.codec();
+    const blk = await codec.encode(testData);
+    expect(blk.length).toBeGreaterThanOrEqual(12 + testData.length);
+    const dec = await codec.decode(blk);
+    expect(dec).toEqual(testData);
+  });
+
+  it("codec implict iv same for multiple clients", async () => {
+    const testData = kb.rt.crypto.randomBytes(1024);
+    const codec = kycr.codec();
+    const blk = await codec.encode(testData);
+    const blk2 = await codec.encode(testData);
+    expect(blk).toEqual(blk2);
+  });
+});
+
+// describe("KeyedCryptoStore RunLength", () => {
+//   const logger = MockLogger().logger;
+//   it("de/encode", () => {
+//     for (const data of [
+//       new Uint8Array(),
+//       new Uint8Array(10).fill(10),
+//       new Uint8Array(127).fill(127),
+//       new Uint8Array(128).fill(128),
+//       new Uint8Array(1024).fill(17),
+//     ]) {
+//       const res = rt.kc.encodeRunLength(data, logger);
+//       expect(res.length).toBeLessThanOrEqual(data.length + (data.length > 127 ? 4 : 1));
+//       for (let ofs = 0; ofs < 1024; ofs += 61) {
+//         const ofsRes = new Uint8Array([...new Uint8Array(ofs).fill(23), ...res]);
+//         const dec = rt.kc.decodeRunLength(ofsRes, ofs, logger);
+//         expect(dec.data).toEqual(data);
+//         expect(dec.data.length).toBe(data.length);
+//         expect(dec.next).toBe(ofs + data.length + (data.length > 127 ? 4 : 1));
+//       }
+//     }
+//   });
+// });

package/tests/blockstore/loader.test.ts

@@ -1,18 +1,20 @@
 import * as codec from "@ipld/dag-cbor";
 import { sha256 as hasher } from "multiformats/hashes/sha2";
 import { BlockView } from "multiformats";
-import { encode } from "multiformats/block";
 import { CID } from "multiformats/cid";
 import { MemoryBlockstore } from "@web3-storage/pail/block";
-import { CRDTMeta, IndexTransactionMeta, bs, rt } from "@fireproof/core";
+import { CRDTMeta, IndexTransactionMeta, SuperThis, bs, rt } from "@fireproof/core";
+import { mockSuperThis } from "../helpers";
 
 class MyMemoryBlockStore extends bs.EncryptedBlockstore {
   readonly memblock = new MemoryBlockstore();
-  constructor() {
+  loader: bs.Loader;
+  constructor(sthis: SuperThis) {
     const ebOpts = {
       name: "MyMemoryBlockStore",
     };
-    super(ebOpts);
+    super(sthis, ebOpts);
+    this.loader = new bs.Loader("MyMemoryBlockStore", {}, sthis);
   }
   ready(): Promise<void> {
     return Promise.resolve();
@@ -20,7 +22,6 @@ class MyMemoryBlockStore extends bs.EncryptedBlockstore {
   close(): Promise<void> {
     return this.loader.close();
   }
-  loader = new bs.Loader("MyMemoryBlockStore", {});
   readonly transactions = new Set<bs.CarTransaction>();
   // readonly lastTxMeta?: TransactionMeta;
   readonly compacting: boolean = false;
@@ -50,6 +51,7 @@ describe("basic Loader simple", function () {
   let loader: bs.Loader;
   let block: BlockView;
   let t: bs.CarTransaction;
+  const sthis = mockSuperThis();
 
   afterEach(async function () {
     await loader.close();
@@ -58,12 +60,12 @@ describe("basic Loader simple", function () {
 
   beforeEach(async function () {
     const testDbName = "test-loader-commit";
-    await rt.SysContainer.start();
-    const mockM = new MyMemoryBlockStore();
+    await sthis.start();
+    const mockM = new MyMemoryBlockStore(sthis);
     t = new bs.CarTransaction(mockM as bs.EncryptedBlockstore);
-    loader = new bs.Loader(testDbName, { public: true });
+    loader = new bs.Loader(testDbName, { public: true }, sthis);
     await loader.ready();
-    block = await encode({
+    block = await rt.mf.block.encode({
       value: { hello: "world" },
       hasher,
       codec,
@@ -97,17 +99,19 @@ describe("basic Loader with two commits", function () {
   let carCid: bs.CarGroup;
   let carCid0: bs.CarGroup;
 
+  const sthis = mockSuperThis();
+
   afterEach(async function () {
     await loader.close();
     await loader.destroy();
   });
 
   beforeEach(async function () {
-    await rt.SysContainer.start();
-    const mockM = new MyMemoryBlockStore();
+    await sthis.start();
+    const mockM = new MyMemoryBlockStore(sthis);
     t = new bs.CarTransaction(mockM);
-    loader = new bs.Loader("test-loader-two-commit", { public: true });
-    block = await encode({
+    loader = new bs.Loader("test-loader-two-commit", { public: true }, sthis);
+    block = await rt.mf.block.encode({
       value: { hello: "world" },
       hasher,
       codec,
@@ -115,7 +119,7 @@ describe("basic Loader with two commits", function () {
     await t.put(block.cid, block.bytes);
     carCid0 = await loader.commit(t, { head: [block.cid] });
 
-    block2 = await encode({
+    block2 = await rt.mf.block.encode({
       value: { hello: "universe" },
       hasher,
       codec,
@@ -123,14 +127,14 @@ describe("basic Loader with two commits", function () {
     await t.put(block2.cid, block2.bytes);
     carCid = await loader.commit(t, { head: [block2.cid] });
 
-    block3 = await encode({
+    block3 = await rt.mf.block.encode({
       value: { hello: "multiverse" },
       hasher,
       codec,
     });
     await t.put(block3.cid, block3.bytes);
 
-    block4 = await encode({
+    block4 = await rt.mf.block.encode({
       value: { hello: "megaverse" },
       hasher,
       codec,
@@ -197,6 +201,7 @@ describe("basic Loader with index commits", function () {
   let indexerResult: IndexTransactionMeta;
   let cid: CID;
   // let indexMap: Map<string, CID>;
+  const sthis = mockSuperThis();
 
   afterEach(async function () {
     await ib.close();
@@ -205,10 +210,10 @@ describe("basic Loader with index commits", function () {
 
   beforeEach(async function () {
     const name = "test-loader-index" + Math.random();
-    await rt.SysContainer.start();
+    await sthis.start();
     // t = new CarTransaction()
-    ib = new bs.EncryptedBlockstore({ name });
-    block = await encode({
+    ib = new bs.EncryptedBlockstore(sthis, { name });
+    block = await rt.mf.block.encode({
       value: { hello: "world" },
       hasher,
       codec,