@fireproof/core-base 0.23.0 → 0.23.1-dev-issue-1057

package/crdt.ts

@@ -1,275 +0,0 @@
-import type { Block } from "multiformats";
-import { Logger, ResolveOnce } from "@adviser/cement";
-import { EncryptedBlockstore, toStoreRuntime } from "@fireproof/core-blockstore";
-import {
-  clockChangesSince,
-  applyBulkUpdateToCrdt,
-  getValueFromCrdt,
-  readFiles,
-  getAllEntries,
-  clockVis,
-  getBlock,
-  sanitizeDocumentFields,
-} from "./crdt-helpers.js";
-import {
-  type DocUpdate,
-  type CRDTMeta,
-  type ClockHead,
-  type ChangesOptions,
-  type DocValue,
-  type IndexKeyType,
-  type DocWithId,
-  type Falsy,
-  type SuperThis,
-  type IndexTransactionMeta,
-  type LedgerOpts,
-  type BaseBlockstore,
-  type CRDT,
-  type CRDTClock,
-  type CarTransaction,
-  type DocTypes,
-  PARAM,
-  Ledger,
-  TraceEvent,
-} from "@fireproof/core-types-base";
-import { index, type Index } from "./indexer.js";
-// import { blockstoreFactory } from "./blockstore/transaction.js";
-import { ensureLogger, getCompactStrategy } from "@fireproof/core-runtime";
-import { CRDTClockImpl } from "./crdt-clock.js";
-import { TransactionMeta, BlockstoreOpts } from "@fireproof/core-types-blockstore";
-
-export type CRDTOpts = Omit<LedgerOpts, "storeUrls"> & {
-  readonly storeUrls: {
-    readonly data: LedgerOpts["storeUrls"]["data"];
-    readonly idx?: LedgerOpts["storeUrls"]["idx"];
-  };
-};
-
-function tracerAction(opts: CRDTOpts, parent?: Ledger) {
-  return (event: TraceEvent) => {
-    switch (event.event) {
-      case "idleFromCommitQueue":
-        opts.tracer({
-          event: "idleFromBlockstore",
-          blockstore: "data",
-          ledger: parent,
-        });
-        break;
-      case "busyFromCommitQueue":
-        opts.tracer({
-          event: "busyFromBlockstore",
-          blockstore: "data",
-          ledger: parent,
-          queueLen: event.queueLen,
-        });
-        break;
-      default:
-        return opts.tracer(event);
-    }
-  };
-}
-
-export class CRDTImpl implements CRDT {
-  readonly opts: CRDTOpts;
-
-  readonly blockstore: BaseBlockstore;
-  // we can run without an index instance
-  readonly indexBlockstore?: BaseBlockstore;
-  readonly indexers = new Map<string, Index<DocTypes, IndexKeyType>>();
-  readonly clock: CRDTClock;
-
-  readonly logger: Logger;
-  readonly sthis: SuperThis;
-  // self reference to fullfill HasCRDT
-  readonly crdt: CRDT;
-
-  readonly ledgerParent?: Ledger;
-
-  constructor(sthis: SuperThis, opts: CRDTOpts, parent?: Ledger) {
-    this.sthis = sthis;
-    this.ledgerParent = parent;
-    this.crdt = this;
-    this.logger = ensureLogger(sthis, "CRDTImpl");
-    this.opts = opts;
-    const rCompactStrategy = getCompactStrategy(this.opts.compactStrategy);
-    if (rCompactStrategy.isErr()) {
-      throw this.logger
-        .Error()
-        .Err(rCompactStrategy.Err())
-        .Str("compactorName", this.opts.compactStrategy)
-        .Msg("compactor not found")
-        .AsError();
-    }
-    const blockstoreOpts = {
-      tracer: tracerAction(opts, parent),
-      applyMeta: async (meta: TransactionMeta) => {
-        const crdtMeta = meta as CRDTMeta;
-        if (!crdtMeta.head) throw this.logger.Error().Msg("missing head").AsError();
-        // console.log("applyMeta-pre", crdtMeta.head, this.clock.head);
-        await this.clock.applyHead(crdtMeta.head, []);
-        // console.log("applyMeta-post", crdtMeta.head, this.clock.head);
-      },
-      compactStrategy: rCompactStrategy.Ok(),
-      gatewayInterceptor: opts.gatewayInterceptor,
-      // autoCompact: this.opts.autoCompact || 100,
-      storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
-      storeUrls: this.opts.storeUrls.data,
-      keyBag: this.opts.keyBag,
-      // public: this.opts.public,
-      meta: this.opts.meta,
-      // threshold: this.opts.threshold,
-    } satisfies BlockstoreOpts;
-
-    this.blockstore = new EncryptedBlockstore(sthis, blockstoreOpts, this);
-    if (this.opts.storeUrls.idx) {
-      this.indexBlockstore = new EncryptedBlockstore(
-        sthis,
-        {
-          tracer: opts.tracer,
-          // name: opts.name,
-          applyMeta: async (meta: TransactionMeta) => {
-            const idxCarMeta = meta as IndexTransactionMeta;
-            if (!idxCarMeta.indexes) throw this.logger.Error().Msg("missing indexes").AsError();
-            for (const [name, idx] of Object.entries(idxCarMeta.indexes)) {
-              index(this, name, undefined, idx);
-            }
-          },
-          gatewayInterceptor: opts.gatewayInterceptor,
-          storeRuntime: toStoreRuntime(this.sthis, this.opts.storeEnDe),
-          storeUrls: this.opts.storeUrls.idx,
-          keyBag: this.opts.keyBag,
-          // public: this.opts.public,
-        },
-        this,
-      );
-    }
-    this.clock = new CRDTClockImpl(this.blockstore);
-    this.clock.onZoom(() => {
-      for (const idx of this.indexers.values()) {
-        idx._resetIndex();
-      }
-    });
-  }
-
-  async bulk<T extends DocTypes>(updates: DocUpdate<T>[]): Promise<CRDTMeta> {
-    await this.ready();
-    updates = updates.map((dupdate: DocUpdate<T>) => ({
-      ...dupdate,
-      value: sanitizeDocumentFields(dupdate.value),
-    }));
-
-    if (this.clock.head.length === 0) {
-      // INJECT GENESIS Block
-      const value = { id: PARAM.GENESIS_CID, value: { _id: PARAM.GENESIS_CID } }; // satisfies DocUpdate<DocSet<DocTypes>>;
-      // const block = await encode({ value, hasher: sha256, codec: dagCodec });
-      await this._bulk([value]);
-    }
-    return await this._bulk(updates);
-  }
-
-  async _bulk<T extends DocTypes>(updates: DocUpdate<T>[]): Promise<CRDTMeta> {
-    const prevHead = [...this.clock.head];
-    const done = await this.blockstore.transaction<CRDTMeta>(async (blocks: CarTransaction): Promise<CRDTMeta> => {
-      const { head } = await applyBulkUpdateToCrdt<DocTypes>(
-        this.blockstore.ebOpts.storeRuntime,
-        blocks,
-        this.clock.head,
-        updates,
-        this.logger,
-      );
-      updates = updates.map((dupdate: DocUpdate<T>) => {
-        // if (!dupdate.value) throw new Error("missing value");
-        readFiles(this.blockstore, { doc: dupdate.value as DocWithId<DocTypes> });
-        return dupdate;
-      });
-      return { head };
-    });
-    await this.clock.applyHead(done.meta.head, prevHead, updates);
-    return done.meta;
-  }
-
-  readonly onceReady: ResolveOnce<void> = new ResolveOnce<void>();
-  async ready(): Promise<void> {
-    return this.onceReady.once(async () => {
-      try {
-        // console.log("bs-ready-pre")
-        // await this.blockstore.ready();
-        // console.log("bs-ready-post-1")
-        // await this.indexBlockstore?.ready();
-        // console.log("bs-ready-post-2")
-        // await this.clock.ready();
-        // console.log("bs-ready-post-3")
-        await Promise.all([
-          this.blockstore.ready(),
-          this.indexBlockstore ? this.indexBlockstore.ready() : Promise.resolve(),
-          this.clock.ready(),
-        ]);
-      } catch (e) {
-        throw this.logger.Error().Err(e).Msg(`CRDT is not ready`).AsError();
-      }
-    });
-  }
-
-  async close(): Promise<void> {
-    // await this.blockstore.close();
-    // await this.indexBlockstore.close();
-    // await this.clock.close();
-    await Promise.all([
-      this.blockstore.close(),
-      this.indexBlockstore ? this.indexBlockstore.close() : Promise.resolve(),
-      this.clock.close(),
-    ]);
-  }
-
-  async destroy(): Promise<void> {
-    await Promise.all([this.blockstore.destroy(), this.indexBlockstore ? this.indexBlockstore.destroy() : Promise.resolve()]);
-  }
-
-  // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
-
-  async allDocs<T extends DocTypes>(): Promise<{ result: DocUpdate<T>[]; head: ClockHead }> {
-    await this.ready();
-    const result: DocUpdate<T>[] = [];
-    for await (const entry of getAllEntries<DocTypes>(this.blockstore, this.clock.head, this.logger)) {
-      result.push(entry as DocUpdate<T>);
-    }
-    return { result, head: this.clock.head };
-  }
-
-  async vis(): Promise<string> {
-    await this.ready();
-    const txt: string[] = [];
-    for await (const line of clockVis(this.blockstore, this.clock.head)) {
-      txt.push(line);
-    }
-    return txt.join("\n");
-  }
-
-  async getBlock(cidString: string): Promise<Block> {
-    await this.ready();
-    return await getBlock(this.blockstore, cidString);
-  }
-
-  async get(key: string): Promise<DocValue<DocTypes> | Falsy> {
-    await this.ready();
-    const result = await getValueFromCrdt<DocTypes>(this.blockstore, this.clock.head, key, this.logger);
-    if (result.del) return undefined;
-    return result;
-  }
-
-  async changes<T extends DocTypes>(
-    since: ClockHead = [],
-    opts: ChangesOptions = {},
-  ): Promise<{
-    result: DocUpdate<T>[];
-    head: ClockHead;
-  }> {
-    await this.ready();
-    return await clockChangesSince<T>(this.blockstore, this.clock.head, since, opts, this.logger);
-  }
-
-  async compact(): Promise<void> {
-    const blocks = this.blockstore as EncryptedBlockstore;
-    return await blocks.compact();
-  }
-}

package/database.ts

@@ -1,200 +0,0 @@
-import { Logger } from "@adviser/cement";
-import { index } from "./indexer.js";
-import {
-  ClockHead,
-  MapFn,
-  QueryOpts,
-  ChangesOptions,
-  DocSet,
-  DocWithId,
-  IndexKeyType,
-  ListenerFn,
-  DocResponse,
-  BulkResponse,
-  ChangesResponse,
-  DocTypes,
-  IndexRows,
-  DocFragment,
-  ChangesResponseRow,
-  CRDTMeta,
-  AllDocsQueryOpts,
-  AllDocsResponse,
-  SuperThis,
-  Database,
-  Ledger,
-  Attachable,
-  Attached,
-  NotFoundError,
-} from "@fireproof/core-types-base";
-import { ensureLogger, makeName } from "@fireproof/core-runtime";
-
-export function isDatabase(db: unknown): db is Database {
-  return db instanceof DatabaseImpl;
-}
-
-export class DatabaseImpl implements Database {
-  onClosed(fn: () => void) {
-    this.ledger.onClosed(fn);
-  }
-  close() {
-    return this.ledger.close();
-  }
-
-  destroy() {
-    return this.ledger.destroy();
-  }
-
-  ready(): Promise<void> {
-    return this.ledger.ready();
-  }
-
-  readonly ledger: Ledger;
-  readonly logger: Logger;
-  readonly sthis: SuperThis;
-  // readonly id: string;
-
-  constructor(ledger: Ledger) {
-    this.sthis = ledger.sthis;
-    this.ledger = ledger;
-    // this.id = ledger.id;
-    this.logger = ensureLogger(this.sthis, "Database");
-  }
-
-  attach(a: Attachable): Promise<Attached> {
-    return this.ledger.attach(a);
-  }
-
-  get name() {
-    return this.ledger.name;
-  }
-  async get<T extends DocTypes>(id: string): Promise<DocWithId<T>> {
-    if (!id) throw this.logger.Error().Str("db", this.name).Msg(`Doc id is required`).AsError();
-
-    await this.ready();
-    this.logger.Debug().Str("id", id).Msg("get");
-    try {
-      const got = await this.ledger.crdt.get(id);
-      if (!got) throw new NotFoundError(`Not found: ${id}`);
-      const { doc } = got;
-      return { ...(doc as unknown as DocWithId<T>), _id: id };
-    } catch (e) {
-      throw new NotFoundError(`Not found: ${id} - ${e instanceof Error ? e.message : String(e)}`);
-    }
-  }
-
-  async put<T extends DocTypes>(doc: DocSet<T>): Promise<DocResponse> {
-    await this.ready();
-    this.logger.Debug().Str("id", doc._id).Msg("put");
-    const { _id, ...value } = doc;
-    const docId = _id || this.sthis.timeOrderedNextId().str;
-    const result = (await this.ledger.writeQueue.push({
-      id: docId,
-      value: {
-        ...(value as unknown as DocSet<T>),
-        _id: docId,
-      },
-    })) as CRDTMeta;
-    return { id: docId, clock: result?.head, name: this.name } as DocResponse;
-  }
-
-  async bulk<T extends DocTypes>(docs: DocSet<T>[]): Promise<BulkResponse> {
-    await this.ready();
-
-    const updates = docs.map((doc) => {
-      const id = doc._id || this.sthis.timeOrderedNextId().str;
-      return {
-        id,
-        value: {
-          ...(doc as unknown as DocSet<T>),
-          _id: id,
-        },
-      };
-    });
-    const result = (await this.ledger.writeQueue.bulk(updates)) as CRDTMeta;
-    return { ids: updates.map((u) => u.id), clock: result.head, name: this.name } as BulkResponse;
-  }
-
-  async del(id: string): Promise<DocResponse> {
-    await this.ready();
-    this.logger.Debug().Str("id", id).Msg("del");
-    const result = (await this.ledger.writeQueue.push({ id: id, del: true })) as CRDTMeta;
-    return { id, clock: result?.head, name: this.name } as DocResponse;
-  }
-
-  async remove(id: string): Promise<DocResponse> {
-    return this.del(id);
-  }
-
-  async changes<T extends DocTypes>(since: ClockHead = [], opts: ChangesOptions = {}): Promise<ChangesResponse<T>> {
-    await this.ready();
-    this.logger.Debug().Any("since", since).Any("opts", opts).Msg("changes");
-    const { result, head } = await this.ledger.crdt.changes(since, opts);
-    const rows: ChangesResponseRow<T>[] = result.map(({ id: key, value, del, clock }) => ({
-      key,
-      value: (del ? { _id: key, _deleted: true } : { _id: key, ...value }) as DocWithId<T>,
-      clock,
-    }));
-    return { rows, clock: head, name: this.name };
-  }
-
-  async allDocs<T extends DocTypes>(opts: Partial<AllDocsQueryOpts> = {}): Promise<AllDocsResponse<T>> {
-    await this.ready();
-    this.logger.Debug().Msg("allDocs");
-    const { result, head } = await this.ledger.crdt.allDocs();
-
-    // Map all docs to the expected format
-    let rows = result.map(({ id: key, value, del }) => ({
-      key,
-      value: (del ? { _id: key, _deleted: true } : { _id: key, ...value }) as DocWithId<T>,
-    }));
-
-    // Filter out deleted documents unless includeDeleted is true
-    if (!opts.includeDeleted) {
-      rows = rows.filter((row) => !(row.value as DocWithId<T> & { _deleted?: boolean })._deleted);
-    }
-
-    // Apply limit if specified
-    if (typeof opts.limit === "number" && opts.limit >= 0) {
-      rows = rows.slice(0, opts.limit);
-    }
-
-    return { rows, clock: head, name: this.name };
-  }
-
-  async allDocuments<T extends DocTypes>(): Promise<{
-    rows: {
-      key: string;
-      value: DocWithId<T>;
-    }[];
-    clock: ClockHead;
-  }> {
-    return this.allDocs<T>();
-  }
-
-  subscribe<T extends DocTypes>(listener: ListenerFn<T>, updates?: boolean): () => void {
-    return this.ledger.subscribe(listener, updates);
-  }
-
-  // todo if we add this onto dbs in fireproof.ts then we can make index.ts a separate package
-  async query<T extends DocTypes, K extends IndexKeyType = string, R extends DocFragment = T>(
-    field: string | MapFn<T>,
-    opts: QueryOpts<K> = {},
-  ): Promise<IndexRows<T, K, R>> {
-    await this.ready();
-    this.logger.Debug().Any("field", field).Any("opts", opts).Msg("query");
-    // const _crdt = this.ledger.crdt as unknown as CRDT<T>;
-    const idx = typeof field === "string" ? index<T, K, R>(this, field) : index<T, K, R>(this, makeName(field.toString()), field);
-    const result = await idx.query(opts);
-
-    // Add docs property to match useLiveQuery behavior
-    return {
-      rows: result.rows,
-      docs: result.rows.map((r) => r.doc).filter((r): r is DocWithId<T> => !!r),
-    };
-  }
-
-  async compact() {
-    await this.ready();
-    await this.ledger.crdt.compact();
-  }
-}

package/index.ts

@@ -1,15 +0,0 @@
-export * from "./ledger.js";
-export * from "./database.js";
-
-export * from "./crdt.js";
-
-export * from "./indexer.js";
-export * from "./indexer-helpers.js";
-
-export * from "./version.js";
-
-import "./compact-strategies.js";
-
-export { getCompactStrategy, registerCompactStrategy, getCompactStrategyThrow } from "@fireproof/core-runtime";
-
-export { registerStoreProtocol } from "@fireproof/core-blockstore";