@fireproof/core-base 0.0.0-smoke-1b31059-1752074105

package/index.ts

@@ -0,0 +1,9 @@
+export * from "./ledger.js";
+export * from "./database.js";
+
+export * from "./crdt.js";
+
+export * from "./indexer.js";
+export * from "./indexer-helpers.js";
+
+export * from "./version.js";

package/indexer-helpers.d.ts

@@ -0,0 +1,25 @@
+import { DocUpdate, MapFn, DocFragment, IndexUpdate, QueryOpts, IndexRows, IndexKeyType, IndexKey, DocTypes, DocObject, IndexUpdateString, CarTransaction, CRDT, IndexTree } from "@fireproof/core-types-base";
+import { BlockFetcher, AnyLink } from "@fireproof/core-types-blockstore";
+import { Logger } from "@adviser/cement";
+import { StaticProllyOptions, BaseNode as ProllyNode, IndexRow } from "prolly-trees/base";
+type CompareRef = string | number;
+export type CompareKey = [string | number, CompareRef];
+export declare const byKeyOpts: StaticProllyOptions<CompareKey>;
+export declare const byIdOpts: StaticProllyOptions<string | number>;
+export interface IndexDoc<K extends IndexKeyType> {
+    readonly key: IndexKey<K>;
+    readonly value: DocFragment;
+}
+export interface IndexDocString {
+    readonly key: string;
+    readonly value: DocFragment;
+}
+export declare function indexEntriesForChanges<T extends DocTypes, K extends IndexKeyType>(changes: DocUpdate<T>[], mapFn: MapFn<T>): IndexDoc<K>[];
+export declare function bulkIndex<K extends IndexKeyType, T extends DocFragment, CT>(logger: Logger, tblocks: CarTransaction, inIndex: IndexTree<K, T>, indexEntries: (IndexUpdate<K> | IndexUpdateString)[], opts: StaticProllyOptions<CT>): Promise<IndexTree<K, T>>;
+export declare function loadIndex<K extends IndexKeyType, T extends DocFragment, CT>(tblocks: BlockFetcher, cid: AnyLink, opts: StaticProllyOptions<CT>): Promise<ProllyNode<K, T>>;
+export declare function applyQuery<T extends DocObject, K extends IndexKeyType, R extends DocFragment>(crdt: CRDT, resp: {
+    result: IndexRow<K, R>[];
+}, query: QueryOpts<K>): Promise<IndexRows<T, K, R>>;
+export declare function encodeRange(range: [IndexKeyType, IndexKeyType]): [string, string];
+export declare function encodeKey(key: DocFragment): string;
+export {};

package/indexer-helpers.js

@@ -0,0 +1,155 @@
+import { sha256 as hasher } from "multiformats/hashes/sha2";
+import * as codec from "@ipld/dag-cbor";
+import charwise from "charwise";
+import * as DbIndex from "prolly-trees/db-index";
+import { bf, simpleCompare } from "prolly-trees/utils";
+import { nocache as cache } from "prolly-trees/cache";
+import { anyBlock2FPBlock } from "@fireproof/core-blockstore";
+import { asyncBlockCreate } from "@fireproof/core-runtime";
+function refCompare(aRef, bRef) {
+    if (Number.isNaN(aRef))
+        return -1;
+    if (Number.isNaN(bRef))
+        throw new Error("ref may not be Infinity or NaN");
+    if (aRef === Infinity)
+        return 1;
+    return simpleCompare(aRef, bRef);
+}
+function compare(a, b) {
+    const [aKey, aRef] = a;
+    const [bKey, bRef] = b;
+    const comp = simpleCompare(aKey, bKey);
+    if (comp !== 0)
+        return comp;
+    return refCompare(aRef, bRef);
+}
+export const byKeyOpts = { cache, chunker: bf(30), codec, hasher, compare };
+export const byIdOpts = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare };
+export function indexEntriesForChanges(changes, mapFn) {
+    const indexEntries = [];
+    changes.forEach(({ id: key, value, del }) => {
+        if (del || !value)
+            return;
+        let mapCalled = false;
+        const mapReturn = mapFn({ ...value, _id: key }, (k, v) => {
+            mapCalled = true;
+            if (typeof k === "undefined")
+                return;
+            indexEntries.push({
+                key: [charwise.encode(k), key],
+                value: v || null,
+            });
+        });
+        if (!mapCalled && typeof mapReturn !== "undefined") {
+            indexEntries.push({
+                key: [charwise.encode(mapReturn), key],
+                value: null,
+            });
+        }
+    });
+    return indexEntries;
+}
+function makeProllyGetBlock(blocks) {
+    return async (address) => {
+        const block = await blocks.get(address);
+        if (!block)
+            throw new Error(`Missing block ${address.toString()}`);
+        const { cid, bytes } = block;
+        return asyncBlockCreate({ cid, bytes, hasher, codec });
+    };
+}
+export async function bulkIndex(logger, tblocks, inIndex, indexEntries, opts) {
+    logger.Debug().Msg("enter bulkIndex");
+    if (!indexEntries.length)
+        return inIndex;
+    if (!inIndex.root) {
+        if (!inIndex.cid) {
+            let returnRootBlock = undefined;
+            let returnNode = undefined;
+            for await (const node of (await DbIndex.create({
+                get: makeProllyGetBlock(tblocks),
+                list: indexEntries,
+                ...opts,
+            }))) {
+                const block = await node.block;
+                await tblocks.put(await anyBlock2FPBlock(block));
+                returnRootBlock = block;
+                returnNode = node;
+            }
+            if (!returnNode || !returnRootBlock)
+                throw new Error("failed to create index");
+            logger.Debug().Msg("exit !root bulkIndex");
+            return { root: returnNode, cid: returnRootBlock.cid };
+        }
+        else {
+            inIndex.root = (await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts }));
+        }
+    }
+    logger.Debug().Msg("pre bulk bulkIndex");
+    const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
+    if (root) {
+        logger.Debug().Msg("pre root put bulkIndex");
+        for await (const block of newBlocks) {
+            await tblocks.put(await anyBlock2FPBlock(block));
+        }
+        return { root, cid: (await root.block).cid };
+    }
+    else {
+        logger.Debug().Msg("pre !root bulkIndex");
+        return { root: undefined, cid: undefined };
+    }
+}
+export async function loadIndex(tblocks, cid, opts) {
+    return (await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts }));
+}
+export async function applyQuery(crdt, resp, query) {
+    if (query.descending) {
+        resp.result = resp.result.reverse();
+    }
+    if (query.limit) {
+        resp.result = resp.result.slice(0, query.limit);
+    }
+    if (query.includeDocs) {
+        resp.result = await Promise.all(resp.result.map(async (row) => {
+            const val = await crdt.get(row.id);
+            const doc = val ? { ...val.doc, _id: row.id } : undefined;
+            return { ...row, doc };
+        }));
+    }
+    const rows = resp.result.map(({ key, ...row }) => {
+        const decodedKey = charwise.decode(key);
+        const dynamicRow = row;
+        if ("row" in dynamicRow && !("value" in dynamicRow)) {
+            const normalizedRow = {};
+            Object.keys(dynamicRow).forEach((k) => {
+                if (k === "row") {
+                    normalizedRow.value = dynamicRow[k];
+                }
+                else {
+                    normalizedRow[k] = dynamicRow[k];
+                }
+            });
+            return {
+                key: decodedKey,
+                ...normalizedRow,
+            };
+        }
+        return {
+            key: decodedKey,
+            ...row,
+        };
+    });
+    const typedRows = rows;
+    const docs = typedRows.filter((r) => !!r.doc);
+    return {
+        rows: typedRows,
+        docs,
+    };
+}
+export function encodeRange(range) {
+    return [charwise.encode(range[0]), charwise.encode(range[1])];
+}
+export function encodeKey(key) {
+    return charwise.encode(key);
+}
+//# sourceMappingURL=indexer-helpers.js.map

package/indexer-helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer-helpers.js","sourceRoot":"","sources":["indexer-helpers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,KAAK,KAAK,MAAM,gBAAgB,CAAC;AAGxC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,KAAK,OAAO,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,OAAO,IAAI,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAsBtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAE9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAK3D,SAAS,UAAU,CAAC,IAAgB,EAAE,IAAgB;IACpD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IAClC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC1E,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IAGhC,OAAO,aAAa,CAAC,IAAI,EAAE,IAAI,CAAW,CAAC;AAC7C,CAAC;AAED,SAAS,OAAO,CAAC,CAAa,EAAE,CAAa;IAC3C,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,IAAI,GAAW,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC/C,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5B,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAChC,CAAC;AAID,MAAM,CAAC,MAAM,SAAS,GAAoC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAE7G,MAAM,CAAC,MAAM,QAAQ,GAAyC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAYhI,MAAM,UAAU,sBAAsB,CACpC,OAAuB,EACvB,KAAe;IAEf,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE;QAC1C,IAAI,GAAG,IAAI,CAAC,KAAK;YAAE,OAAO;QAC1B,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,MAAM,SAAS,GAAG,KAAK,CAAC,EAAE,GAAI,KAAsB,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAe,EAAE,CAAe,EAAE,EAAE;YACrG,SAAS,GAAG,IAAI,CAAC;YACjB,IAAI,OAAO,CAAC,KAAK,WAAW;gBAAE,OAAO;YACrC,YAAY,CAAC,IAAI,CAAC;gBAChB,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,GAAG,CAAC;gBACnC,KAAK,EAAE,CAAC,IAAI,IAAI;aACjB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,WAAW,EAAE,CAAC;YACnD,YAAY,CAAC,IAAI,CAAC;gBAChB,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAM,EAAE,GAAG,CAAC;gBAC3C,KAAK,EAAE,IAAI;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAoB;IAC9C,OAAO,KAAK,EAAE,OAAgB,EAAE,EAAE;QAChC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;QAC7B,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAsB,CAAC;IAC9E,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAc,EACd,OAAuB,EACvB,OAAwB,EACxB,YAAoD,EACpD,IAA6B;IAE7B,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IACtC,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YACjB,IAAI,eAAe,GAAsB,SAAS,CAAC;YACnD,IAAI,UAAU,GAAiC,SAAS,CAAC;YAEzD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC;gBAC7C,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC;gBAChC,IAAI,EAAE,YAAY;gBAClB,GAAG,IAAI;aACR,CAAC,CAAuB,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC;gBAC/B,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,eAAe,GAAG,KAAK,CAAC;gBACxB,UAAU,GAAG,IAAI,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,UAAU,IAAI,CAAC,eAAe;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC/E,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,eAAe,CAAC,GAAG,EAAE,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAqB,CAAC;QAC3H,CAAC;IACH,CAAC;IACD,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1E,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;QAC1C,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;IAC7C,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,OAAqB,EACrB,GAAY,EACZ,IAA6B;IAE7B,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,kBAAkB,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAqB,CAAC;AACtG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAU,EACV,IAAkC,EAClC,KAAmB;IAEnB,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACtC,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAC5B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAE,EAAE,GAAG,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAmB,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5E,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC;QACzB,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE;QAE/C,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAIxC,MAAM,UAAU,GAAG,GAA8B,CAAC;QAClD,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,CAAC,OAAO,IAAI,UAAU,CAAC,EAAE,CAAC;YAGpD,MAAM,aAAa,GAA4B,EAAE,CAAC;YAClD,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;oBAChB,aAAa,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBACtC,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO;gBACL,GAAG,EAAE,UAAU;gBACf,GAAG,aAAa;aACC,CAAC;QACtB,CAAC;QAGD,OAAO;YACL,GAAG,EAAE,UAAU;YACf,GAAG,GAAG;SACW,CAAC;IACtB,CAAC,CAAC,CAAC;IAGH,MAAM,SAAS,GAAG,IAA6B,CAAC;IAGhD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAA8B,CAAC;IAE3E,OAAO;QACL,IAAI,EAAE,SAAS;QACf,IAAI;KACL,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAgB;IACxC,OAAO,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAW,CAAC;AACxC,CAAC"}

package/indexer-helpers.ts

@@ -0,0 +1,263 @@
+/// <reference types="@fireproof/core-types-base/prolly-trees.d.ts" />
+import type { Block } from "multiformats";
+import { sha256 as hasher } from "multiformats/hashes/sha2";
+import * as codec from "@ipld/dag-cbor";
+
+// @ts-expect-error "charwise" has no types
+import charwise from "charwise";
+import * as DbIndex from "prolly-trees/db-index";
+import { bf, simpleCompare } from "prolly-trees/utils";
+import { nocache as cache } from "prolly-trees/cache";
+
+import {
+  DocUpdate,
+  MapFn,
+  DocFragment,
+  IndexUpdate,
+  QueryOpts,
+  IndexRows,
+  DocWithId,
+  IndexKeyType,
+  IndexKey,
+  DocTypes,
+  DocObject,
+  IndexUpdateString,
+  CarTransaction,
+  CRDT,
+  IndexTree,
+  FPIndexRow,
+} from "@fireproof/core-types-base";
+import { BlockFetcher, AnyLink, AnyBlock } from "@fireproof/core-types-blockstore";
+import { Logger } from "@adviser/cement";
+import { anyBlock2FPBlock } from "@fireproof/core-blockstore";
+import { StaticProllyOptions, BaseNode as ProllyNode, IndexRow } from "prolly-trees/base";
+import { asyncBlockCreate } from "@fireproof/core-runtime";
+
+type CompareRef = string | number;
+export type CompareKey = [string | number, CompareRef];
+
+function refCompare(aRef: CompareRef, bRef: CompareRef) {
+  if (Number.isNaN(aRef)) return -1;
+  if (Number.isNaN(bRef)) throw new Error("ref may not be Infinity or NaN");
+  if (aRef === Infinity) return 1;
+  // if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
+
+  return simpleCompare(aRef, bRef) as number;
+}
+
+function compare(a: CompareKey, b: CompareKey) {
+  const [aKey, aRef] = a;
+  const [bKey, bRef] = b;
+
+  const comp: number = simpleCompare(aKey, bKey);
+  if (comp !== 0) return comp;
+  return refCompare(aRef, bRef);
+}
+
+// declare function bf<T>(factor: number): (entry: T, dist: number) => Promise<boolean>;
+// chunker: (entry: T, distance: number) => Promise<boolean>;
+export const byKeyOpts: StaticProllyOptions<CompareKey> = { cache, chunker: bf(30), codec, hasher, compare };
+
+export const byIdOpts: StaticProllyOptions<string | number> = { cache, chunker: bf(30), codec, hasher, compare: simpleCompare };
+
+export interface IndexDoc<K extends IndexKeyType> {
+  readonly key: IndexKey<K>;
+  readonly value: DocFragment;
+}
+
+export interface IndexDocString {
+  readonly key: string;
+  readonly value: DocFragment;
+}
+
+export function indexEntriesForChanges<T extends DocTypes, K extends IndexKeyType>(
+  changes: DocUpdate<T>[],
+  mapFn: MapFn<T>,
+): IndexDoc<K>[] {
+  const indexEntries: IndexDoc<K>[] = [];
+  changes.forEach(({ id: key, value, del }) => {
+    if (del || !value) return;
+    let mapCalled = false;
+    const mapReturn = mapFn({ ...(value as DocWithId<T>), _id: key }, (k: IndexKeyType, v?: DocFragment) => {
+      mapCalled = true;
+      if (typeof k === "undefined") return;
+      indexEntries.push({
+        key: [charwise.encode(k) as K, key],
+        value: v || null,
+      });
+    });
+    if (!mapCalled && typeof mapReturn !== "undefined") {
+      indexEntries.push({
+        key: [charwise.encode(mapReturn) as K, key],
+        value: null,
+      });
+    }
+  });
+  return indexEntries;
+}
+
+function makeProllyGetBlock(blocks: BlockFetcher): (address: AnyLink) => Promise<AnyBlock> {
+  return async (address: AnyLink) => {
+    const block = await blocks.get(address);
+    if (!block) throw new Error(`Missing block ${address.toString()}`);
+    const { cid, bytes } = block;
+    return asyncBlockCreate({ cid, bytes, hasher, codec }) as Promise<AnyBlock>;
+  };
+}
+
+export async function bulkIndex<K extends IndexKeyType, T extends DocFragment, CT>(
+  logger: Logger,
+  tblocks: CarTransaction,
+  inIndex: IndexTree<K, T>,
+  indexEntries: (IndexUpdate<K> | IndexUpdateString)[],
+  opts: StaticProllyOptions<CT>,
+): Promise<IndexTree<K, T>> {
+  logger.Debug().Msg("enter bulkIndex");
+  if (!indexEntries.length) return inIndex;
+  if (!inIndex.root) {
+    if (!inIndex.cid) {
+      let returnRootBlock: Block | undefined = undefined;
+      let returnNode: ProllyNode<K, T> | undefined = undefined;
+
+      for await (const node of (await DbIndex.create({
+        get: makeProllyGetBlock(tblocks),
+        list: indexEntries,
+        ...opts,
+      })) as ProllyNode<K, T>[]) {
+        const block = await node.block;
+        await tblocks.put(await anyBlock2FPBlock(block));
+        returnRootBlock = block;
+        returnNode = node;
+      }
+      if (!returnNode || !returnRootBlock) throw new Error("failed to create index");
+      logger.Debug().Msg("exit !root bulkIndex");
+      return { root: returnNode, cid: returnRootBlock.cid };
+    } else {
+      inIndex.root = (await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts })) as ProllyNode<K, T>;
+    }
+  }
+  logger.Debug().Msg("pre bulk bulkIndex");
+  const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
+  if (root) {
+    logger.Debug().Msg("pre root put bulkIndex");
+    for await (const block of newBlocks) {
+      await tblocks.put(await anyBlock2FPBlock(block));
+    }
+    return { root, cid: (await root.block).cid };
+  } else {
+    logger.Debug().Msg("pre !root bulkIndex");
+    return { root: undefined, cid: undefined };
+  }
+}
+
+export async function loadIndex<K extends IndexKeyType, T extends DocFragment, CT>(
+  tblocks: BlockFetcher,
+  cid: AnyLink,
+  opts: StaticProllyOptions<CT>,
+): Promise<ProllyNode<K, T>> {
+  return (await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts })) as ProllyNode<K, T>;
+}
+
+export async function applyQuery<T extends DocObject, K extends IndexKeyType, R extends DocFragment>(
+  crdt: CRDT,
+  resp: { result: IndexRow<K, R>[] },
+  query: QueryOpts<K>,
+): Promise<IndexRows<T, K, R>> {
+  if (query.descending) {
+    resp.result = resp.result.reverse();
+  }
+  if (query.limit) {
+    resp.result = resp.result.slice(0, query.limit);
+  }
+  if (query.includeDocs) {
+    resp.result = await Promise.all(
+      resp.result.map(async (row) => {
+        const val = await crdt.get(row.id);
+        const doc = val ? ({ ...val.doc, _id: row.id } as DocWithId<T>) : undefined;
+        return { ...row, doc };
+      }),
+    );
+  }
+  const rows = resp.result.map(({ key, ...row }) => {
+    // First decode the key
+    const decodedKey = charwise.decode(key);
+
+    // Use a type-safe approach with Record to check for potentially missing properties
+    // This handles the case where some query results use 'row' instead of 'value'
+    const dynamicRow = row as Record<string, unknown>;
+    if ("row" in dynamicRow && !("value" in dynamicRow)) {
+      // We found a result with 'row' property but no 'value' property
+      // Create a new normalized object with the 'value' property
+      const normalizedRow: Record<string, unknown> = {};
+      Object.keys(dynamicRow).forEach((k) => {
+        if (k === "row") {
+          normalizedRow.value = dynamicRow[k];
+        } else {
+          normalizedRow[k] = dynamicRow[k];
+        }
+      });
+
+      return {
+        key: decodedKey,
+        ...normalizedRow,
+      } as IndexRow<K, R>;
+    }
+
+    // Standard case - use the properties as they are
+    return {
+      key: decodedKey,
+      ...row,
+    } as IndexRow<K, R>;
+  });
+
+  // We need to be explicit about the document types here
+  const typedRows = rows as FPIndexRow<K, T, R>[];
+
+  // Simply filter out null/undefined docs and cast the result
+  const docs = typedRows.filter((r) => !!r.doc) as unknown as DocWithId<T>[];
+
+  return {
+    rows: typedRows,
+    docs,
+  };
+}
+
+export function encodeRange(range: [IndexKeyType, IndexKeyType]): [string, string] {
+  return [charwise.encode(range[0]), charwise.encode(range[1])];
+}
+
+export function encodeKey(key: DocFragment): string {
+  return charwise.encode(key) as string;
+}
+
+// export interface ProllyIndexRow<K extends IndexKeyType, T extends DocFragment> {
+//   readonly id: string;
+//   readonly key: IndexKey<K>;
+//   readonly value: T;
+//   readonly doc?: DocWithId<DocObject>;
+// }
+
+// // ProllyNode type based on the ProllyNode from 'prolly-trees/base'
+// interface ProllyNode<K extends IndexKeyType, T extends DocFragment> extends BaseNode<K, T> {
+//   getAllEntries(): PromiseLike<{ [x: string]: unknown; result: ProllyIndexRow<K, T>[] }>;
+//   getMany<KI extends IndexKeyType>(removeIds: KI[]): Promise<{ /* [x: K]: unknown; */ result: IndexKey<K>[] }>;
+//   range(a: string, b: string): Promise<{ result: ProllyIndexRow<K, T>[] }>;
+//   get(key: string): Promise<{ result: ProllyIndexRow<K, T>[] }>;
+//   bulk(bulk: (IndexUpdate<K> | IndexUpdateString)[]): PromiseLike<{
+//     readonly root?: ProllyNode<K, T>;
+//     readonly blocks: Block[];
+//   }>;
+//   readonly address: Promise<Link>;
+//   readonly distance: number;
+//   compare: (a: unknown, b: unknown) => number;
+//   readonly cache: unknown;
+//   readonly block: Promise<Block>;
+// }
+
+// interface StaticProllyOptions<T> {
+//   readonly cache: unknown;
+//   chunker: (entry: T, distance: number) => boolean;
+//   readonly codec: unknown;
+//   readonly hasher: unknown;
+//   compare: (a: T, b: T) => number;
+// }

package/indexer.d.ts

@@ -0,0 +1,22 @@
+import { type ClockHead, type MapFn, type QueryOpts, type IdxMeta, type DocFragment, type IndexKeyType, type IndexRows, type DocTypes, type IndexTransactionMeta, type SuperThis, type BaseBlockstore, type CRDT, type HasCRDT, type HasLogger, type HasSuperThis, type RefLedger, IndexIf, IndexTree } from "@fireproof/core-types-base";
+import { Logger } from "@adviser/cement";
+export declare function index<T extends DocTypes = DocTypes, K extends IndexKeyType = string, R extends DocFragment = T>(refDb: HasLogger & HasSuperThis & (HasCRDT | RefLedger), name: string, mapFn?: MapFn<T>, meta?: IdxMeta): Index<T, K, R>;
+export declare class Index<T extends DocTypes, K extends IndexKeyType = string, R extends DocFragment = T> implements IndexIf<T, K, R> {
+    readonly blockstore: BaseBlockstore;
+    readonly crdt: CRDT;
+    readonly name: string;
+    mapFn?: MapFn<T>;
+    mapFnString: string;
+    byKey: IndexTree<K, R>;
+    byId: IndexTree<K, R>;
+    indexHead?: ClockHead;
+    initError?: Error;
+    ready(): Promise<void>;
+    readonly logger: Logger;
+    constructor(sthis: SuperThis, crdt: CRDT, name: string, mapFn?: MapFn<T>, meta?: IdxMeta);
+    applyMapFn(name: string, mapFn?: MapFn<T>, meta?: IdxMeta): void;
+    query(opts?: QueryOpts<K>): Promise<IndexRows<T, K, R>>;
+    _resetIndex(): void;
+    _hydrateIndex(): Promise<void>;
+    _updateIndex(): Promise<IndexTransactionMeta>;
+}