@peerbit/indexer-interface 0.0.1-cccc078

package/dist/src/utils.js

@@ -0,0 +1,88 @@
+import { field, getSchema, } from "@dao-xyz/borsh";
+import { SortDirection } from "./query.js";
+export const stringArraysEquals = (a, b) => {
+    if (a.length !== b.length) {
+        return false;
+    }
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i]) {
+            return false;
+        }
+    }
+    return true;
+};
+/* export const resolvedSort = async <
+    Q extends { indexed: Record<string, any> }
+>(
+    arr: Q[],
+    sorts: Sort[],
+    aliases?: Map<string, string>
+) => {
+    arr.sort((a, b) => extractSortCompare(a.indexed, b.indexed, sorts));
+    return arr;
+};
+ */
+export const extractSortCompare = (a, b, sorts, aliases) => {
+    for (const sort of sorts) {
+        const av = extractFieldValue(a, sort.key, aliases);
+        const bv = extractFieldValue(b, sort.key, aliases);
+        const cmp = sortCompare(av, bv);
+        if (cmp !== 0) {
+            if (sort.direction === SortDirection.ASC) {
+                return cmp;
+            }
+            else {
+                return -cmp;
+            }
+        }
+    }
+    return 0;
+};
+export const sortCompare = (av, bv) => {
+    if (typeof av === "string" && typeof bv === "string") {
+        return av.localeCompare(bv);
+    }
+    if (av < bv) {
+        return -1;
+    }
+    else if (av > bv) {
+        return 1;
+    }
+    return 0;
+};
+export const extractFieldValue = (doc, path, aliases) => {
+    for (let i = 0; i < path.length; i++) {
+        doc = doc[aliases?.get(path[i]) || path[i]];
+    }
+    return doc;
+};
+const INDEXED_ID_META_PROPERY = "_index_id";
+export function id(properties) {
+    const innerFn = field(properties);
+    return (target, name) => {
+        innerFn(target, name);
+        target.constructor[INDEXED_ID_META_PROPERY] = name;
+    };
+}
+export const getIdProperty = (clazz) => {
+    // TODO nested id property
+    const property = clazz[INDEXED_ID_META_PROPERY];
+    if (!property) {
+        // look into children
+        const fields = getSchema(clazz)?.fields;
+        if (!fields) {
+            return;
+        }
+        for (const field of fields) {
+            if (typeof field.type === "function") {
+                const idFromChild = getIdProperty(field.type);
+                if (idFromChild) {
+                    return [field.key, ...idFromChild];
+                }
+            }
+        }
+        return undefined;
+    }
+    return [property];
+};
+//# sourceMappingURL=utils.js.map

package/dist/src/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAIN,KAAK,EACL,SAAS,GACT,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAa,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CACjC,CAAoB,EACpB,CAAoB,EACnB,EAAE;IACH,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACd,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CACjC,CAAsB,EACtB,CAAsB,EACtB,KAAa,EACb,OAA6B,EAC5B,EAAE;IACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,iBAAiB,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,EAAE,GAAG,iBAAiB,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAChC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;YACf,IAAI,IAAI,CAAC,SAAS,KAAK,aAAa,CAAC,GAAG,EAAE,CAAC;gBAC1C,OAAO,GAAG,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC;YACb,CAAC;QACF,CAAC;IACF,CAAC;IACD,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAO,EAAE,EAAO,EAAE,EAAE;IAC/C,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QACtD,OAAO,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QACb,OAAO,CAAC,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QACpB,OAAO,CAAC,CAAC;IACV,CAAC;IACD,OAAO,CAAC,CAAC;AACV,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAChC,GAAQ,EACR,IAAc,EACd,OAA6B,EACzB,EAAE;IACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,GAAG,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,WAAW,CAAC;AAE5C,MAAM,UAAU,EAAE,CAAC,UAA0C;IAC5D,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAClC,OAAO,CAAC,MAAW,EAAE,IAAY,EAAE,EAAE;QACpC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACtB,MAAM,CAAC,WAAW,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC;IACpD,CAAC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAC5B,KAAwB,EACD,EAAE;IACzB,0BAA0B;IAC1B,MAAM,QAAQ,GAAI,KAAa,CAAC,uBAAuB,CAAC,CAAC;IACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,qBAAqB;QAErB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO;QACR,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBACtC,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9C,IAAI,WAAW,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,CAAC;gBACpC,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,QAAkB,CAAC,CAAC;AAC7B,CAAC,CAAC"}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@peerbit/indexer-interface",
+  "version": "0.0.1-cccc078",
+  "description": "Indexer store interface",
+  "sideEffects": false,
+  "type": "module",
+  "types": "./dist/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ],
+      "src/*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ]
+    }
+  },
+  "files": [
+    "src",
+    "dist",
+    "!dist/e2e",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "peerbit",
+    "parserOptions": {
+      "project": true,
+      "sourceType": "module"
+    },
+    "ignorePatterns": [
+      "!.aegir.js",
+      "test/ts-use",
+      "*.d.ts"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --no-bundle",
+    "test": "aegir test",
+    "lint": "aegir lint"
+  },
+  "author": "dao.xyz",
+  "license": "MIT",
+  "dependencies": {
+    "uuid": "^9.0.0",
+    "@dao-xyz/borsh": "^5.2.3",
+    "@peerbit/crypto": "2.2.0-cccc078"
+  }
+}

package/src/id.ts

@@ -0,0 +1,191 @@
+import { field, variant } from "@dao-xyz/borsh";
+import { toBase64 } from "@peerbit/crypto";
+import {
+	decodeUint8Array,
+	encodeUint8Array,
+	encodingLength,
+} from "uint8-varint";
+
+export abstract class PrimitiveValue {}
+
+@variant(0)
+export class StringValue extends PrimitiveValue {
+	@field({ type: "string" })
+	string: string;
+
+	constructor(string: string) {
+		super();
+		this.string = string;
+	}
+}
+
+@variant(1)
+export abstract class NumberValue extends PrimitiveValue {
+	abstract get value(): number | bigint;
+}
+
+@variant(0)
+export abstract class IntegerValue extends NumberValue {}
+
+@variant(0)
+export class UnsignedIntegerValue extends IntegerValue {
+	@field({ type: "u32" })
+	number: number;
+
+	constructor(number: number) {
+		super();
+		if (!Number.isInteger(number) || number > 4294967295 || number < 0) {
+			throw new Error("Number is not u32");
+		}
+		this.number = number;
+	}
+
+	get value() {
+		return this.number;
+	}
+}
+
+@variant(1)
+export class BigUnsignedIntegerValue extends IntegerValue {
+	@field({ type: "u64" })
+	number: bigint;
+
+	constructor(number: bigint) {
+		super();
+		if (number > 18446744073709551615n || number < 0) {
+			throw new Error("Number is not u32");
+		}
+		this.number = number;
+	}
+	get value() {
+		return this.number;
+	}
+}
+
+export type IdPrimitive = string | number | bigint;
+
+export abstract class IdKey {
+	abstract get key(): string | bigint | number | Uint8Array;
+	abstract get primitive(): IdPrimitive;
+}
+
+@variant(0)
+export class StringKey extends IdKey {
+	@field({ type: "string" })
+	key: string;
+
+	constructor(key: string) {
+		super();
+		this.key = key;
+	}
+	get primitive() {
+		return this.key;
+	}
+}
+
+@variant(1)
+export class Uint8ArrayKey extends IdKey {
+	@field({ type: Uint8Array })
+	key: Uint8Array;
+
+	constructor(key: Uint8Array) {
+		super();
+		this.key = key;
+	}
+
+	private _keyString: string;
+	get primitive(): string {
+		return this._keyString || (this._keyString = toBase64(this.key));
+	}
+}
+
+const varint53 = {
+	deserialize: (reader: any) => {
+		const number = decodeUint8Array(reader._buf, reader._offset);
+		const len = encodingLength(number);
+		reader._offset += len;
+		return number;
+	},
+	serialize: (value: any, writer: any) => {
+		const offset = writer.totalSize;
+		writer["_writes"] = writer["_writes"].next = () =>
+			encodeUint8Array(value, writer["_buf"], offset);
+		writer.totalSize += encodingLength(value);
+	},
+};
+
+@variant(2)
+export class IntegerKey extends IdKey {
+	@field(varint53) // max value is 2^53 - 1 (9007199254740991)
+	key: number;
+
+	constructor(key: number) {
+		super();
+		this.key = key;
+	}
+
+	get primitive() {
+		return this.key;
+	}
+}
+
+export type Ideable = string | number | bigint | Uint8Array;
+
+const idKeyTypes = new Set(["string", "number", "bigint"]);
+
+export const toId = (obj: Ideable): IdKey => {
+	if (typeof obj === "string") {
+		return new StringKey(obj);
+	}
+	if (typeof obj === "number") {
+		return new IntegerKey(obj);
+	}
+	if (typeof obj === "bigint") {
+		if (obj <= Number.MAX_SAFE_INTEGER && obj >= 0) {
+			return new IntegerKey(Number(obj));
+		}
+		throw new Error(
+			"BigInt is not less than 2^53. Max value is 9007199254740991",
+		);
+	}
+	if (obj instanceof Uint8Array) {
+		return new Uint8ArrayKey(obj);
+	}
+	throw new Error(
+		"Unexpected index key: " +
+			typeof obj +
+			", expected: string, number, bigint or Uint8Array",
+	);
+};
+
+export const toIdeable = (
+	key: IdKey | Ideable,
+): string | number | bigint | Uint8Array => {
+	if (key instanceof IdKey) {
+		return key.key;
+	}
+	return key;
+};
+
+export const checkId = (obj: Ideable) => {
+	if (obj == null) {
+		throw new Error(
+			`The provided key value is null or undefined, expecting string, number, bigint, or Uint8array`,
+		);
+	}
+	const type = typeof obj;
+
+	if (type === "number") {
+		if (!Number.isInteger(obj)) {
+			throw new Error(`The provided key number value is not an integer`);
+		}
+	}
+
+	if (idKeyTypes.has(type) || obj instanceof Uint8Array) {
+		return;
+	}
+
+	throw new Error(
+		`Key is not ${[...idKeyTypes]}, provided key value type: ${typeof obj}`,
+	);
+};

package/src/index-engine.ts

@@ -0,0 +1,213 @@
+import type { AbstractType } from "@dao-xyz/borsh";
+import { type IdKey } from "./id.js";
+import {
+	CloseIteratorRequest,
+	CollectNextRequest,
+	type CountRequest,
+	type DeleteRequest,
+	type SearchRequest,
+	type SumRequest,
+} from "./query.js";
+
+type MaybePromise<T = void> = Promise<T> | T;
+
+export interface IndexedResult<
+	T extends Record<string, any> = Record<string, any>,
+> {
+	id: IdKey;
+	value: T;
+}
+
+export interface IndexedResults<
+	T extends Record<string, any> = Record<string, any>,
+> {
+	results: IndexedResult<T>[];
+	kept: number;
+}
+
+export interface IndexedValue<T = Record<string, any>> {
+	id: IdKey;
+	value: T;
+}
+
+export type NestedProperties<T> = {
+	match: (obj: any) => obj is T;
+	query: (nested: T, query: SearchRequest) => Promise<Record<string, any>[]>;
+};
+
+export type IteratorBatchProperties = {
+	maxSize: number;
+	sizeProperty: string[];
+};
+export type IndexEngineInitProperties<T, N> = {
+	indexBy?: string[];
+	nested?: NestedProperties<N>;
+	schema: AbstractType<T>;
+	/* iterator: { batch: IteratorBatchProperties }; */
+};
+
+/* type ResolveOptions = 'match' | 'full';
+export type QueryOptions = { resolve: ResolveOptions };
+export type ReturnTypeFromQueryOptions<T, O> = O extends { resolve: 'full' } ? T : Partial<T>;
+ */
+
+export type Shape = { [key: string]: true | Shape };
+export interface Index<T extends Record<string, any>, NestedType = any> {
+	init(
+		properties: IndexEngineInitProperties<T, NestedType>,
+	): MaybePromise<Index<T, NestedType>>;
+	drop(): MaybePromise<void>;
+	get(
+		id: IdKey,
+		options?: { shape: Shape },
+	): MaybePromise<IndexedResult<T> | undefined>;
+	put(value: T, id?: IdKey): MaybePromise<void>;
+	del(query: DeleteRequest): MaybePromise<IdKey[]>;
+	sum(query: SumRequest): MaybePromise<bigint | number>;
+	count(query: CountRequest): MaybePromise<number>;
+	query(
+		query: SearchRequest,
+		options?: { shape?: Shape; reference?: boolean },
+	): MaybePromise<IndexedResults<T>>;
+	next(
+		query: CollectNextRequest,
+		options?: { shape?: Shape },
+	): MaybePromise<IndexedResults<T>>;
+	close(query: CloseIteratorRequest): MaybePromise<void>;
+
+	/*
+	query<O extends QueryOptions>(query: SearchRequest, options?: O): MaybePromise<IndexedResults<ReturnTypeFromQueryOptions<T, O>>>;
+	next<O extends QueryOptions>(query: CollectNextRequest, options?: O): MaybePromise<IndexedResults<ReturnTypeFromQueryOptions<T, O>>>;
+	*/
+
+	getSize(): MaybePromise<number>;
+	getPending(cursorId: string): number | undefined;
+	start(): MaybePromise<void>;
+	stop(): MaybePromise<void>;
+	get cursorCount(): number;
+}
+
+export type IndexIterator<T> = ReturnType<typeof iterate<T>>;
+export const iterate = <T>(index: Index<T, any>, query: SearchRequest) => {
+	let isDone = false;
+	let fetchedOnce = false;
+	const next = async (count: number, options?: { shape?: Shape }) => {
+		let res: IndexedResults<T>;
+		if (!fetchedOnce) {
+			fetchedOnce = true;
+			query.fetch = count;
+			res = await index.query(query, options);
+		} else {
+			res = await index.next(
+				new CollectNextRequest({ id: query.id, amount: count }),
+				options,
+			);
+		}
+		isDone = res.kept === 0;
+		return res;
+	};
+	const done = () => isDone;
+	const close = () => {
+		return index.close(new CloseIteratorRequest({ id: query.id }));
+	};
+	return {
+		next,
+		done,
+		close,
+		all: async (options?: { shape: Shape }) => {
+			const results: IndexedResult<T>[] = [];
+			while (!done()) {
+				for (const element of (await next(100, options)).results) {
+					results.push(element);
+				}
+			}
+			await close();
+			return results;
+		},
+	};
+};
+
+export type ResultsIterator<T> = ReturnType<typeof iterate<T>>;
+
+export const iteratorInSeries = <T>(...iterators: IndexIterator<T>[]) => {
+	let i = 0;
+	const done = () => i >= iterators.length;
+	let current = iterators[i];
+	const next = async (count: number) => {
+		let acc: IndexedResults<T> = {
+			kept: 0,
+			results: [],
+		};
+
+		while (!current.done() && i < iterators.length) {
+			const next = await current.next(count);
+			acc.kept += next.kept;
+			acc.results.push(...next.results);
+
+			if (current.done()) {
+				i++;
+				if (i >= iterators.length) {
+					break;
+				}
+				current = iterators[i];
+			}
+
+			if (acc.results.length >= count) {
+				break;
+			}
+		}
+
+		return acc;
+	};
+	return {
+		next,
+		done,
+		close: async () => {
+			for (const iterator of iterators) {
+				await iterator.close();
+			}
+		},
+		all: async () => {
+			const results = [];
+			while (!done()) {
+				for (const element of (await next(100)).results) {
+					results.push(element);
+				}
+			}
+			return results;
+		},
+	};
+};
+
+export interface Indices {
+	init<T extends Record<string, any>, NestedType>(
+		properties: IndexEngineInitProperties<T, NestedType>,
+	): MaybePromise<Index<T, NestedType>>;
+	scope(name: string): MaybePromise<Indices>;
+	start(): MaybePromise<void>;
+	stop(): MaybePromise<void>;
+	drop(): MaybePromise<void>;
+}
+
+/* export interface IndexEngine<T extends Record<string, any> = Record<string, any>, NestedType = any> {
+init(properties: IndexEngineInitProperties<NestedType>): MaybePromise<void>
+start?(): MaybePromise<void>;
+stop?(): MaybePromise<void>;
+clear(): MaybePromise<void>;
+scope: <TS extends Record<string, any> = Record<string, any>>(name: string) => MaybePromise<IndexEngine<TS>>;
+get(
+	id: IdKey
+): MaybePromise<IndexedResult<T> | undefined>;
+put(value: IndexedValue<T>): MaybePromise<void>;
+del(id: IdKey): MaybePromise<void>;
+sum(query: SumRequest): MaybePromise<number>;
+count(query: CountRequest): MaybePromise<number>;
+query(query: SearchRequest, from: PublicSignKey): MaybePromise<IndexedResults<T>>;
+next(query: CollectNextRequest, from: PublicSignKey): MaybePromise<IndexedResults<T>>;
+close(query: CloseIteratorRequest, from: PublicSignKey): MaybePromise<void>;
+getSize(): MaybePromise<number>;
+getPending(cursorId: string): number | undefined;
+get cursorCount(): number;
+}
+
+export type IndexEngineConstuctor<T extends Record<string, any>> = (directory?: string) => MaybePromise<IndexEngine<T, any>>; */

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./id.js";
+export * from "./query.js";
+export * from "./index-engine.js";
+export * from "./store.js";
+export * from "./utils.js";