nostr-idb 0.1.0

package/README.md

@@ -0,0 +1,3 @@
+# nostr-idb
+
+A collection of helper methods for storing nostr events in IndexedDB

package/dist/common.d.ts

@@ -0,0 +1 @@
+export declare const GENERIC_TAGS: string[];

package/dist/common.js

@@ -0,0 +1,2 @@
+const LETTERS = "abcdefghijklmnopqrstuvwxyz";
+export const GENERIC_TAGS = (LETTERS + LETTERS.toUpperCase()).split("");

package/dist/database.d.ts

@@ -0,0 +1,6 @@
+import { DeleteDBCallbacks, OpenDBCallbacks } from "idb";
+import { Schema } from "./schema.js";
+export declare const NOSTR_IDB_NAME = "nostr-idb";
+export declare const NOSTR_IDB_VERSION = 1;
+export declare function openDatabase(name?: string, callbacks?: OpenDBCallbacks<Schema>): Promise<import("idb").IDBPDatabase<Schema>>;
+export declare function deleteDatabase(name?: string, callbacks?: DeleteDBCallbacks): Promise<void>;

package/dist/database.js

@@ -0,0 +1,26 @@
+import { deleteDB, openDB } from "idb";
+export const NOSTR_IDB_NAME = "nostr-idb";
+export const NOSTR_IDB_VERSION = 1;
+export async function openDatabase(name = NOSTR_IDB_NAME, callbacks) {
+    return await openDB(name, NOSTR_IDB_VERSION, {
+        ...callbacks,
+        upgrade(db, oldVersion, newVersion, transaction, event) {
+            const events = db.createObjectStore("events", { keyPath: "event.id" });
+            events.createIndex("id", "event.id", { unique: true });
+            events.createIndex("pubkey", "event.pubkey");
+            events.createIndex("kind", "event.kind");
+            events.createIndex("create_at", "event.created_at");
+            events.createIndex("tags", "tags", { multiEntry: true });
+            const seen = db.createObjectStore("seen", { keyPath: "id" });
+            seen.createIndex("date", "date");
+            seen.createIndex("relay", "relays", { multiEntry: true });
+            const used = db.createObjectStore("used", { keyPath: "id" });
+            used.createIndex("date", "date");
+            if (callbacks?.upgrade)
+                callbacks.upgrade(db, oldVersion, newVersion, transaction, event);
+        },
+    });
+}
+export async function deleteDatabase(name = NOSTR_IDB_NAME, callbacks) {
+    return await deleteDB(name, callbacks);
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./database.js";
+export * from "./query-filter.js";
+export * from "./query-misc.js";
+export * from "./ingest.js";

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from "./database.js";
+export * from "./query-filter.js";
+export * from "./query-misc.js";
+export * from "./ingest.js";

package/dist/ingest.d.ts

@@ -0,0 +1,8 @@
+import { IDBPTransaction } from "idb";
+import { Event } from "nostr-tools";
+import type { NostrIDB, Schema } from "./schema.js";
+export declare function createWriteTransaction(db: NostrIDB): IDBPTransaction<Schema, ["events"], "readwrite">;
+export declare function getEventTags(event: Event): string[];
+export declare function addEvent(db: NostrIDB, event: Event, transaction?: IDBPTransaction<Schema, ["events"], "readwrite">): Promise<void>;
+export declare function addEvents(db: NostrIDB, events: Event[]): Promise<void>;
+export declare function updateUsed(db: NostrIDB, ids: string[]): Promise<void>;

package/dist/ingest.js

@@ -0,0 +1,39 @@
+import { validateEvent } from "nostr-tools";
+import { GENERIC_TAGS } from "./common.js";
+export function createWriteTransaction(db) {
+    return db.transaction("events", "readwrite");
+}
+export function getEventTags(event) {
+    return event.tags
+        .filter((t) => t.length >= 2 && t[0].length === 1 && GENERIC_TAGS.includes(t[0]))
+        .map((t) => t[0] + t[1]);
+}
+export async function addEvent(db, event, transaction) {
+    if (!validateEvent(event))
+        throw new Error("Invalid Event");
+    const trans = transaction || createWriteTransaction(db);
+    trans.objectStore("events").put({
+        event,
+        tags: getEventTags(event),
+    });
+    if (!transaction)
+        await trans.commit();
+}
+export async function addEvents(db, events) {
+    const trans = db.transaction("events", "readwrite");
+    for (const event of events) {
+        await addEvent(db, event, trans);
+    }
+    await trans.commit();
+}
+export async function updateUsed(db, ids) {
+    const trans = db.transaction("used", "readwrite");
+    const nowUnix = Math.floor(new Date().valueOf() / 1000);
+    for (const id of ids) {
+        trans.objectStore("used").put({
+            id,
+            date: nowUnix,
+        });
+    }
+    await trans.commit();
+}

package/dist/query-filter.d.ts

@@ -0,0 +1,10 @@
+import type { Event, Filter } from "nostr-tools";
+import type { NostrIDB } from "./schema.js";
+export declare function queryForPubkeys(db: NostrIDB, authors?: Filter["authors"]): Promise<Set<string>>;
+export declare function queryForTag(db: NostrIDB, tag: string, values: string[]): Promise<Set<string>>;
+export declare function queryForKinds(db: NostrIDB, kinds?: Filter["kinds"]): Promise<Set<string>>;
+export declare function queryForTime(db: NostrIDB, since: number | undefined, until: number | undefined): Promise<Set<string>>;
+export declare function getIdsForFilter(db: NostrIDB, filter: Filter): Promise<Set<string>>;
+export declare function getIdsForFilters(db: NostrIDB, filters: Filter[]): Promise<Set<string>>;
+export declare function getEventsForFilters(db: NostrIDB, filters: Filter[]): Promise<Event<number>[]>;
+export declare function countEventsForFilters(db: NostrIDB, filters: Filter[]): Promise<void>;

package/dist/query-filter.js

@@ -0,0 +1,128 @@
+import { GENERIC_TAGS } from "./common.js";
+export function queryForPubkeys(db, authors = []) {
+    const ids = new Set();
+    const trans = db.transaction("events", "readonly");
+    const objectStore = trans.objectStore("events");
+    const index = objectStore.index("pubkey");
+    const handleEvents = (result) => {
+        for (const id of result)
+            ids.add(id);
+    };
+    const promises = authors.map((pubkey) => index.getAllKeys(pubkey).then(handleEvents));
+    const result = Promise.all(promises).then(() => ids);
+    trans.commit();
+    return result;
+}
+export function queryForTag(db, tag, values) {
+    const ids = new Set();
+    const trans = db.transaction("events", "readonly");
+    const objectStore = trans.objectStore("events");
+    const index = objectStore.index("tags");
+    const handleEvents = (result) => {
+        for (const id of result)
+            ids.add(id);
+    };
+    const promises = values.map((v) => index.getAllKeys(tag + v).then(handleEvents));
+    const result = Promise.all(promises).then(() => ids);
+    trans.commit();
+    return result;
+}
+export function queryForKinds(db, kinds = []) {
+    const ids = new Set();
+    const trans = db.transaction("events", "readonly");
+    const objectStore = trans.objectStore("events");
+    const index = objectStore.index("kind");
+    const handleEvents = (result) => {
+        for (const id of result)
+            ids.add(id);
+    };
+    const promises = kinds.map((kind) => index.getAllKeys(kind).then(handleEvents));
+    const result = Promise.all(promises).then(() => ids);
+    trans.commit();
+    return result;
+}
+export async function queryForTime(db, since, until) {
+    let range;
+    if (since !== undefined && until !== undefined)
+        range = IDBKeyRange.bound(since, until);
+    else if (since !== undefined)
+        range = IDBKeyRange.lowerBound(since);
+    else if (until !== undefined)
+        range = IDBKeyRange.upperBound(until);
+    else
+        throw new Error("Missing since or until");
+    const arr = await db.getAllKeysFromIndex("events", "create_at", range);
+    const ids = new Set(arr);
+    return ids;
+}
+export async function getIdsForFilter(db, filter) {
+    if (filter.ids)
+        return new Set(filter.ids);
+    let ids = null;
+    const and = (set) => {
+        if (!ids)
+            ids = set;
+        for (const id of ids) {
+            if (!set.has(id))
+                ids.delete(id);
+        }
+        return ids;
+    };
+    // query for time first if both are set
+    if (filter.since && filter.until)
+        and(await queryForTime(db, filter.since, filter.until));
+    for (const t of GENERIC_TAGS) {
+        const key = `#${t}`;
+        const values = filter[key];
+        if (values?.length)
+            and(await queryForTag(db, t, values));
+    }
+    if (filter.authors)
+        and(await queryForPubkeys(db, filter.authors));
+    if (filter.kinds)
+        and(await queryForKinds(db, filter.kinds));
+    // query for time last if only one is set
+    if ((filter.since === undefined && filter.until) ||
+        (filter.since && filter.until === undefined))
+        and(await queryForTime(db, filter.since, filter.until));
+    if (ids === null)
+        throw new Error("Empty filter");
+    return ids;
+}
+export async function getIdsForFilters(db, filters) {
+    if (filters.length === 0)
+        throw new Error("No Filters");
+    let ids = null;
+    for (const filter of filters) {
+        const filterIds = await getIdsForFilter(db, filter);
+        if (!ids)
+            ids = filterIds;
+        else
+            for (const id of ids)
+                if (!filterIds.has(id))
+                    ids.delete(id);
+    }
+    if (ids === null)
+        throw new Error("Empty filters");
+    return ids;
+}
+export async function getEventsForFilters(db, filters) {
+    const ids = await getIdsForFilters(db, filters);
+    const events = [];
+    const trans = db.transaction("events", "readonly");
+    const objectStore = trans.objectStore("events");
+    const index = objectStore.index("id");
+    const handleEntry = (e) => e && events.push(e.event);
+    const promises = Array.from(ids).map((id) => index.get(id).then(handleEntry));
+    const sorted = await Promise.all(promises).then(() => events.sort((a, b) => b.created_at - a.created_at));
+    trans.commit();
+    let minLimit = Infinity;
+    for (const filter of filters) {
+        if (filter.limit && filter.limit < minLimit)
+            minLimit = filter.limit;
+    }
+    if (sorted.length > minLimit)
+        sorted.length = minLimit;
+    return sorted;
+}
+export async function countEventsForFilters(db, filters) { }

package/dist/query-misc.d.ts

@@ -0,0 +1,4 @@
+import { NostrIDB } from "./schema.js";
+export declare function countEventsByAllPubkeys(db: NostrIDB): Promise<Record<string, number>>;
+export declare function countEventsByPubkey(db: NostrIDB, pubkey: string): Promise<number>;
+export declare function countEvents(db: NostrIDB): Promise<number>;

package/dist/query-misc.js

@@ -0,0 +1,26 @@
+export async function countEventsByAllPubkeys(db) {
+    let cursor = await db
+        .transaction("events", "readonly")
+        .objectStore("events")
+        .index("pubkey")
+        .openKeyCursor();
+    if (!cursor)
+        return {};
+    const counts = {};
+    while (cursor) {
+        const pubkey = cursor.key;
+        counts[pubkey] = (counts[pubkey] || 0) + 1;
+        cursor = await cursor.continue();
+    }
+    return counts;
+}
+export async function countEventsByPubkey(db, pubkey) {
+    return await db
+        .transaction("events", "readonly")
+        .objectStore("events")
+        .index("pubkey")
+        .count(pubkey);
+}
+export function countEvents(db) {
+    return db.transaction("events", "readonly").store.count();
+}

package/dist/schema.d.ts

@@ -0,0 +1,43 @@
+import type { DBSchema, IDBPDatabase } from "idb";
+import type { Event } from "nostr-tools";
+export type NostrIDB = IDBPDatabase<Schema>;
+export interface Schema extends DBSchema {
+    events: {
+        key: "id";
+        value: {
+            event: Event;
+            tags: string[];
+        };
+        indexes: {
+            id: string;
+            pubkey: string;
+            kind: number;
+            create_at: number;
+            tags: string;
+        };
+    };
+    seen: {
+        key: "id";
+        value: {
+            id: string;
+            date: number;
+            relays: string[];
+        };
+        indexes: {
+            id: string;
+            date: string;
+            relay: string;
+        };
+    };
+    used: {
+        key: "id";
+        value: {
+            id: string;
+            date: number;
+        };
+        indexes: {
+            id: string;
+            date: string;
+        };
+    };
+}

package/dist/schema.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "nostr-idb",
+  "version": "0.1.0",
+  "description": "A collection of helper methods for storing nostr events in IndexedDB",
+  "license": "MIT",
+  "type": "module",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc --project tsconfig.json"
+  },
+  "dependencies": {
+    "idb": "^8.0.0",
+    "nostr-tools": "^1.17.0"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.12",
+    "@types/react": "^18.2.37",
+    "@types/react-dom": "^18.2.15",
+    "@vitejs/plugin-react-swc": "^3.5.0",
+    "dayjs": "^1.11.10",
+    "prettier": "^3.1.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "typescript": "^5.3.3",
+    "vite": "^5.0.0"
+  }
+}