nostr-idb 0.2.0 → 1.0.0

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # nostr-idb
 
+## 1.0.0
+
+### Major Changes
+
+- 9bbdbc7: Upgrade nostr-tools to v2
+
+### Minor Changes
+
+- 9bbdbc7: Support ephemeral events
+- 446cb1e: Support replaceable and parameterized replaceable events
+
 ## 0.2.0
 
 ### Minor Changes

package/dist/database.d.ts

@@ -1,7 +1,7 @@
 import { DeleteDBCallbacks, OpenDBCallbacks } from "idb";
 import { NostrIDB, Schema } from "./schema.js";
 export declare const NOSTR_IDB_NAME = "nostr-idb";
-export declare const NOSTR_IDB_VERSION = 1;
+export declare const NOSTR_IDB_VERSION = 2;
 export declare function openDB(name?: string, callbacks?: OpenDBCallbacks<Schema>): Promise<import("idb").IDBPDatabase<Schema>>;
 export declare function deleteDB(name?: string, callbacks?: DeleteDBCallbacks): Promise<void>;
 export declare function clearDB(db: NostrIDB): Promise<void>;

package/dist/database.js

@@ -1,21 +1,25 @@
 import { deleteDB as idbDeleteDB, openDB as idbOpenDB, } from "idb";
 export const NOSTR_IDB_NAME = "nostr-idb";
-export const NOSTR_IDB_VERSION = 1;
+export const NOSTR_IDB_VERSION = 2;
 export async function openDB(name = NOSTR_IDB_NAME, callbacks) {
     return await idbOpenDB(name, NOSTR_IDB_VERSION, {
         ...callbacks,
         upgrade(db, oldVersion, newVersion, transaction, event) {
-            const events = db.createObjectStore("events", { keyPath: "event.id" });
+            if (oldVersion < 2) {
+                debugger;
+                console.log("resetting DB");
+                db.deleteObjectStore("events");
+                db.deleteObjectStore("used");
+                // @ts-ignore
+                db.deleteObjectStore("seen");
+            }
+            const events = db.createObjectStore("events");
             events.createIndex("id", "event.id", { unique: true });
             events.createIndex("pubkey", "event.pubkey");
             events.createIndex("kind", "event.kind");
             events.createIndex("created_at", "event.created_at");
             events.createIndex("tags", "tags", { multiEntry: true });
-            events.createIndex("addressPointer", ["kind", "pubkey", "identifier"]);
-            const seen = db.createObjectStore("seen", { keyPath: "id" });
-            seen.createIndex("date", "date");
-            seen.createIndex("relay", "relays", { multiEntry: true });
-            const used = db.createObjectStore("used", { keyPath: "id" });
+            const used = db.createObjectStore("used", { keyPath: "uid" });
             used.createIndex("date", "date");
             if (callbacks?.upgrade)
                 callbacks.upgrade(db, oldVersion, newVersion, transaction, event);
@@ -28,5 +32,4 @@ export async function deleteDB(name = NOSTR_IDB_NAME, callbacks) {
 export async function clearDB(db) {
     await db.clear("events");
     await db.clear("used");
-    await db.clear("seen");
 }

package/dist/ingest.d.ts

@@ -1,16 +1,10 @@
 import { IDBPTransaction } from "idb";
 import { Event } from "nostr-tools";
 import type { NostrIDB, Schema } from "./schema.js";
-/**
- * Returns an events tags as an array of string for indexing
- */
+/** Returns an events tags as an array of string for indexing */
 export declare function getEventTags(event: Event): string[];
-export declare function isReplaceable(kind: number): boolean;
-export type ReplaceableEventAddress = {
-    kind: number;
-    pubkey: string;
-    identifier?: string;
-};
+/** returns the events Unique ID */
+export declare function getEventUID(event: Event): string;
 export declare function addEvent(db: NostrIDB, event: Event, transaction?: IDBPTransaction<Schema, ["events"], "readwrite">): Promise<void>;
 export declare function addEvents(db: NostrIDB, events: Event[]): Promise<void>;
-export declare function updateUsed(db: NostrIDB, ids: string[]): Promise<void>;
+export declare function updateUsed(db: NostrIDB, uids: Iterable<string>): Promise<void>;

package/dist/ingest.js

@@ -1,28 +1,19 @@
-import { validateEvent } from "nostr-tools";
+import { kinds, validateEvent } from "nostr-tools";
 import { GENERIC_TAGS } from "./common.js";
-/**
- * Returns an events tags as an array of string for indexing
- */
+/** Returns an events tags as an array of string for indexing */
 export function getEventTags(event) {
     return event.tags
         .filter((t) => t.length >= 2 && t[0].length === 1 && GENERIC_TAGS.includes(t[0]))
         .map((t) => t[0] + t[1]);
 }
-// based on replaceable kinds from https://github.com/nostr-protocol/nips/blob/master/01.md#kinds
-export function isReplaceable(kind) {
-    return ((kind >= 30000 && kind < 40000) ||
-        (kind >= 10000 && kind < 20000) ||
-        kind === 0 ||
-        kind === 3 ||
-        kind === 41);
-}
-function getReplaceableEventAddress(event) {
-    // only create addresses for replaceable events
-    if (!isReplaceable(event.kind))
-        return undefined;
-    // get d tag
-    const identifier = event.tags.find((t) => t[0] === "d" && t[1])?.[1];
-    return { kind: event.kind, pubkey: event.pubkey, identifier };
+/** returns the events Unique ID */
+export function getEventUID(event) {
+    if (kinds.isReplaceableKind(event.kind) ||
+        kinds.isParameterizedReplaceableKind(event.kind)) {
+        const d = event.tags.find((t) => t[0] === "d")?.[1];
+        return `${event.kind}:${event.pubkey}:${d ?? ""}`;
+    }
+    return event.id;
 }
 export async function addEvent(db, event, transaction) {
     if (!validateEvent(event))
@@ -31,8 +22,7 @@ export async function addEvent(db, event, transaction) {
     trans.objectStore("events").put({
         event,
         tags: getEventTags(event),
-        identifier: getReplaceableEventAddress(event)?.identifier,
-    });
+    }, getEventUID(event));
     if (!transaction)
         await trans.commit();
 }
@@ -43,12 +33,12 @@ export async function addEvents(db, events) {
     }
     await trans.commit();
 }
-export async function updateUsed(db, ids) {
+export async function updateUsed(db, uids) {
     const trans = db.transaction("used", "readwrite");
     const nowUnix = Math.floor(new Date().valueOf() / 1000);
-    for (const id of ids) {
+    for (const uid of uids) {
         trans.objectStore("used").put({
-            id,
+            uid,
             date: nowUnix,
         });
     }

package/dist/query-filter.d.ts

@@ -4,10 +4,10 @@ import { IndexCache } from "./index-cache";
 export declare function queryForPubkeys(db: NostrIDB, authors?: Filter["authors"], indexCache?: IndexCache): Set<string> | Promise<Set<string>>;
 export declare function queryForTag(db: NostrIDB, tag: string, values: string[], indexCache?: IndexCache): Set<string> | Promise<Set<string>>;
 export declare function queryForKinds(db: NostrIDB, kinds?: Filter["kinds"], indexCache?: IndexCache): Set<string> | Promise<Set<string>>;
-export declare function queryForTime(db: NostrIDB, since: number | undefined, until: number | undefined): Promise<Set<string>>;
+export declare function queryForTime(db: NostrIDB, since: number | undefined, until: number | undefined): Promise<string[]>;
 export declare function getIdsForFilter(db: NostrIDB, filter: Filter, indexCache?: IndexCache): Promise<Set<string>>;
 export declare function getIdsForFilters(db: NostrIDB, filters: Filter[], indexCache?: IndexCache): Promise<Set<string>>;
-export declare function getEventsForFilter(db: NostrIDB, filter: Filter, indexCache?: IndexCache): Promise<Event<number>[]>;
-export declare function getEventsForFilters(db: NostrIDB, filters: Filter[], indexCache?: IndexCache): Promise<Event<number>[]>;
+export declare function getEventsForFilter(db: NostrIDB, filter: Filter, indexCache?: IndexCache): Promise<Event[]>;
+export declare function getEventsForFilters(db: NostrIDB, filters: Filter[], indexCache?: IndexCache): Promise<Event[]>;
 export declare function countEventsForFilter(db: NostrIDB, filter: Filter, indexCache?: IndexCache): Promise<number>;
 export declare function countEventsForFilters(db: NostrIDB, filters: Filter[], indexCache?: IndexCache): Promise<number>;

package/dist/query-filter.js

@@ -107,8 +107,7 @@ export async function queryForTime(db, since, until) {
         range = IDBKeyRange.upperBound(until);
     else
         throw new Error("Missing since or until");
-    const arr = await db.getAllKeysFromIndex("events", "created_at", range);
-    const ids = new Set(arr);
+    const ids = (await db.getAllKeysFromIndex("events", "created_at", range)).reverse();
     return ids;
 }
 export async function getIdsForFilter(db, filter, indexCache) {
@@ -118,18 +117,22 @@ export async function getIdsForFilter(db, filter, indexCache) {
     if (filter.ids)
         return new Set(filter.ids);
     let ids = null;
-    const and = (set) => {
+    const and = (iterable) => {
+        const set = iterable instanceof Set ? iterable : new Set(iterable);
         if (!ids)
             ids = set;
-        for (const id of ids) {
-            if (!set.has(id))
-                ids.delete(id);
-        }
+        else
+            for (const id of ids)
+                if (!set.has(id))
+                    ids.delete(id);
         return ids;
     };
-    // query for time first if both are set
-    if (filter.since && filter.until)
-        and(await queryForTime(db, filter.since, filter.until));
+    let timeFilterIds = null;
+    // query for time first if since is set
+    if (filter.since) {
+        timeFilterIds = await queryForTime(db, filter.since, filter.until);
+        and(timeFilterIds);
+    }
     for (const t of GENERIC_TAGS) {
         const key = `#${t}`;
         const values = filter[key];
@@ -140,10 +143,22 @@ export async function getIdsForFilter(db, filter, indexCache) {
         and(await queryForPubkeys(db, filter.authors, indexCache));
     if (filter.kinds)
         and(await queryForKinds(db, filter.kinds, indexCache));
-    // query for time last if only one is set
-    if ((filter.since === undefined && filter.until) ||
-        (filter.since && filter.until === undefined))
-        and(await queryForTime(db, filter.since, filter.until));
+    // query for time last if only until is set
+    if (filter.since === undefined && filter.until) {
+        timeFilterIds = await queryForTime(db, filter.since, filter.until);
+        and(timeFilterIds);
+    }
+    // if the filter queried on time and has a limit. truncate the ids now
+    if (filter.limit && timeFilterIds) {
+        const limitIds = new Set();
+        for (const id of timeFilterIds) {
+            if (limitIds.size >= filter.limit)
+                break;
+            if (ids.has(id))
+                limitIds.add(id);
+        }
+        return limitIds;
+    }
     if (ids === null)
         throw new Error("Empty filter");
     return ids;
@@ -165,15 +180,14 @@ export async function getIdsForFilters(db, filters, indexCache) {
         throw new Error("Empty filters");
     return ids;
 }
-async function loadEventsById(db, ids, filters) {
-    const events = [];
+async function loadEventsByUID(db, uids, filters) {
+    const eventBuffer = [];
     const trans = db.transaction("events", "readonly");
     const objectStore = trans.objectStore("events");
-    const index = objectStore.index("id");
-    const handleEntry = (e) => e && events.push(e.event);
-    const promises = Array.from(ids).map((id) => index.get(id).then(handleEntry));
-    const sorted = await Promise.all(promises).then(() => events.sort(sortByDate));
+    const handleEntry = (e) => e && eventBuffer.push(e.event);
+    const promises = Array.from(uids).map((uid) => objectStore.get(uid).then(handleEntry));
     trans.commit();
+    const sorted = await Promise.all(promises).then(() => eventBuffer.sort(sortByDate));
     let minLimit = Infinity;
     for (const filter of filters) {
         if (filter.limit && filter.limit < minLimit)
@@ -185,11 +199,11 @@ async function loadEventsById(db, ids, filters) {
 }
 export async function getEventsForFilter(db, filter, indexCache) {
     const ids = await getIdsForFilter(db, filter, indexCache);
-    return await loadEventsById(db, Array.from(ids), [filter]);
+    return await loadEventsByUID(db, Array.from(ids), [filter]);
 }
 export async function getEventsForFilters(db, filters, indexCache) {
     const ids = await getIdsForFilters(db, filters, indexCache);
-    return await loadEventsById(db, Array.from(ids), filters);
+    return await loadEventsByUID(db, Array.from(ids), filters);
 }
 export async function countEventsForFilter(db, filter, indexCache) {
     const ids = await getIdsForFilter(db, filter, indexCache);

package/dist/query-misc.d.ts

@@ -1,7 +1,10 @@
 import type { Event } from "nostr-tools";
 import { NostrIDB } from "./schema.js";
-import { ReplaceableEventAddress } from "./ingest";
-export declare function getEventsFromAddressPointers(db: NostrIDB, pointers: ReplaceableEventAddress[]): Promise<Event<number>[]>;
+export declare function getEventsFromAddressPointers(db: NostrIDB, pointers: {
+    kind: number;
+    pubkey: string;
+    identifier?: string;
+}[]): Promise<Event[]>;
 export declare function countEventsByPubkeys(db: NostrIDB): Promise<Record<string, number>>;
 export declare function countEventsByKind(db: NostrIDB): Promise<Record<string, number>>;
 export declare function countEvents(db: NostrIDB): Promise<number>;

package/dist/query-misc.js

@@ -1,19 +1,18 @@
 export async function getEventsFromAddressPointers(db, pointers) {
     const trans = db.transaction("events", "readonly");
-    const index = trans.objectStore("events").index("addressPointer");
+    const objectStore = trans.objectStore("events");
     const events = {};
     const promises = pointers.map(async (pointer) => {
-        const key = [pointer.kind, pointer.pubkey, pointer.identifier];
-        const row = await index.get(key);
+        const key = `${pointer.kind}:${pointer.pubkey}:${pointer.identifier ?? ""}`;
+        const row = await objectStore.get(key);
         if (row) {
-            const keyStr = key.join(":");
-            const existing = events[keyStr];
+            const existing = events[key];
             if (!existing || row.event.created_at > existing.created_at)
-                events[keyStr] = row.event;
+                events[key] = row.event;
         }
     });
-    const sorted = await Promise.all(promises).then(() => Object.values(events).sort((a, b) => b.created_at - a.created_at));
     trans.commit();
+    const sorted = await Promise.all(promises).then(() => Object.values(events).sort((a, b) => b.created_at - a.created_at));
     return sorted;
 }
 export async function countEventsByPubkeys(db) {

package/dist/relay.js

@@ -1,4 +1,4 @@
-import { matchFilters } from "nostr-tools";
+import { kinds, matchFilters } from "nostr-tools";
 import { WriteQueue } from "./write-queue";
 import { countEventsForFilters, getEventsForFilters } from "./query-filter";
 import { sortByDate } from "./utils";
@@ -41,8 +41,10 @@ export class CacheRelay {
         }
     }
     async publish(event) {
-        this.writeQueue.addEvent(event);
-        this.indexCache.addEventToIndexes(event);
+        if (!kinds.isEphemeralKind(event.kind)) {
+            this.writeQueue.addEvent(event);
+            this.indexCache.addEventToIndexes(event);
+        }
         let subs = 0;
         for (const { onevent, filters } of this.subscriptions) {
             if (onevent && matchFilters(filters, event)) {
@@ -57,7 +59,7 @@ export class CacheRelay {
     }
     async executeSubscription(sub) {
         // load any events from the write queue
-        const eventsFromQueue = this.writeQueue.queue.filter((e) => matchFilters(sub.filters, e));
+        const eventsFromQueue = this.writeQueue.matchPending(sub.filters);
         // get events
         await getEventsForFilters(this.db, sub.filters, this.indexCache).then((filterEvents) => {
             if (sub.onevent) {
@@ -70,6 +72,7 @@ export class CacheRelay {
                     : filterEvents;
                 for (const event of events) {
                     sub.onevent(event);
+                    this.writeQueue.useEvent(event);
                 }
             }
             if (sub.oneose)

package/dist/schema.d.ts

@@ -3,11 +3,10 @@ import type { Event } from "nostr-tools";
 export type NostrIDB = IDBPDatabase<Schema>;
 export interface Schema extends DBSchema {
     events: {
-        key: "id";
+        key: string;
         value: {
             event: Event;
             tags: string[];
-            identifier?: string;
         };
         indexes: {
             id: string;
@@ -15,30 +14,16 @@ export interface Schema extends DBSchema {
             kind: number;
             created_at: number;
             tags: string;
-            addressPointer: [number, string, string | undefined];
-        };
-    };
-    seen: {
-        key: "id";
-        value: {
-            id: string;
-            date: number;
-            relays: string[];
-        };
-        indexes: {
-            id: string;
-            date: string;
-            relay: string;
         };
     };
     used: {
-        key: "id";
+        key: string;
         value: {
-            id: string;
+            uid: string;
             date: number;
         };
         indexes: {
-            id: string;
+            uid: string;
             date: string;
         };
     };

package/dist/write-queue.d.ts

@@ -1,11 +1,15 @@
-import type { Event } from "nostr-tools";
+import { type Filter, type NostrEvent } from "nostr-tools";
 import { NostrIDB } from "./schema";
 export declare class WriteQueue {
     db: NostrIDB;
-    queue: Event[];
+    private eventQueue;
+    private lastUsedQueue;
     constructor(db: NostrIDB);
-    addEvent(event: Event): void;
-    addEvents(events: Event[]): void;
-    flush(count?: number): Promise<number>;
+    addEvent(event: NostrEvent): void;
+    addEvents(events: NostrEvent[]): void;
+    useEvent(event: NostrEvent): void;
+    useEvents(events: NostrEvent[]): void;
+    matchPending(filters: Filter[]): import("nostr-tools").Event[];
+    flush(count?: number): Promise<void>;
     clear(): void;
 }

package/dist/write-queue.js

@@ -1,29 +1,47 @@
-import { addEvents, updateUsed } from "./ingest";
+import { matchFilters } from "nostr-tools";
+import { addEvents, getEventUID, updateUsed } from "./ingest";
 export class WriteQueue {
     db;
-    queue = [];
+    eventQueue = [];
+    lastUsedQueue = new Set();
     constructor(db) {
         this.db = db;
     }
     addEvent(event) {
-        this.queue.push(event);
+        this.eventQueue.push(event);
     }
     addEvents(events) {
-        this.queue.push(...events);
+        this.eventQueue.push(...events);
+    }
+    useEvent(event) {
+        this.lastUsedQueue.add(getEventUID(event));
+    }
+    useEvents(events) {
+        for (const event of events)
+            this.lastUsedQueue.add(getEventUID(event));
+    }
+    matchPending(filters) {
+        return this.eventQueue.filter((e) => matchFilters(filters, e));
     }
     async flush(count = 1000) {
-        const events = [];
-        for (let i = 0; i < count; i++) {
-            const event = this.queue.shift();
-            if (!event)
-                break;
-            events.push(event);
+        if (this.eventQueue.length > 0) {
+            const events = [];
+            for (let i = 0; i < count; i++) {
+                const event = this.eventQueue.shift();
+                if (!event)
+                    break;
+                events.push(event);
+            }
+            await addEvents(this.db, events);
+        }
+        if (this.lastUsedQueue.size > 0) {
+            console.time("Save Used");
+            await updateUsed(this.db, this.lastUsedQueue);
+            console.timeEnd("Save Used");
+            this.lastUsedQueue.clear();
         }
-        await addEvents(this.db, events);
-        await updateUsed(this.db, events.map((e) => e.id));
-        return events.length;
     }
     clear() {
-        this.queue = [];
+        this.eventQueue = [];
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nostr-idb",
-  "version": "0.2.0",
+  "version": "1.0.0",
   "description": "A collection of helper methods for storing nostr events in IndexedDB",
   "author": {
     "name": "hzrd149"
@@ -24,7 +24,7 @@
   },
   "dependencies": {
     "idb": "^8.0.0",
-    "nostr-tools": "^1.17.0"
+    "nostr-tools": "^2.1.3"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",