applesauce-loaders 5.1.0 → 6.1.0

package/dist/helpers/address-pointer.d.ts

@@ -7,8 +7,8 @@ export declare function createFiltersFromAddressPointers(pointers: AddressPointe
 /** Checks if a relay will understand an address pointer */
 export declare function isLoadableAddressPointer<T extends AddressPointerWithoutD>(pointer: T): boolean;
 /** Group an array of address pointers by kind */
-export declare function groupAddressPointersByKind(pointers: AddressPointerWithoutD[]): Map<number, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByKind<T extends AddressPointerWithoutD>(pointers: T[]): Map<number, T[]>;
 /** Group an array of address pointers by pubkey */
-export declare function groupAddressPointersByPubkey(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByPubkey<T extends AddressPointerWithoutD>(pointers: T[]): Map<string, T[]>;
 /** Groups address pointers by kind or pubkey depending on which is most optimal */
-export declare function groupAddressPointersByPubkeyOrKind(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]> | Map<number, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByPubkeyOrKind<T extends AddressPointerWithoutD>(pointers: T[]): Map<string, T[]> | Map<number, T[]>;

package/dist/helpers/address-pointer.js

@@ -16,14 +16,14 @@ export function createFiltersFromAddressPointers(pointers) {
     const replaceable = pointers.filter((p) => isReplaceableKind(p.kind));
     const addressable = pointers.filter((p) => isAddressableKind(p.kind));
     const filters = [];
-    if (replaceable.length > 0) {
-        const groups = groupAddressPointersByPubkeyOrKind(replaceable);
+    const addGroupFilters = (group) => {
+        if (group.length === 0)
+            return;
+        const groups = groupAddressPointersByPubkeyOrKind(group);
         filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
-    }
-    if (addressable.length > 0) {
-        const groups = groupAddressPointersByPubkeyOrKind(addressable);
-        filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
-    }
+    };
+    addGroupFilters(replaceable);
+    addGroupFilters(addressable);
     return filters;
 }
 /** Checks if a relay will understand an address pointer */

package/dist/loaders/social-graph.d.ts

@@ -1,16 +1,28 @@
+import { IAsyncEventStoreActions, IAsyncEventStoreRead, IEventStoreActions, IEventStoreRead } from "applesauce-core";
 import { NostrEvent } from "applesauce-core/helpers/event";
 import { ProfilePointer } from "applesauce-core/helpers/pointers";
-import { mapEventsToStore } from "applesauce-core/observable";
 import { Observable } from "rxjs";
-import { AddressPointerLoader } from "./address-loader.js";
-/** A loader that loads the social graph of a user out to a set distance */
+import { CacheRequest, UpstreamPool } from "../types.js";
+/**
+ * A loader that loads the social graph of a user out to a set distance.
+ *
+ * Pass `since` (unix seconds) to skip follow lists the relay already knows are older.
+ * When `since` is set and the relay returns nothing for a user, the loader falls back
+ * to `eventStore.getReplaceable(kinds.Contacts, pubkey)` so crawl expansion still
+ * happens from the cached copy.
+ */
 export type SocialGraphLoader = (user: ProfilePointer & {
     distance: number;
+    since?: number;
 }) => Observable<NostrEvent>;
+/** An event store that the social graph loader can both write to and read from */
+export type SocialGraphEventStore = (IEventStoreActions | IAsyncEventStoreActions) & (IEventStoreRead | IAsyncEventStoreRead);
 export type SocialGraphLoaderOptions = Partial<{
-    /** An event store to send all the events to */
-    eventStore?: Parameters<typeof mapEventsToStore>[0];
-    /** The number of parallel requests to make (default 300) */
+    /** An event store to send all the events to and fall back to when the relay returns nothing */
+    eventStore?: SocialGraphEventStore;
+    /** A method used to load events from a local cache */
+    cacheRequest: CacheRequest;
+    /** The number of parallel contacts to load at once (default 300) */
     parallel: number;
     /** Extra relays to load from */
     extraRelays?: string[] | Observable<string[]>;
@@ -18,4 +30,4 @@ export type SocialGraphLoaderOptions = Partial<{
     hints?: boolean;
 }>;
 /** Create a social graph loader */
-export declare function createSocialGraphLoader(addressLoader: AddressPointerLoader, opts?: SocialGraphLoaderOptions): SocialGraphLoader;
+export declare function createSocialGraphLoader(pool: UpstreamPool, opts?: SocialGraphLoaderOptions): SocialGraphLoader;

package/dist/loaders/social-graph.js

@@ -2,50 +2,108 @@ import { getPublicContacts } from "applesauce-core/helpers";
 import { kinds } from "applesauce-core/helpers/event";
 import { mergeRelaySets } from "applesauce-core/helpers/relays";
 import { mapEventsToStore } from "applesauce-core/observable";
-import { firstValueFrom, identity, isObservable, lastValueFrom, toArray } from "rxjs";
+import { catchError, EMPTY, filter, firstValueFrom, identity, isObservable, tap } from "rxjs";
+import { makeCacheRequest } from "../helpers/cache.js";
+import { wrapUpstreamPool } from "../helpers/upstream.js";
 import { wrapGeneratorFunction } from "../operators/generator.js";
+/** Create filters for loading contact lists, keeping different since windows separate. */
+function createContactsFilters(pointers) {
+    const bySince = new Map();
+    for (const pointer of pointers) {
+        const authors = bySince.get(pointer.since);
+        if (authors)
+            authors.push(pointer.pubkey);
+        else
+            bySince.set(pointer.since, [pointer.pubkey]);
+    }
+    return Array.from(bySince.entries()).map(([since, authors]) => {
+        const filter = { kinds: [kinds.Contacts], authors };
+        if (since !== undefined)
+            filter.since = since;
+        return filter;
+    });
+}
+function isRequestedContactsEvent(event, pointers) {
+    return event.kind === kinds.Contacts && pointers.some((pointer) => pointer.pubkey === event.pubkey);
+}
+function getBatchRelays(pointers, baseRelays, hints) {
+    if (hints)
+        return mergeRelaySets(baseRelays, ...pointers.map((pointer) => pointer.relays));
+    else
+        return baseRelays;
+}
 /** Create a social graph loader */
-export function createSocialGraphLoader(addressLoader, opts) {
+export function createSocialGraphLoader(pool, opts) {
+    const request = wrapUpstreamPool(pool);
     return wrapGeneratorFunction(async function* (user) {
         const seen = new Set();
+        // Carry `since` on every queue entry so descendants share the same window
         const queue = [user];
         // Maximum parallel requests (default to 300)
         const maxParallel = opts?.parallel ?? 300;
         // get the relays to load from
-        const relays = mergeRelaySets(user.relays, isObservable(opts?.extraRelays) ? await firstValueFrom(opts?.extraRelays) : opts?.extraRelays);
+        const baseRelays = mergeRelaySets(user.relays, isObservable(opts?.extraRelays) ? await firstValueFrom(opts?.extraRelays) : opts?.extraRelays);
         // Keep loading while the queue has items
         while (queue.length > 0) {
             // Process up to maxParallel items at once
             const batch = queue.splice(0, maxParallel);
-            const promises = batch.map(async (pointer) => {
-                const address = {
-                    kind: kinds.Contacts,
-                    pubkey: pointer.pubkey,
-                    relays: opts?.hints ? mergeRelaySets(pointer.relays, relays) : relays,
-                };
-                // load the contacts events
-                const events = await lastValueFrom(addressLoader(address).pipe(
+            let remaining = batch;
+            // Track the latest contacts event per pubkey so we can expand the queue once
+            // the batch observable completes. Using a side-effect here lets us stream every
+            // event out to subscribers as it arrives rather than buffering to arrays.
+            const latestByPubkey = new Map();
+            const trackLatest = tap((event) => {
+                const current = latestByPubkey.get(event.pubkey);
+                if (!current || event.created_at > current.created_at)
+                    latestByPubkey.set(event.pubkey, event);
+            });
+            if (opts?.cacheRequest) {
+                yield makeCacheRequest(opts.cacheRequest, createContactsFilters(batch)).pipe(filter((event) => isRequestedContactsEvent(event, batch)), 
                 // Pass all events to the store if set
                 opts?.eventStore ? mapEventsToStore(opts.eventStore) : identity, 
-                // Conver to array
-                toArray()));
-                if (events.length === 0)
-                    return;
-                const contacts = getPublicContacts(events[events.length - 1]);
+                // Remember the newest contacts event per pubkey for queue expansion
+                trackLatest, 
+                // If the cache throws an error, skip it
+                catchError(() => EMPTY));
+                remaining = batch.filter((pointer) => pointer.since !== undefined || !latestByPubkey.has(pointer.pubkey));
+            }
+            const relays = getBatchRelays(remaining, baseRelays, opts?.hints);
+            if (remaining.length > 0 && relays.length > 0) {
+                // Yield the relay observable so every event streams out to subscribers
+                // as it arrives from the relay.
+                yield request(relays, createContactsFilters(remaining)).pipe(filter((event) => isRequestedContactsEvent(event, remaining)), 
+                // Pass all events to the store if set
+                opts?.eventStore ? mapEventsToStore(opts.eventStore) : identity, 
+                // Remember the newest contacts event per pubkey for queue expansion
+                trackLatest, 
+                // If the relay request throws an error, continue expanding from cache/store
+                catchError(() => EMPTY));
+            }
+            // Batch has completed — expand the queue using the latest contacts event
+            // for each pointer, falling back to the event store if the relay returned
+            // nothing (typically because `since` let it skip).
+            for (const pointer of batch) {
+                let latest = latestByPubkey.get(pointer.pubkey);
+                if (!latest && opts?.eventStore) {
+                    const cached = await opts.eventStore.getReplaceable(kinds.Contacts, pointer.pubkey);
+                    if (cached)
+                        latest = cached;
+                }
+                if (!latest)
+                    continue;
                 // if the distance is greater than 0, add the contacts to the queue
                 if (pointer.distance > 0) {
+                    const contacts = getPublicContacts(latest);
                     for (const contact of contacts) {
                         // Dont add any contacts that have already been seen
                         if (seen.has(contact.pubkey))
                             continue;
                         seen.add(contact.pubkey);
-                        // Add to queue
-                        queue.push({ ...contact, distance: pointer.distance - 1 });
+                        // Forward `since` onto descendants so the whole crawl shares the window
+                        queue.push({ ...contact, distance: pointer.distance - 1, since: pointer.since });
                     }
                 }
-            });
-            // Wait for all parallel operations to complete
-            await Promise.all(promises);
+            }
         }
     });
 }

package/dist/loaders/timeline-loader.js

@@ -42,13 +42,18 @@ export function loadBackwardBlocks(request, opts) {
             loading = true;
             // Count returned events so complete set
             let count = 0;
+            let minCreatedAt;
+            const until = cursor;
             log?.(`Loading block since:${cursor}`);
             // Request the next block of events
-            return request(cursor).pipe(tap((event) => {
+            return request(until).pipe(tap((event) => {
                 count++;
-                // Track the minimum created_at seen from the events
-                cursor = Math.min(event.created_at, cursor ?? Infinity);
+                minCreatedAt = Math.min(event.created_at, minCreatedAt ?? Infinity);
             }), finalize(() => {
+                // NIP-01 defines `until` as inclusive. Move past the oldest event in
+                // the block so the next request does not repeat the boundary second.
+                if (minCreatedAt !== undefined)
+                    cursor = minCreatedAt - 1;
                 loading = false;
                 complete = count === 0;
                 log?.(`Found ${count} events`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "applesauce-loaders",
-  "version": "5.1.0",
+  "version": "6.1.0",
   "description": "A collection of observable based loaders built on rx-nostr",
   "type": "module",
   "main": "dist/index.js",
@@ -52,13 +52,13 @@
     }
   },
   "dependencies": {
-    "applesauce-core": "^5.1.0",
+    "applesauce-core": "^6.1.0",
     "nanoid": "^5.0.9",
     "rxjs": "^7.8.1"
   },
   "devDependencies": {
     "@hirez_io/observer-spy": "^2.2.0",
-    "applesauce-signers": "^5.1.0",
+    "applesauce-signers": "^6.0.0",
     "rimraf": "^6.0.1",
     "typescript": "^5.8.3",
     "vitest": "^4.0.15",