applesauce-loaders 5.0.2 → 6.0.0

package/README.md

@@ -2,7 +2,7 @@
 
 A collection of functional loading methods to make common event loading patterns easier.
 
-[Documentation](https://hzrd149.github.io/applesauce/loaders/package.html) [typedoc](https://hzrd149.github.io/applesauce/typedoc/modules/applesauce-loaders.html)
+[Documentation](https://applesauce.build/loaders/package.html) [typedoc](https://applesauce.build/typedoc/modules/applesauce-loaders.html)
 
 ## Address Loader
 

package/dist/helpers/address-pointer.d.ts

@@ -7,8 +7,8 @@ export declare function createFiltersFromAddressPointers(pointers: AddressPointe
 /** Checks if a relay will understand an address pointer */
 export declare function isLoadableAddressPointer<T extends AddressPointerWithoutD>(pointer: T): boolean;
 /** Group an array of address pointers by kind */
-export declare function groupAddressPointersByKind(pointers: AddressPointerWithoutD[]): Map<number, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByKind<T extends AddressPointerWithoutD>(pointers: T[]): Map<number, T[]>;
 /** Group an array of address pointers by pubkey */
-export declare function groupAddressPointersByPubkey(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByPubkey<T extends AddressPointerWithoutD>(pointers: T[]): Map<string, T[]>;
 /** Groups address pointers by kind or pubkey depending on which is most optimal */
-export declare function groupAddressPointersByPubkeyOrKind(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]> | Map<number, AddressPointerWithoutD[]>;
+export declare function groupAddressPointersByPubkeyOrKind<T extends AddressPointerWithoutD>(pointers: T[]): Map<string, T[]> | Map<number, T[]>;

package/dist/helpers/address-pointer.js

@@ -16,14 +16,14 @@ export function createFiltersFromAddressPointers(pointers) {
     const replaceable = pointers.filter((p) => isReplaceableKind(p.kind));
     const addressable = pointers.filter((p) => isAddressableKind(p.kind));
     const filters = [];
-    if (replaceable.length > 0) {
-        const groups = groupAddressPointersByPubkeyOrKind(replaceable);
+    const addGroupFilters = (group) => {
+        if (group.length === 0)
+            return;
+        const groups = groupAddressPointersByPubkeyOrKind(group);
         filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
-    }
-    if (addressable.length > 0) {
-        const groups = groupAddressPointersByPubkeyOrKind(addressable);
-        filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
-    }
+    };
+    addGroupFilters(replaceable);
+    addGroupFilters(addressable);
     return filters;
 }
 /** Checks if a relay will understand an address pointer */

package/dist/helpers/async-map.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Resolves multiple promises concurrently, similar to RxJS combineLatest but for async operations.
+ * Each promise races against the provided timeout. If a promise doesn't resolve within the timeout,
+ * its value will be `undefined` in the returned object.
+ *
+ * @param map - An object where each value is a Promise
+ * @param timeout - Global timeout in milliseconds for all fields. If a field doesn't resolve within this time, it will be `undefined`
+ * @returns A promise that resolves to an object with the same keys, where each value is either the resolved value or `undefined`
+ *
+ * @example
+ * ```ts
+ * const { profile, mailboxes, notes } = await loadAsyncMap(
+ *   {
+ *     profile: user.profile$.$first(),
+ *     mailboxes: user.mailboxes$.$first(1000),
+ *     notes: lastValueFrom(someObservable),
+ *   },
+ *   30 * 1000, // 30 second timeout
+ * );
+ * ```
+ */
+export declare function loadAsyncMap<T extends Record<string, Promise<any>>>(map: T, timeout: number): Promise<{
+    [K in keyof T]: Awaited<T[K]> | undefined;
+}>;

package/dist/helpers/async-map.js

@@ -0,0 +1,64 @@
+/**
+ * Resolves multiple promises concurrently, similar to RxJS combineLatest but for async operations.
+ * Each promise races against the provided timeout. If a promise doesn't resolve within the timeout,
+ * its value will be `undefined` in the returned object.
+ *
+ * @param map - An object where each value is a Promise
+ * @param timeout - Global timeout in milliseconds for all fields. If a field doesn't resolve within this time, it will be `undefined`
+ * @returns A promise that resolves to an object with the same keys, where each value is either the resolved value or `undefined`
+ *
+ * @example
+ * ```ts
+ * const { profile, mailboxes, notes } = await loadAsyncMap(
+ *   {
+ *     profile: user.profile$.$first(),
+ *     mailboxes: user.mailboxes$.$first(1000),
+ *     notes: lastValueFrom(someObservable),
+ *   },
+ *   30 * 1000, // 30 second timeout
+ * );
+ * ```
+ */
+export async function loadAsyncMap(map, timeout) {
+    // Create a timeout promise that resolves with a special marker after the timeout
+    const TIMEOUT_MARKER = Symbol("timeout");
+    const createTimeoutPromise = () => {
+        return new Promise((resolve) => {
+            setTimeout(() => resolve(TIMEOUT_MARKER), timeout);
+        });
+    };
+    // Race each promise against the timeout
+    // If the promise resolves first, use its value
+    // If the timeout happens first, use undefined
+    // If the promise rejects, catch it and return undefined
+    const entries = Object.entries(map);
+    const results = await Promise.allSettled(entries.map(async ([key, promise]) => {
+        // Wrap promise to handle rejections gracefully and prevent unhandled rejections
+        const safePromise = promise
+            .then((value) => ({ type: "resolved", value }))
+            .catch(() => ({ type: "rejected" }));
+        const result = await Promise.race([
+            safePromise,
+            createTimeoutPromise().then(() => ({ type: "timeout" })),
+        ]);
+        if (result.type === "timeout" || result.type === "rejected") {
+            return [key, undefined];
+        }
+        else {
+            return [key, result.value];
+        }
+    }));
+    // Extract values from settled results, defaulting to undefined if anything went wrong
+    const extractedResults = results.map((settled, index) => {
+        if (settled.status === "fulfilled") {
+            return settled.value;
+        }
+        else {
+            // If the outer promise somehow rejected, return undefined for that key
+            const key = entries[index][0];
+            return [key, undefined];
+        }
+    });
+    // Reconstruct the object with the same keys
+    return Object.fromEntries(extractedResults);
+}

package/dist/helpers/index.d.ts

@@ -1,6 +1,7 @@
-export * from "./dns-identity.js";
+export * from "./address-pointer.js";
+export * from "./async-map.js";
 export * from "./cache.js";
+export * from "./dns-identity.js";
 export * from "./event-pointer.js";
-export * from "./address-pointer.js";
 export * from "./loaders.js";
 export * from "./upstream.js";

package/dist/helpers/index.js

@@ -1,6 +1,7 @@
-export * from "./dns-identity.js";
+export * from "./address-pointer.js";
+export * from "./async-map.js";
 export * from "./cache.js";
+export * from "./dns-identity.js";
 export * from "./event-pointer.js";
-export * from "./address-pointer.js";
 export * from "./loaders.js";
 export * from "./upstream.js";

package/dist/loaders/social-graph.d.ts

@@ -1,16 +1,28 @@
+import { IAsyncEventStoreActions, IAsyncEventStoreRead, IEventStoreActions, IEventStoreRead } from "applesauce-core";
 import { NostrEvent } from "applesauce-core/helpers/event";
 import { ProfilePointer } from "applesauce-core/helpers/pointers";
-import { mapEventsToStore } from "applesauce-core/observable";
 import { Observable } from "rxjs";
-import { AddressPointerLoader } from "./address-loader.js";
-/** A loader that loads the social graph of a user out to a set distance */
+import { CacheRequest, UpstreamPool } from "../types.js";
+/**
+ * A loader that loads the social graph of a user out to a set distance.
+ *
+ * Pass `since` (unix seconds) to skip follow lists the relay already knows are older.
+ * When `since` is set and the relay returns nothing for a user, the loader falls back
+ * to `eventStore.getReplaceable(kinds.Contacts, pubkey)` so crawl expansion still
+ * happens from the cached copy.
+ */
 export type SocialGraphLoader = (user: ProfilePointer & {
     distance: number;
+    since?: number;
 }) => Observable<NostrEvent>;
+/** An event store that the social graph loader can both write to and read from */
+export type SocialGraphEventStore = (IEventStoreActions | IAsyncEventStoreActions) & (IEventStoreRead | IAsyncEventStoreRead);
 export type SocialGraphLoaderOptions = Partial<{
-    /** An event store to send all the events to */
-    eventStore?: Parameters<typeof mapEventsToStore>[0];
-    /** The number of parallel requests to make (default 300) */
+    /** An event store to send all the events to and fall back to when the relay returns nothing */
+    eventStore?: SocialGraphEventStore;
+    /** A method used to load events from a local cache */
+    cacheRequest: CacheRequest;
+    /** The number of parallel contacts to load at once (default 300) */
     parallel: number;
     /** Extra relays to load from */
     extraRelays?: string[] | Observable<string[]>;
@@ -18,4 +30,4 @@ export type SocialGraphLoaderOptions = Partial<{
     hints?: boolean;
 }>;
 /** Create a social graph loader */
-export declare function createSocialGraphLoader(addressLoader: AddressPointerLoader, opts?: SocialGraphLoaderOptions): SocialGraphLoader;
+export declare function createSocialGraphLoader(pool: UpstreamPool, opts?: SocialGraphLoaderOptions): SocialGraphLoader;

package/dist/loaders/social-graph.js

@@ -2,50 +2,108 @@ import { getPublicContacts } from "applesauce-core/helpers";
 import { kinds } from "applesauce-core/helpers/event";
 import { mergeRelaySets } from "applesauce-core/helpers/relays";
 import { mapEventsToStore } from "applesauce-core/observable";
-import { firstValueFrom, identity, isObservable, lastValueFrom, toArray } from "rxjs";
+import { catchError, EMPTY, filter, firstValueFrom, identity, isObservable, tap } from "rxjs";
+import { makeCacheRequest } from "../helpers/cache.js";
+import { wrapUpstreamPool } from "../helpers/upstream.js";
 import { wrapGeneratorFunction } from "../operators/generator.js";
+/** Create filters for loading contact lists, keeping different since windows separate. */
+function createContactsFilters(pointers) {
+    const bySince = new Map();
+    for (const pointer of pointers) {
+        const authors = bySince.get(pointer.since);
+        if (authors)
+            authors.push(pointer.pubkey);
+        else
+            bySince.set(pointer.since, [pointer.pubkey]);
+    }
+    return Array.from(bySince.entries()).map(([since, authors]) => {
+        const filter = { kinds: [kinds.Contacts], authors };
+        if (since !== undefined)
+            filter.since = since;
+        return filter;
+    });
+}
+function isRequestedContactsEvent(event, pointers) {
+    return event.kind === kinds.Contacts && pointers.some((pointer) => pointer.pubkey === event.pubkey);
+}
+function getBatchRelays(pointers, baseRelays, hints) {
+    if (hints)
+        return mergeRelaySets(baseRelays, ...pointers.map((pointer) => pointer.relays));
+    else
+        return baseRelays;
+}
 /** Create a social graph loader */
-export function createSocialGraphLoader(addressLoader, opts) {
+export function createSocialGraphLoader(pool, opts) {
+    const request = wrapUpstreamPool(pool);
     return wrapGeneratorFunction(async function* (user) {
         const seen = new Set();
+        // Carry `since` on every queue entry so descendants share the same window
         const queue = [user];
         // Maximum parallel requests (default to 300)
         const maxParallel = opts?.parallel ?? 300;
         // get the relays to load from
-        const relays = mergeRelaySets(user.relays, isObservable(opts?.extraRelays) ? await firstValueFrom(opts?.extraRelays) : opts?.extraRelays);
+        const baseRelays = mergeRelaySets(user.relays, isObservable(opts?.extraRelays) ? await firstValueFrom(opts?.extraRelays) : opts?.extraRelays);
         // Keep loading while the queue has items
         while (queue.length > 0) {
             // Process up to maxParallel items at once
             const batch = queue.splice(0, maxParallel);
-            const promises = batch.map(async (pointer) => {
-                const address = {
-                    kind: kinds.Contacts,
-                    pubkey: pointer.pubkey,
-                    relays: opts?.hints ? mergeRelaySets(pointer.relays, relays) : relays,
-                };
-                // load the contacts events
-                const events = await lastValueFrom(addressLoader(address).pipe(
+            let remaining = batch;
+            // Track the latest contacts event per pubkey so we can expand the queue once
+            // the batch observable completes. Using a side-effect here lets us stream every
+            // event out to subscribers as it arrives rather than buffering to arrays.
+            const latestByPubkey = new Map();
+            const trackLatest = tap((event) => {
+                const current = latestByPubkey.get(event.pubkey);
+                if (!current || event.created_at > current.created_at)
+                    latestByPubkey.set(event.pubkey, event);
+            });
+            if (opts?.cacheRequest) {
+                yield makeCacheRequest(opts.cacheRequest, createContactsFilters(batch)).pipe(filter((event) => isRequestedContactsEvent(event, batch)), 
                 // Pass all events to the store if set
                 opts?.eventStore ? mapEventsToStore(opts.eventStore) : identity, 
-                // Conver to array
-                toArray()));
-                if (events.length === 0)
-                    return;
-                const contacts = getPublicContacts(events[events.length - 1]);
+                // Remember the newest contacts event per pubkey for queue expansion
+                trackLatest, 
+                // If the cache throws an error, skip it
+                catchError(() => EMPTY));
+                remaining = batch.filter((pointer) => pointer.since !== undefined || !latestByPubkey.has(pointer.pubkey));
+            }
+            const relays = getBatchRelays(remaining, baseRelays, opts?.hints);
+            if (remaining.length > 0 && relays.length > 0) {
+                // Yield the relay observable so every event streams out to subscribers
+                // as it arrives from the relay.
+                yield request(relays, createContactsFilters(remaining)).pipe(filter((event) => isRequestedContactsEvent(event, remaining)), 
+                // Pass all events to the store if set
+                opts?.eventStore ? mapEventsToStore(opts.eventStore) : identity, 
+                // Remember the newest contacts event per pubkey for queue expansion
+                trackLatest, 
+                // If the relay request throws an error, continue expanding from cache/store
+                catchError(() => EMPTY));
+            }
+            // Batch has completed — expand the queue using the latest contacts event
+            // for each pointer, falling back to the event store if the relay returned
+            // nothing (typically because `since` let it skip).
+            for (const pointer of batch) {
+                let latest = latestByPubkey.get(pointer.pubkey);
+                if (!latest && opts?.eventStore) {
+                    const cached = await opts.eventStore.getReplaceable(kinds.Contacts, pointer.pubkey);
+                    if (cached)
+                        latest = cached;
+                }
+                if (!latest)
+                    continue;
                 // if the distance is greater than 0, add the contacts to the queue
                 if (pointer.distance > 0) {
+                    const contacts = getPublicContacts(latest);
                     for (const contact of contacts) {
                         // Dont add any contacts that have already been seen
                         if (seen.has(contact.pubkey))
                             continue;
                         seen.add(contact.pubkey);
-                        // Add to queue
-                        queue.push({ ...contact, distance: pointer.distance - 1 });
+                        // Forward `since` onto descendants so the whole crawl shares the window
+                        queue.push({ ...contact, distance: pointer.distance - 1, since: pointer.since });
                     }
                 }
-            });
-            // Wait for all parallel operations to complete
-            await Promise.all(promises);
+            }
         }
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "applesauce-loaders",
-  "version": "5.0.2",
+  "version": "6.0.0",
   "description": "A collection of observable based loaders built on rx-nostr",
   "type": "module",
   "main": "dist/index.js",
@@ -52,13 +52,13 @@
     }
   },
   "dependencies": {
-    "applesauce-core": "^5.0.0",
+    "applesauce-core": "^6.0.0",
     "nanoid": "^5.0.9",
     "rxjs": "^7.8.1"
   },
   "devDependencies": {
     "@hirez_io/observer-spy": "^2.2.0",
-    "applesauce-signers": "^5.0.0",
+    "applesauce-signers": "^6.0.0",
     "rimraf": "^6.0.1",
     "typescript": "^5.8.3",
     "vitest": "^4.0.15",