applesauce-loaders 0.0.0-next-20250123214405 → 0.0.0-next-20250124150519

package/dist/helpers/address-pointer.d.ts

@@ -0,0 +1,25 @@
+import { AddressPointerWithoutD } from "applesauce-core/helpers";
+import { Filter } from "nostr-tools";
+import { AddressPointer } from "nostr-tools/nip19";
+/** Converts an array of address pointers to a filter */
+export declare function createFilterFromAddressPointers(pointers: AddressPointerWithoutD[] | AddressPointer[]): Filter;
+/** Takes a set of address pointers, groups them, then returns filters for the groups */
+export declare function createFiltersFromAddressPointers(pointers: AddressPointerWithoutD[]): Filter[];
+/** Checks if a relay will understand an address pointer */
+export declare function isLoadableAddressPointer<T extends AddressPointerWithoutD>(pointer: T): boolean;
+/** Group an array of address pointers by kind */
+export declare function groupAddressPointersByKind(pointers: AddressPointerWithoutD[]): Map<number, AddressPointerWithoutD[]>;
+/** Group an array of address pointers by pubkey */
+export declare function groupAddressPointersByPubkey(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]>;
+/** Groups address pointers by kind or pubkey depending on which is most optimal */
+export declare function groupAddressPointersByPubkeyOrKind(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]> | Map<number, AddressPointerWithoutD[]>;
+export declare function getRelaysFromPointers(pointers: AddressPointerWithoutD[]): Set<string>;
+export declare function getAddressPointerId<T extends AddressPointerWithoutD>(pointer: T): string;
+/** deduplicates an array of address pointers and merges their relays array */
+export declare function consolidateAddressPointers<T extends {
+    kind: number;
+    pubkey: string;
+    identifier?: string;
+    relays?: string[];
+    force?: boolean;
+}>(pointers: T[]): T[];

package/dist/helpers/address-pointer.js

@@ -0,0 +1,102 @@
+import { getReplaceableUID } from "applesauce-core/helpers";
+import { isParameterizedReplaceableKind, isReplaceableKind } from "nostr-tools/kinds";
+import { unique } from "./array.js";
+/** Converts an array of address pointers to a filter */
+export function createFilterFromAddressPointers(pointers) {
+    const filter = {};
+    filter.kinds = unique(pointers.map((p) => p.kind));
+    filter.authors = unique(pointers.map((p) => p.pubkey));
+    const identifiers = unique(pointers.map((p) => p.identifier).filter((d) => !!d));
+    if (identifiers.length > 0)
+        filter["#d"] = identifiers;
+    return filter;
+}
+/** Takes a set of address pointers, groups them, then returns filters for the groups */
+export function createFiltersFromAddressPointers(pointers) {
+    // split the points in to two groups so they they don't mix in the filters
+    const parameterizedReplaceable = pointers.filter((p) => isParameterizedReplaceableKind(p.kind));
+    const replaceable = pointers.filter((p) => isReplaceableKind(p.kind));
+    const filters = [];
+    if (replaceable.length > 0) {
+        const groups = groupAddressPointersByPubkeyOrKind(replaceable);
+        filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
+    }
+    if (parameterizedReplaceable.length > 0) {
+        const groups = groupAddressPointersByPubkeyOrKind(parameterizedReplaceable);
+        filters.push(...Array.from(groups.values()).map(createFilterFromAddressPointers));
+    }
+    return filters;
+}
+/** Checks if a relay will understand an address pointer */
+export function isLoadableAddressPointer(pointer) {
+    if (isParameterizedReplaceableKind(pointer.kind))
+        return !!pointer.identifier;
+    else
+        return isReplaceableKind(pointer.kind);
+}
+/** Group an array of address pointers by kind */
+export function groupAddressPointersByKind(pointers) {
+    const byKind = new Map();
+    for (const pointer of pointers) {
+        if (byKind.has(pointer.kind))
+            byKind.get(pointer.kind).push(pointer);
+        else
+            byKind.set(pointer.kind, [pointer]);
+    }
+    return byKind;
+}
+/** Group an array of address pointers by pubkey */
+export function groupAddressPointersByPubkey(pointers) {
+    const byPubkey = new Map();
+    for (const pointer of pointers) {
+        if (byPubkey.has(pointer.pubkey))
+            byPubkey.get(pointer.pubkey).push(pointer);
+        else
+            byPubkey.set(pointer.pubkey, [pointer]);
+    }
+    return byPubkey;
+}
+/** Groups address pointers by kind or pubkey depending on which is most optimal */
+export function groupAddressPointersByPubkeyOrKind(pointers) {
+    const kinds = new Set(pointers.map((p) => p.kind));
+    const pubkeys = new Set(pointers.map((p) => p.pubkey));
+    return pubkeys.size < kinds.size ? groupAddressPointersByPubkey(pointers) : groupAddressPointersByKind(pointers);
+}
+export function getRelaysFromPointers(pointers) {
+    const relays = new Set();
+    for (const pointer of pointers) {
+        if (!pointer.relays)
+            continue;
+        for (const relay of pointer.relays) {
+            relays.add(relay);
+        }
+    }
+    return relays;
+}
+export function getAddressPointerId(pointer) {
+    return getReplaceableUID(pointer.kind, pointer.pubkey, pointer.identifier);
+}
+/** deduplicates an array of address pointers and merges their relays array */
+export function consolidateAddressPointers(pointers) {
+    const byCoordinate = new Map();
+    for (const pointer of pointers) {
+        let cord = getAddressPointerId(pointer);
+        let existing = byCoordinate.get(cord);
+        if (existing) {
+            // override force flag
+            if (pointer.force)
+                existing.force = pointer.force;
+            // merge relays
+            if (pointer.relays) {
+                if (!existing.relays)
+                    existing.relays = [...pointer.relays];
+                else
+                    existing.relays = [...existing.relays, ...pointer.relays.filter((r) => !existing.relays.includes(r))];
+            }
+        }
+        else {
+            byCoordinate.set(cord, pointer);
+        }
+    }
+    return Array.from(byCoordinate.values());
+}

package/dist/helpers/address-pointer.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/helpers/address-pointer.test.js

@@ -0,0 +1,19 @@
+import { describe, it, expect } from "vitest";
+import { createFiltersFromAddressPointers } from "./address-pointer.js";
+import { kinds } from "nostr-tools";
+describe("address pointer helpers", () => {
+    describe("createFiltersFromAddressPointers", () => {
+        it("should separate replaceable and parameterized replaceable pointers", () => {
+            expect(createFiltersFromAddressPointers([
+                { kind: kinds.BookmarkList, pubkey: "pubkey" },
+                { kind: kinds.Metadata, pubkey: "pubkey" },
+                { kind: kinds.Metadata, pubkey: "pubkey2" },
+                { kind: kinds.Bookmarksets, identifier: "funny", pubkey: "pubkey" },
+            ])).toEqual(expect.arrayContaining([
+                { kinds: [kinds.Metadata], authors: ["pubkey", "pubkey2"] },
+                { kinds: [kinds.BookmarkList], authors: ["pubkey"] },
+                { "#d": ["funny"], authors: ["pubkey"], kinds: [kinds.Bookmarksets] },
+            ]));
+        });
+    });
+});

package/dist/helpers/array.d.ts

@@ -0,0 +1,4 @@
+/** Does not preserve order */
+export declare function unique<T>(arr: T[]): T[];
+/** split array into a set of arrays no larger than batchSize */
+export declare function reduceToBatches<T>(arr: T[], batchSize: number): T[][];

package/dist/helpers/array.js

@@ -0,0 +1,21 @@
+/** Does not preserve order */
+export function unique(arr) {
+    return Array.from(new Set(arr));
+}
+/** split array into a set of arrays no larger than batchSize */
+export function reduceToBatches(arr, batchSize) {
+    const batches = [];
+    for (const value of arr) {
+        if (batches.length === 0) {
+            batches.push([value]);
+            continue;
+        }
+        const current = batches[batches.length - 1];
+        if (current.length <= batchSize) {
+            current.push(value);
+        }
+        else
+            batches.push([value]);
+    }
+    return batches;
+}

package/dist/helpers/event-pointer.d.ts

@@ -0,0 +1,5 @@
+/** deduplicates an array of event pointers and merges their relays array */
+export declare function consolidateEventPointers<T extends {
+    id: string;
+    relays?: string[];
+}>(pointers: T[]): T[];

package/dist/helpers/event-pointer.js

@@ -0,0 +1,19 @@
+/** deduplicates an array of event pointers and merges their relays array */
+export function consolidateEventPointers(pointers) {
+    let ids = new Map();
+    for (let pointer of pointers) {
+        let existing = ids.get(pointer.id);
+        if (existing) {
+            // merge relays
+            if (pointer.relays) {
+                if (!existing.relays)
+                    existing.relays = [...pointer.relays];
+                else
+                    existing.relays = [...existing.relays, ...pointer.relays.filter((r) => !existing.relays.includes(r))];
+            }
+        }
+        else
+            ids.set(pointer.id, pointer);
+    }
+    return Array.from(ids.values());
+}

package/dist/helpers/pointer.d.ts

@@ -0,0 +1,3 @@
+export declare function groupByRelay<T extends {
+    relays?: string[];
+}>(pointers: T[], defaultKey?: string): Map<string, T[]>;

package/dist/helpers/pointer.js

@@ -0,0 +1,13 @@
+export function groupByRelay(pointers, defaultKey) {
+    let byRelay = new Map();
+    for (const pointer of pointers) {
+        let relays = pointer.relays?.length ? pointer.relays : defaultKey ? [defaultKey] : [];
+        for (const relay of relays) {
+            if (!byRelay.has(relay))
+                byRelay.set(relay, [pointer]);
+            else
+                byRelay.get(relay)?.push(pointer);
+        }
+    }
+    return byRelay;
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./loaders/index.js";
+export * as Operators from "./operators/index.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from "./loaders/index.js";
+export * as Operators from "./operators/index.js";

package/dist/loaders/index.d.ts

@@ -0,0 +1,5 @@
+export * from "./loader.js";
+export * from "./replaceable-loader.js";
+export * from "./single-event-loader.js";
+export * from "./single-relay-replaceable-loader.js";
+export * from "./user-sets-loader.js";

package/dist/loaders/index.js

@@ -0,0 +1,5 @@
+export * from "./loader.js";
+export * from "./replaceable-loader.js";
+export * from "./single-event-loader.js";
+export * from "./single-relay-replaceable-loader.js";
+export * from "./user-sets-loader.js";

package/dist/loaders/loader.d.ts

@@ -0,0 +1,16 @@
+import { Filter, NostrEvent } from "nostr-tools";
+import { Observable, OperatorFunction, Subject, Subscribable } from "rxjs";
+export type CacheRequest = (filters: Filter[]) => Observable<NostrEvent>;
+export interface ILoader<T, R> extends Subscribable<R> {
+    next: (value: T) => void;
+    pipe: Observable<R>["pipe"];
+}
+/** Base loader class */
+export declare class Loader<T, R> implements ILoader<T, R> {
+    protected subject: Subject<T>;
+    protected observable: Observable<R>;
+    pipe: Observable<R>["pipe"];
+    subscribe: Observable<R>["subscribe"];
+    constructor(transform: OperatorFunction<T, R>);
+    next(value: T): void;
+}

package/dist/loaders/loader.js

@@ -0,0 +1,17 @@
+import { Subject } from "rxjs";
+/** Base loader class */
+export class Loader {
+    subject = new Subject();
+    observable;
+    pipe;
+    subscribe;
+    constructor(transform) {
+        this.observable = this.subject.pipe(transform);
+        // copy pipe function
+        this.pipe = this.observable.pipe.bind(this.observable);
+        this.subscribe = this.observable.subscribe.bind(this.observable);
+    }
+    next(value) {
+        this.subject.next(value);
+    }
+}

package/dist/loaders/replaceable-loader.d.ts

@@ -0,0 +1,28 @@
+import { EventPacket, RxNostr } from "rx-nostr";
+import { logger } from "applesauce-core";
+import { CacheRequest, Loader } from "./loader.js";
+export type LoadableAddressPointer = {
+    kind: number;
+    pubkey: string;
+    /** Optional "d" tag for paramaritized replaceable */
+    identifier?: string;
+    /** Relays to load from */
+    relays?: string[];
+    /** Load this address pointer even if it has already been loaded */
+    force?: boolean;
+};
+export type ReplaceableLoaderOptions = {
+    /**
+     * Time interval to buffer requests in ms
+     * @default 1000
+     */
+    bufferTime?: number;
+    /** A method used to load events from a local cache */
+    cacheRequest?: CacheRequest;
+    /** Fallback lookup relays to check when event cant be found */
+    lookupRelays?: string[];
+};
+export declare class ReplaceableLoader extends Loader<LoadableAddressPointer, EventPacket> {
+    log: typeof logger;
+    constructor(rxNostr: RxNostr, opts?: ReplaceableLoaderOptions);
+}

package/dist/loaders/replaceable-loader.js

@@ -0,0 +1,163 @@
+import { share, tap, from, filter, bufferTime, map, mergeMap } from "rxjs";
+import { getEventUID, getReplaceableUID, markFromCache } from "applesauce-core/helpers";
+import { logger } from "applesauce-core";
+import { nanoid } from "nanoid";
+import { Loader } from "./loader.js";
+import { generatorSequence } from "../operators/generator-sequence.js";
+import { addressPointersRequest } from "../operators/address-pointers-request.js";
+import { createFiltersFromAddressPointers, getAddressPointerId, getRelaysFromPointers, isLoadableAddressPointer, } from "../helpers/address-pointer.js";
+import { unique } from "../helpers/array.js";
+/** deep clone a loadable pointer to ensure its safe to modify */
+function cloneLoadablePointer(pointer) {
+    const clone = { ...pointer };
+    if (pointer.relays)
+        clone.relays = [...pointer.relays];
+    return clone;
+}
+/** A generator that tries to load the address pointers from the cache first, then tries the relays */
+function* cacheFirstSequence(rxNostr, pointers, log, opts) {
+    const id = nanoid(8);
+    log = log.extend(id);
+    let remaining = Array.from(pointers);
+    const pointerRelays = Array.from(getRelaysFromPointers(pointers));
+    // handle previous step results and decide if to exit
+    const handleResults = (results) => {
+        if (results.length) {
+            const coordinates = new Set(results.map((p) => getEventUID(p.event)));
+            // if there where results, filter out any pointers that where found
+            remaining = remaining.filter((pointer) => {
+                const found = coordinates.has(getReplaceableUID(pointer.kind, pointer.pubkey, pointer.identifier));
+                if (found && pointer.force !== true)
+                    return false;
+                else
+                    return true;
+            });
+            if (remaining.length === 0)
+                return true;
+        }
+        return false;
+    };
+    // first attempt, load from cache relays
+    if (opts?.cacheRequest) {
+        log(`Checking cache`, remaining);
+        const results = yield from([remaining]).pipe(
+        // convert pointers to filters
+        map(createFiltersFromAddressPointers), 
+        // make requests
+        mergeMap((filters) => opts.cacheRequest(filters)), 
+        // mark the event as from the cache
+        tap((event) => markFromCache(event)), 
+        // convert to event packets
+        map((e) => ({ event: e, from: "cache", subId: "cache", type: "EVENT" })));
+        if (handleResults(results))
+            return;
+    }
+    // load from pointer relays and default relays
+    const defaultRelays = Object.entries(rxNostr.getDefaultRelays())
+        .filter(([_relay, config]) => config.read)
+        .map(([relay]) => relay);
+    const remoteRelays = [...pointerRelays, ...defaultRelays];
+    if (remoteRelays.length > 0) {
+        log(`Requesting`, remoteRelays, remaining);
+        const results = yield from([remaining]).pipe(addressPointersRequest(rxNostr, id, remoteRelays));
+        if (handleResults(results))
+            return;
+    }
+    // finally load from lookup relays
+    if (opts?.lookupRelays) {
+        // make sure we aren't asking a relay twice
+        const relays = opts.lookupRelays.filter((r) => !pointerRelays.includes(r));
+        if (relays.length > 0) {
+            log(`Request from lookup`, relays, remaining);
+            const results = yield from([remaining]).pipe(addressPointersRequest(rxNostr, id, relays));
+            if (handleResults(results))
+                return;
+        }
+    }
+}
+/** Batches address pointers and consolidates them */
+function multiRelayBatcher(buffer) {
+    const requestedFrom = new Map();
+    const requestedDefault = new Set();
+    return (source) => source.pipe(
+    // buffer on time
+    bufferTime(buffer), 
+    // ignore empty buffers
+    filter((buffer) => buffer.length > 0), 
+    // consolidate buffered pointers
+    map((pointers) => {
+        const byId = new Map();
+        for (const pointer of pointers) {
+            const id = getAddressPointerId(pointer);
+            if (byId.has(id)) {
+                // duplicate, merge pointers
+                const current = byId.get(id);
+                // merge relays
+                if (pointer.relays) {
+                    if (current.relays)
+                        current.relays = unique([...current.relays, ...pointer.relays]);
+                    else
+                        current.relays = pointer.relays;
+                }
+                // merge force flag
+                if (pointer.force !== undefined) {
+                    current.force = current.force || pointer.force;
+                }
+            }
+            else
+                byId.set(id, cloneLoadablePointer(pointer));
+        }
+        // return consolidated pointers
+        return Array.from(byId.values());
+    }), 
+    // ignore empty buffer
+    filter((buffer) => buffer.length > 0), 
+    // ensure pointers are only requested from each relay once
+    map((pointers) => {
+        return pointers.filter((pointer) => {
+            const id = getAddressPointerId(pointer);
+            // skip if forced
+            if (pointer.force)
+                return true;
+            // skip if already requested from default relays
+            if (!pointer.relays) {
+                if (requestedDefault.has(id))
+                    return false;
+                else {
+                    requestedDefault.add(id);
+                    return true;
+                }
+            }
+            let set = requestedFrom.get(id);
+            if (!set) {
+                set = new Set();
+                requestedFrom.set(id, set);
+            }
+            // remove any relays that have already been used
+            pointer.relays = pointer.relays.filter((relay) => !set.has(relay));
+            // remember used relays
+            for (const relay of pointer.relays)
+                set.add(relay);
+            // if there are no new relays, ignore pointer
+            return pointer.relays.length > 0;
+        });
+    }), 
+    // ignore empty buffer
+    filter((buffer) => buffer.length > 0));
+}
+export class ReplaceableLoader extends Loader {
+    log = logger.extend("ReplaceableLoader");
+    constructor(rxNostr, opts) {
+        super((source) => {
+            return source.pipe(
+            // filter out invalid pointers
+            filter(isLoadableAddressPointer), 
+            // batch and filter
+            multiRelayBatcher(opts?.bufferTime ?? 1000), 
+            // check cache, relays, lookup relays in that order
+            generatorSequence((pointers) => cacheFirstSequence(rxNostr, pointers, this.log, opts)), 
+            // share the response with all subscribers
+            share());
+        });
+    }
+}

package/dist/loaders/single-event-loader.d.ts

@@ -0,0 +1,26 @@
+import { EventPacket, RxNostr } from "rx-nostr";
+import { logger } from "applesauce-core";
+import { CacheRequest, Loader } from "./loader.js";
+export type LoadableEventPointer = {
+    id: string;
+    /** Relays to load from */
+    relays?: string[];
+};
+export type SingleEventLoaderOptions = {
+    /**
+     * Time interval to buffer requests in ms
+     * @default 1000
+     */
+    bufferTime?: number;
+    /** A method used to load events from a local cache */
+    cacheRequest?: CacheRequest;
+    /**
+     * How long the loader should wait before it allows an event pointer to be refreshed from a relay
+     * @default 60000
+     */
+    refreshTimeout?: number;
+};
+export declare class SingleEventLoader extends Loader<LoadableEventPointer, EventPacket> {
+    log: typeof logger;
+    constructor(rxNostr: RxNostr, opts?: SingleEventLoaderOptions);
+}

package/dist/loaders/single-event-loader.js

@@ -0,0 +1,78 @@
+import { bufferTime, filter, from, map, mergeAll, share, tap } from "rxjs";
+import { createRxOneshotReq } from "rx-nostr";
+import { markFromCache } from "applesauce-core/helpers";
+import { logger } from "applesauce-core";
+import { nanoid } from "nanoid";
+import { Loader } from "./loader.js";
+import { generatorSequence } from "../operators/generator-sequence.js";
+import { distinctRelays } from "../operators/distinct-by-relays.js";
+import { groupByRelay } from "../helpers/pointer.js";
+import { consolidateEventPointers } from "../helpers/event-pointer.js";
+/** Consolidate a batch of event pointers */
+function consolidateLoadableEventPointers() {
+    return (source) => source.pipe(map(consolidateEventPointers));
+}
+function* cacheFirstSequence(rxNostr, pointers, opts, log) {
+    let remaining = [...pointers];
+    const id = nanoid(8);
+    log = log.extend(id);
+    const loaded = (packets) => {
+        const ids = new Set(packets.map((p) => p.event.id));
+        remaining = remaining.filter((p) => !ids.has(p.id));
+    };
+    if (opts?.cacheRequest) {
+        let filter = { ids: remaining.map((e) => e.id) };
+        const results = yield opts.cacheRequest([filter]).pipe(
+        // mark the event as from the cache
+        tap((event) => markFromCache(event)), 
+        // convert to event packets
+        map((e) => ({ event: e, from: "cache", subId: "cache", type: "EVENT" })));
+        if (results.length > 0) {
+            log(`Loaded ${results.length} events from cache`);
+            loaded(results);
+        }
+    }
+    // exit early if all pointers are loaded
+    if (remaining.length === 0)
+        return;
+    let byRelay = groupByRelay(remaining, "default");
+    // load remaining pointers from the relays
+    let results = yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
+        let filter = { ids: pointers.map((e) => e.id) };
+        let count = 0;
+        const req = createRxOneshotReq({ filters: [filter], rxReqId: id });
+        log(`Requesting from ${relay}`, filter.ids);
+        let sub$;
+        // don't specify relay if this is the "default" relay
+        if (relay === "default")
+            sub$ = rxNostr.use(req);
+        else
+            sub$ = rxNostr.use(req, { on: { relays: [relay] } });
+        return sub$.pipe(tap({
+            next: () => count++,
+            complete: () => log(`Completed ${relay}, loaded ${count} events`),
+        }));
+    })).pipe(mergeAll());
+    loaded(results);
+    if (remaining.length > 0) {
+        // failed to find remaining
+        log("Failed to load", remaining.map((p) => p.id));
+    }
+}
+export class SingleEventLoader extends Loader {
+    log = logger.extend("SingleEventLoader");
+    constructor(rxNostr, opts) {
+        let options = opts || {};
+        super((source) => source.pipe(distinctRelays((p) => p.id, options.refreshTimeout ?? 60_000), 
+        // load first from cache
+        bufferTime(opts?.bufferTime ?? 1000), 
+        // ignore empty buffers
+        filter((buffer) => buffer.length > 0), 
+        // ensure there is only one of each event pointer
+        consolidateLoadableEventPointers(), 
+        // run the loader sequence
+        generatorSequence((pointers) => cacheFirstSequence(rxNostr, pointers, options, this.log)), 
+        // share the response with all subscribers
+        share()));
+    }
+}

package/dist/loaders/single-relay-replaceable-loader.d.ts

@@ -0,0 +1,14 @@
+import { EventPacket, RxNostr } from "rx-nostr";
+import { Loader } from "./loader.js";
+import { LoadableAddressPointer } from "./replaceable-loader.js";
+export type SingleRelayReplaceableOptions = Partial<{
+    /**
+     * Wait X ms before making a request
+     * @default 1000
+     */
+    bufferTime: number;
+}>;
+/** A wrapper class around {@link singleRelayBatcher'}, {@link replaceableRequest}, and share */
+export declare class SingleRelayReplaceableLoader extends Loader<LoadableAddressPointer, EventPacket> {
+    constructor(rxNostr: RxNostr, relay: string, opts?: SingleRelayReplaceableOptions);
+}

package/dist/loaders/single-relay-replaceable-loader.js

@@ -0,0 +1,51 @@
+import { bufferTime, filter, from, map, mergeMap, share } from "rxjs";
+import { getReplaceableUID } from "applesauce-core/helpers";
+import { nanoid } from "nanoid";
+import { Loader } from "./loader.js";
+import { isLoadableAddressPointer } from "../helpers/address-pointer.js";
+import { addressPointersRequest } from "../operators/address-pointers-request.js";
+const defaultRelayReplaceableLoaderOptions = {
+    bufferTime: 1000,
+};
+/** Ensures an address pointer is only loaded once from a relay */
+function singleRelayBatcher(opts) {
+    return (source) => {
+        const loaded = new Set();
+        const options = opts ? { ...defaultRelayReplaceableLoaderOptions, ...opts } : defaultRelayReplaceableLoaderOptions;
+        return source.pipe(
+        // buffer the requests, so we only send requests every second
+        bufferTime(options.bufferTime), 
+        // ignore empty batches
+        filter((batch) => batch.length > 0), 
+        // filter out duplicates
+        map((batch) => {
+            const filtered = [];
+            for (const pointer of batch) {
+                const uid = getReplaceableUID(pointer.kind, pointer.pubkey, pointer.identifier);
+                if (!loaded.has(uid)) {
+                    filtered.push(pointer);
+                    loaded.add(uid);
+                }
+                else if (pointer.force)
+                    filtered.push(pointer);
+            }
+            return filtered;
+        }), 
+        // ignore empty batches
+        filter((batch) => batch.length > 0));
+    };
+}
+/** A wrapper class around {@link singleRelayBatcher'}, {@link replaceableRequest}, and share */
+export class SingleRelayReplaceableLoader extends Loader {
+    constructor(rxNostr, relay, opts) {
+        super((source) => source.pipe(
+        // ignore invalid address pointers
+        filter(isLoadableAddressPointer), 
+        // batch and filter
+        singleRelayBatcher(opts), 
+        // breakout the batches so they can complete
+        mergeMap((pointers) => from([pointers]).pipe(addressPointersRequest(rxNostr, nanoid(8), [relay]))), 
+        // share the response with all subscribers
+        share()));
+    }
+}

package/dist/loaders/user-sets-loader.d.ts

@@ -0,0 +1,31 @@
+import { EventPacket, RxNostr } from "rx-nostr";
+import { logger } from "applesauce-core";
+import { CacheRequest, Loader } from "./loader.js";
+export type LoadableSetPointer = {
+    /** A replaceable kind >= 30000 & < 40000 */
+    kind: number;
+    pubkey: string;
+    /** Relays to load from */
+    relays?: string[];
+    /** Load the sets even if it has already been loaded */
+    force?: boolean;
+};
+export type UserSetsLoaderOptions = {
+    /**
+     * Time interval to buffer requests in ms
+     * @default 1000
+     */
+    bufferTime?: number;
+    /** A method used to load events from a local cache */
+    cacheRequest?: CacheRequest;
+    /**
+     * How long the loader should wait before it allows an event pointer to be refreshed from a relay
+     * @default 120000
+     */
+    refreshTimeout?: number;
+};
+/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
+export declare class UserSetsLoader extends Loader<LoadableSetPointer, EventPacket> {
+    log: typeof logger;
+    constructor(rxNostr: RxNostr, opts?: UserSetsLoaderOptions);
+}

package/dist/loaders/user-sets-loader.js

@@ -0,0 +1,64 @@
+import { share, tap, from, filter, map, mergeAll, bufferTime } from "rxjs";
+import { createRxOneshotReq } from "rx-nostr";
+import { markFromCache } from "applesauce-core/helpers";
+import { logger } from "applesauce-core";
+import { nanoid } from "nanoid";
+import { Loader } from "./loader.js";
+import { generatorSequence } from "../operators/generator-sequence.js";
+import { consolidateAddressPointers, createFiltersFromAddressPointers } from "../helpers/address-pointer.js";
+import { groupByRelay } from "../helpers/pointer.js";
+import { distinctRelays } from "../operators/distinct-by-relays.js";
+/** A generator that tries to load the address pointers from the cache first, then tries the relays */
+function* cacheFirstSequence(rxNostr, pointers, log, opts) {
+    const id = nanoid(8);
+    log = log.extend(id);
+    // first attempt, load from cache relays
+    if (opts?.cacheRequest) {
+        log(`Checking cache`);
+        const filters = createFiltersFromAddressPointers(pointers);
+        const results = yield opts.cacheRequest(filters).pipe(
+        // mark the event as from the cache
+        tap((event) => markFromCache(event)), 
+        // convert to event packets
+        map((e) => ({ event: e, from: "cache", subId: "cache", type: "EVENT" })));
+        if (results.length > 0) {
+            log(`Loaded ${results.length} events from cache`);
+        }
+    }
+    let byRelay = groupByRelay(pointers, "default");
+    // load sets from relays
+    yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
+        let filters = createFiltersFromAddressPointers(pointers);
+        let count = 0;
+        const req = createRxOneshotReq({ filters, rxReqId: id });
+        log(`Requesting from ${relay}`, pointers);
+        let sub$;
+        // don't specify relay if this is the "default" relay
+        if (relay === "default")
+            sub$ = rxNostr.use(req);
+        else
+            sub$ = rxNostr.use(req, { on: { relays: [relay] } });
+        return sub$.pipe(tap({
+            next: () => count++,
+            complete: () => log(`Completed ${relay}, loaded ${count} events`),
+        }));
+    })).pipe(mergeAll());
+}
+/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
+export class UserSetsLoader extends Loader {
+    log = logger.extend("UserSetsLoader");
+    constructor(rxNostr, opts) {
+        let options = opts || {};
+        super((source) => source.pipe(distinctRelays((p) => p.kind + ":" + p.pubkey, options.refreshTimeout ?? 120_000), 
+        // load first from cache
+        bufferTime(options?.bufferTime ?? 1000), 
+        // ignore empty buffers
+        filter((buffer) => buffer.length > 0), 
+        // deduplicate address pointers
+        map(consolidateAddressPointers), 
+        // check cache, relays, lookup relays in that order
+        generatorSequence((pointers) => cacheFirstSequence(rxNostr, pointers, this.log, options)), 
+        // share the response with all subscribers
+        share()));
+    }
+}

package/dist/operators/address-pointers-request.d.ts

@@ -0,0 +1,5 @@
+import { EventPacket, RxNostr } from "rx-nostr";
+import { OperatorFunction } from "rxjs";
+import { AddressPointerWithoutD } from "applesauce-core/helpers";
+/** Makes a request to relays for every set of address pointers */
+export declare function addressPointersRequest<T extends AddressPointerWithoutD>(rxNostr: RxNostr, id: string, relays?: string[]): OperatorFunction<T[], EventPacket>;

package/dist/operators/address-pointers-request.js

@@ -0,0 +1,25 @@
+import { map, tap } from "rxjs";
+import { logger } from "applesauce-core";
+import { relaysRequest } from "./relay-request.js";
+import { createFiltersFromAddressPointers } from "../helpers/address-pointer.js";
+/** Makes a request to relays for every set of address pointers */
+export function addressPointersRequest(rxNostr, id, relays) {
+    return (source) => {
+        const log = logger.extend(`addressPointersRequest:${id}`);
+        return source.pipe(
+        // convert pointers to filters
+        map(createFiltersFromAddressPointers), 
+        // make requests
+        tap((filters) => {
+            log(`Requesting`, filters, "from", relays);
+        }), 
+        // make requests
+        relaysRequest(rxNostr, id, relays), 
+        // log when complete
+        tap({
+            complete: () => {
+                log(`Complete`);
+            },
+        }));
+    };
+}

package/dist/operators/distinct-by-relays.d.ts

@@ -0,0 +1,9 @@
+import { OperatorFunction } from "rxjs";
+interface Message {
+    relays?: string[];
+    /** Ignore timeout and force message through */
+    force?: boolean;
+    [key: string]: any;
+}
+export declare function distinctRelays<T extends Message>(keyFn: (message: T) => string, timeout?: number): OperatorFunction<T, T>;
+export {};

package/dist/operators/distinct-by-relays.js

@@ -0,0 +1,39 @@
+import { filter, map, share } from "rxjs";
+export function distinctRelays(keyFn, timeout = 1000) {
+    return (source$) => {
+        const cache = new Map();
+        return source$.pipe(map((message) => {
+            if (message.force)
+                return message;
+            let key = keyFn(message);
+            let now = Date.now();
+            if (message.relays) {
+                // requesting from specific relays
+                let relays = message.relays;
+                relays = relays.filter((relay) => {
+                    let relayKey = key + " " + relay;
+                    let last = cache.get(relayKey);
+                    if (!last || now >= last + timeout) {
+                        cache.set(relayKey, now);
+                        return true;
+                    }
+                    else
+                        return false;
+                });
+                if (relays.length === 0)
+                    return null;
+                return { ...message, relays };
+            }
+            else {
+                // requesting from default relays
+                let last = cache.get(key);
+                if (!last || now >= last + timeout) {
+                    cache.set(key, now);
+                    return message;
+                }
+                else
+                    return null;
+            }
+        }), filter((message) => message !== null && !!message.relays?.length), share());
+    };
+}

package/dist/operators/distinct-timeout.d.ts

@@ -0,0 +1,3 @@
+import { OperatorFunction } from "rxjs";
+/** Filters out duplicate values based on a key getter and a TTL */
+export declare function distinctTimeout<T>(keyFn: (value: T) => string, ttl?: number): OperatorFunction<T, T>;

package/dist/operators/distinct-timeout.js

@@ -0,0 +1,15 @@
+import { LRU } from "applesauce-core/helpers";
+import { filter } from "rxjs";
+/** Filters out duplicate values based on a key getter and a TTL */
+export function distinctTimeout(keyFn, ttl = 1000) {
+    const seen = new LRU(undefined, ttl);
+    return (source) => source.pipe(filter((value) => {
+        const key = keyFn(value);
+        if (seen.has(key))
+            return false;
+        else {
+            seen.set(key, Date.now());
+            return true;
+        }
+    }));
+}

package/dist/operators/generator-sequence.d.ts

@@ -0,0 +1,3 @@
+import { Observable, OperatorFunction } from "rxjs";
+/** Keeps retrying a value until the generator returns */
+export declare function generatorSequence<T, R>(createGenerator: (value: T) => Generator<Observable<R>, undefined, R[] | undefined>): OperatorFunction<T, R>;

package/dist/operators/generator-sequence.js

@@ -0,0 +1,45 @@
+import { Observable } from "rxjs";
+/** Keeps retrying a value until the generator returns */
+export function generatorSequence(createGenerator) {
+    return (source) => {
+        return new Observable((observer) => {
+            let complete = false;
+            const sub = source.subscribe({
+                next: (value) => {
+                    const generator = createGenerator(value);
+                    const next = (prevResults) => {
+                        const result = generator.next(prevResults);
+                        // generator complete, exit
+                        if (result.done)
+                            return;
+                        const results = [];
+                        result.value.subscribe({
+                            next: (v) => {
+                                // track results and pass along values
+                                results.push(v);
+                                observer.next(v);
+                            },
+                            error: (err) => {
+                                observer.error(err);
+                            },
+                            complete: () => {
+                                // if the upstream observable was complete. exit
+                                if (complete)
+                                    return observer.complete();
+                                // run next step
+                                next(results);
+                            },
+                        });
+                    };
+                    // start running steps
+                    next();
+                },
+                complete: () => {
+                    // source is complete
+                    complete = true;
+                },
+            });
+            return () => sub.unsubscribe();
+        });
+    };
+}

package/dist/operators/group-by-relays.d.ts

@@ -0,0 +1,8 @@
+import { GroupedObservable, OperatorFunction } from "rxjs";
+interface Message {
+    relays?: string[];
+    [key: string]: any;
+}
+/** Splits a stream of messages out into relay specific observables */
+export declare function groupByRelays<T extends Message>(defaultRelay: string): OperatorFunction<T, GroupedObservable<string, T>>;
+export {};

package/dist/operators/group-by-relays.js

@@ -0,0 +1,10 @@
+import { from, groupBy, map, mergeMap, share } from "rxjs";
+/** Splits a stream of messages out into relay specific observables */
+export function groupByRelays(defaultRelay) {
+    return (source$) => {
+        return source$.pipe(mergeMap((message) => {
+            const relays = message.relays?.length ? message.relays : [defaultRelay];
+            return from(relays).pipe(map((relay) => ({ relay, message })));
+        }), groupBy((pair) => pair.relay, { element: (pair) => pair.message }), share());
+    };
+}

package/dist/operators/index.d.ts

@@ -0,0 +1,7 @@
+export * from "./address-pointers-request.js";
+export * from "./distinct-by-relays.js";
+export * from "./distinct-timeout.js";
+export * from "./generator-sequence.js";
+export * from "./group-by-relays.js";
+export * from "./max-filters.js";
+export * from "./relay-request.js";

package/dist/operators/index.js

@@ -0,0 +1,7 @@
+export * from "./address-pointers-request.js";
+export * from "./distinct-by-relays.js";
+export * from "./distinct-timeout.js";
+export * from "./generator-sequence.js";
+export * from "./group-by-relays.js";
+export * from "./max-filters.js";
+export * from "./relay-request.js";

package/dist/operators/max-filters.d.ts

@@ -0,0 +1,4 @@
+import { LazyFilter } from "rx-nostr";
+import { OperatorFunction } from "rxjs";
+/** Splits a batch of filters into multiple requests if necessary */
+export declare function splitMaxFiltersPerRequest(maxFiltersPerRequest: number): OperatorFunction<LazyFilter[], LazyFilter[]>;

package/dist/operators/max-filters.js

@@ -0,0 +1,8 @@
+import { mergeMap } from "rxjs";
+import { reduceToBatches } from "../helpers/array.js";
+/** Splits a batch of filters into multiple requests if necessary */
+export function splitMaxFiltersPerRequest(maxFiltersPerRequest) {
+    return (source) => source.pipe(
+    // batch the filters into sets no larger than maxFiltersPerRequest
+    mergeMap((filters) => reduceToBatches(filters, maxFiltersPerRequest)));
+}

package/dist/operators/relay-request.d.ts

@@ -0,0 +1,4 @@
+import { EventPacket, LazyFilter, RxNostr } from "rx-nostr";
+import { OperatorFunction } from "rxjs";
+/** Makes a request to relays for every set of filters */
+export declare function relaysRequest(rxNostr: RxNostr, id: string, relays?: string[]): OperatorFunction<LazyFilter | LazyFilter[], EventPacket>;

package/dist/operators/relay-request.js

@@ -0,0 +1,9 @@
+import { createRxOneshotReq } from "rx-nostr";
+import { map, mergeAll } from "rxjs";
+/** Makes a request to relays for every set of filters */
+export function relaysRequest(rxNostr, id, relays) {
+    return (source) => source.pipe(map((filters) => {
+        const req = createRxOneshotReq({ filters, rxReqId: id });
+        return rxNostr.use(req, { on: { relays } });
+    }), mergeAll());
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "applesauce-loaders",
-  "version": "0.0.0-next-20250123214405",
+  "version": "0.0.0-next-20250124150519",
   "description": "A collection of observable based loaders built on rx-nostr",
   "type": "module",
   "main": "dist/index.js",
@@ -41,7 +41,7 @@
     }
   },
   "dependencies": {
-    "applesauce-core": "0.0.0-next-20250123214405",
+    "applesauce-core": "0.0.0-next-20250124150519",
     "nanoid": "^5.0.9",
     "nostr-tools": "^2.10.3",
     "rx-nostr": "^3.4.1",