applesauce-loaders 1.0.0 → 2.0.0

package/dist/loaders/relay-timeline-loader.js

@@ -1,71 +0,0 @@
-import { logger } from "applesauce-core";
-import { unixNow } from "applesauce-core/helpers";
-import { nanoid } from "nanoid";
-import { BehaviorSubject, filter, map, Observable } from "rxjs";
-import { completeOnEOSE } from "../operators/complete-on-eose.js";
-import { Loader } from "./loader.js";
-/** A loader that can be used to load a timeline in chunks */
-export class RelayTimelineLoader extends Loader {
-    relay;
-    filters;
-    id = nanoid(8);
-    loading$ = new BehaviorSubject(false);
-    get loading() {
-        return this.loading$.value;
-    }
-    /** current "until" timestamp */
-    cursor = Infinity;
-    /** if the timeline is complete */
-    complete = false;
-    log = logger.extend("RelayTimelineLoader");
-    constructor(request, relay, filters, opts) {
-        super((source) => new Observable((observer) => {
-            return source
-                .pipe(filter(() => !this.loading && !this.complete), map((limit) => {
-                // build next batch filters
-                return filters.map((filter) => ({
-                    limit: limit || opts?.limit,
-                    ...filter,
-                    // limit curser to now
-                    until: Math.min(unixNow(), this.cursor),
-                }));
-            }), 
-            // ignore empty filters
-            filter((filters) => filters.length > 0))
-                .subscribe((filters) => {
-                // make batch request
-                let count = 0;
-                this.loading$.next(true);
-                this.log(`Next batch starting at ${filters[0].until} limit ${filters[0].limit}`);
-                request([relay], filters)
-                    .pipe(completeOnEOSE())
-                    .subscribe({
-                    next: (event) => {
-                        // update cursor when event is received
-                        this.cursor = Math.min(event.created_at - 1, this.cursor);
-                        count++;
-                        // forward packet
-                        observer.next(event);
-                    },
-                    error: (err) => observer.error(err),
-                    complete: () => {
-                        // set loading to false when batch completes
-                        this.loading$.next(false);
-                        // set complete the observable if 0 events where returned
-                        if (count === 0) {
-                            observer.complete();
-                            this.log(`Got ${count} event, Complete`);
-                        }
-                        else {
-                            this.log(`Finished batch, got ${count} events`);
-                        }
-                    },
-                });
-            });
-        }));
-        this.relay = relay;
-        this.filters = filters;
-        // create a unique logger for this instance
-        this.log = this.log.extend(relay);
-    }
-}

package/dist/loaders/replaceable-loader.d.ts

@@ -1,27 +0,0 @@
-import { logger } from "applesauce-core";
-import { NostrEvent } from "nostr-tools";
-import { LoadableAddressPointer } from "../helpers/address-pointer.js";
-import { CacheRequest, Loader, NostrRequest } from "./loader.js";
-export type ReplaceableLoaderOptions = {
-    /**
-     * Time interval to buffer requests in ms
-     * @default 1000
-     */
-    bufferTime?: number;
-    /** A method used to load events from a local cache */
-    cacheRequest?: CacheRequest;
-    /** Fallback lookup relays to check when event cant be found */
-    lookupRelays?: string[];
-    /** An array of relays to always fetch from */
-    extraRelays?: string[];
-};
-export declare class ReplaceableLoader extends Loader<LoadableAddressPointer, NostrEvent> {
-    log: typeof logger;
-    /** A method used to load events from a local cache */
-    cacheRequest?: CacheRequest;
-    /** Fallback lookup relays to check when event cant be found */
-    lookupRelays?: string[];
-    /** An array of relays to always fetch from */
-    extraRelays?: string[];
-    constructor(request: NostrRequest, opts?: ReplaceableLoaderOptions);
-}

package/dist/loaders/replaceable-loader.js

@@ -1,104 +0,0 @@
-import { logger } from "applesauce-core";
-import { getEventUID, markFromCache, mergeRelaySets } from "applesauce-core/helpers";
-import { nanoid } from "nanoid";
-import { bufferTime, filter, map, tap } from "rxjs";
-import { consolidateAddressPointers, createFiltersFromAddressPointers, getAddressPointerId, getRelaysFromPointers, isLoadableAddressPointer, } from "../helpers/address-pointer.js";
-import { completeOnEOSE } from "../operators/complete-on-eose.js";
-import { distinctRelaysBatch } from "../operators/distinct-relays.js";
-import { generatorSequence } from "../operators/generator-sequence.js";
-import { Loader } from "./loader.js";
-/** A generator that tries to load the address pointers from the cache first, then tries the relays */
-function* cacheFirstSequence(request, pointers, log, opts) {
-    const id = nanoid(4);
-    let remaining = Array.from(pointers);
-    const pointerRelays = Array.from(getRelaysFromPointers(pointers));
-    // handle previous step results and decide if to exit
-    const handleResults = (results) => {
-        if (results.length) {
-            const coordinates = new Set(results.map((event) => getEventUID(event)));
-            // if there where results, filter out any pointers that where found
-            remaining = remaining.filter((pointer) => {
-                const found = coordinates.has(getAddressPointerId(pointer));
-                if (found && pointer.force !== true)
-                    return false;
-                else
-                    return true;
-            });
-            // If there are none left, complete
-            if (remaining.length === 0) {
-                log(`[${id}] Complete`);
-                return true;
-            }
-        }
-        return false;
-    };
-    // first attempt, load from cache relays
-    if (opts?.cacheRequest) {
-        log(`[${id}] Checking cache`, remaining);
-        const filters = createFiltersFromAddressPointers(remaining);
-        const results = yield opts.cacheRequest(filters).pipe(
-        // mark the event as from the cache
-        tap((event) => markFromCache(event)));
-        if (handleResults(results))
-            return;
-    }
-    // load from pointer relays and extra relays
-    const mergedRelays = mergeRelaySets(pointerRelays, opts?.extraRelays);
-    if (mergedRelays.length > 0) {
-        log(`[${id}] Requesting`, mergedRelays, remaining);
-        const filters = createFiltersFromAddressPointers(remaining);
-        const results = yield request(mergedRelays, filters).pipe(completeOnEOSE());
-        if (handleResults(results))
-            return;
-    }
-    // finally load from lookup relays
-    if (opts?.lookupRelays) {
-        // make sure we aren't asking a relay twice
-        const relays = opts.lookupRelays.filter((r) => !pointerRelays.includes(r));
-        if (relays.length > 0) {
-            log(`[${id}] Request from lookup`, relays, remaining);
-            const filters = createFiltersFromAddressPointers(remaining);
-            const results = yield request(relays, filters).pipe(completeOnEOSE());
-            if (handleResults(results))
-                return;
-        }
-    }
-}
-export class ReplaceableLoader extends Loader {
-    log = logger.extend("ReplaceableLoader");
-    /** A method used to load events from a local cache */
-    cacheRequest;
-    /** Fallback lookup relays to check when event cant be found */
-    lookupRelays;
-    /** An array of relays to always fetch from */
-    extraRelays;
-    constructor(request, opts) {
-        super((source) => {
-            return source.pipe(
-            // filter out invalid pointers
-            filter(isLoadableAddressPointer), 
-            // buffer on time
-            bufferTime(opts?.bufferTime ?? 1000), 
-            // ignore empty buffers
-            filter((buffer) => buffer.length > 0), 
-            // only fetch from each relay once
-            distinctRelaysBatch(getAddressPointerId), 
-            // consolidate buffered pointers
-            map(consolidateAddressPointers), 
-            // ignore empty buffer
-            filter((buffer) => buffer.length > 0), 
-            // check cache, relays, lookup relays in that order
-            generatorSequence((pointers) => cacheFirstSequence(request, pointers, this.log, {
-                cacheRequest: this.cacheRequest,
-                lookupRelays: this.lookupRelays,
-                extraRelays: this.extraRelays,
-            }), 
-            // there will always be more events, never complete
-            false));
-        });
-        // set options
-        this.cacheRequest = opts?.cacheRequest;
-        this.lookupRelays = opts?.lookupRelays;
-        this.extraRelays = opts?.extraRelays;
-    }
-}

package/dist/loaders/single-event-loader.d.ts

@@ -1,32 +0,0 @@
-import { logger } from "applesauce-core";
-import { NostrEvent } from "nostr-tools";
-import { CacheRequest, Loader, NostrRequest } from "./loader.js";
-export type LoadableEventPointer = {
-    id: string;
-    /** Relays to load from */
-    relays?: string[];
-};
-export type SingleEventLoaderOptions = {
-    /**
-     * Time interval to buffer requests in ms
-     * @default 1000
-     */
-    bufferTime?: number;
-    /**
-     * How long the loader should wait before it allows an event pointer to be refreshed from a relay
-     * @default 60000
-     */
-    refreshTimeout?: number;
-    /** A method used to load events from a local cache */
-    cacheRequest?: CacheRequest;
-    /** An array of relays to always fetch from */
-    extraRelays?: string[];
-};
-export declare class SingleEventLoader extends Loader<LoadableEventPointer, NostrEvent> {
-    log: typeof logger;
-    /** A method used to load events from a local cache */
-    cacheRequest?: CacheRequest;
-    /** An array of relays to always fetch from */
-    extraRelays?: string[];
-    constructor(request: NostrRequest, opts?: SingleEventLoaderOptions);
-}

package/dist/loaders/single-event-loader.js

@@ -1,74 +0,0 @@
-import { bufferTime, filter, from, map, mergeAll, tap } from "rxjs";
-import { markFromCache } from "applesauce-core/helpers";
-import { logger } from "applesauce-core";
-import { nanoid } from "nanoid";
-import { Loader } from "./loader.js";
-import { generatorSequence } from "../operators/generator-sequence.js";
-import { distinctRelaysBatch } from "../operators/distinct-relays.js";
-import { groupByRelay } from "../helpers/pointer.js";
-import { consolidateEventPointers } from "../helpers/event-pointer.js";
-import { completeOnEOSE } from "../operators/complete-on-eose.js";
-function* cacheFirstSequence(request, pointers, opts, log) {
-    let remaining = [...pointers];
-    const id = nanoid(8);
-    log = log.extend(id);
-    const loaded = (packets) => {
-        const ids = new Set(packets.map((p) => p.id));
-        remaining = remaining.filter((p) => !ids.has(p.id));
-    };
-    if (opts?.cacheRequest) {
-        let filter = { ids: remaining.map((e) => e.id) };
-        const results = yield opts.cacheRequest([filter]).pipe(
-        // mark the event as from the cache
-        tap((event) => markFromCache(event)));
-        if (results.length > 0) {
-            log(`Loaded ${results.length} events from cache`);
-            loaded(results);
-        }
-    }
-    // exit early if all pointers are loaded
-    if (remaining.length === 0)
-        return;
-    let byRelay = groupByRelay(remaining, opts.extraRelays);
-    // load remaining pointers from the relays
-    let results = yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
-        let filter = { ids: pointers.map((e) => e.id) };
-        let count = 0;
-        log(`Requesting from ${relay}`, filter.ids);
-        return request([relay], [filter], id)
-            .pipe(completeOnEOSE())
-            .pipe(tap({
-            next: () => count++,
-            complete: () => log(`Completed ${relay}, loaded ${count} events`),
-        }));
-    })).pipe(mergeAll());
-    loaded(results);
-    if (remaining.length > 0) {
-        // failed to find remaining
-        log("Failed to load", remaining.map((p) => p.id));
-    }
-}
-export class SingleEventLoader extends Loader {
-    log = logger.extend("SingleEventLoader");
-    /** A method used to load events from a local cache */
-    cacheRequest;
-    /** An array of relays to always fetch from */
-    extraRelays;
-    constructor(request, opts) {
-        super((source) => source.pipe(
-        // batch every second
-        bufferTime(opts?.bufferTime ?? 1000), 
-        // ignore empty buffers
-        filter((buffer) => buffer.length > 0), 
-        // only request events from relays once
-        distinctRelaysBatch((p) => p.id, opts?.refreshTimeout ?? 60_000), 
-        // ensure there is only one of each event pointer
-        map(consolidateEventPointers), 
-        // run the loader sequence
-        generatorSequence((pointers) => cacheFirstSequence(request, pointers, { cacheRequest: this.cacheRequest, extraRelays: this.extraRelays }, this.log), 
-        // there will always be more events, never complete
-        false)));
-        this.cacheRequest = opts?.cacheRequest;
-        this.extraRelays = opts?.extraRelays;
-    }
-}

package/dist/loaders/user-sets-loader.d.ts

@@ -1,33 +0,0 @@
-import { logger } from "applesauce-core";
-import { NostrEvent } from "nostr-tools";
-import { CacheRequest, Loader, NostrRequest } from "./loader.js";
-export type LoadableSetPointer = {
-    /** A replaceable kind >= 30000 & < 40000 */
-    kind: number;
-    pubkey: string;
-    /** Relays to load from */
-    relays?: string[];
-    /** Load the sets even if it has already been loaded */
-    force?: boolean;
-};
-export type UserSetsLoaderOptions = {
-    /**
-     * Time interval to buffer requests in ms
-     * @default 1000
-     */
-    bufferTime?: number;
-    /** A method used to load events from a local cache */
-    cacheRequest?: CacheRequest;
-    /**
-     * How long the loader should wait before it allows an event pointer to be refreshed from a relay
-     * @default 120000
-     */
-    refreshTimeout?: number;
-};
-/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
-export declare class UserSetsLoader extends Loader<LoadableSetPointer, NostrEvent> {
-    log: typeof logger;
-    /** An array of relays to always fetch from */
-    extraRelays?: string[];
-    constructor(request: NostrRequest, opts?: UserSetsLoaderOptions);
-}

package/dist/loaders/user-sets-loader.js

@@ -1,59 +0,0 @@
-import { logger } from "applesauce-core";
-import { markFromCache } from "applesauce-core/helpers";
-import { nanoid } from "nanoid";
-import { bufferTime, filter, from, map, mergeAll, tap } from "rxjs";
-import { consolidateAddressPointers, createFiltersFromAddressPointers } from "../helpers/address-pointer.js";
-import { groupByRelay } from "../helpers/pointer.js";
-import { completeOnEOSE } from "../operators/complete-on-eose.js";
-import { distinctRelaysBatch } from "../operators/distinct-relays.js";
-import { generatorSequence } from "../operators/generator-sequence.js";
-import { Loader } from "./loader.js";
-/** A generator that tries to load the address pointers from the cache first, then tries the relays */
-function* cacheFirstSequence(request, pointers, log, opts) {
-    const id = nanoid(8);
-    log = log.extend(id);
-    // first attempt, load from cache relays
-    if (opts?.cacheRequest) {
-        log(`Checking cache`);
-        const filters = createFiltersFromAddressPointers(pointers);
-        const results = yield opts.cacheRequest(filters).pipe(
-        // mark the event as from the cache
-        tap((event) => markFromCache(event)));
-        if (results.length > 0) {
-            log(`Loaded ${results.length} events from cache`);
-        }
-    }
-    let byRelay = groupByRelay(pointers, opts?.extraRelays);
-    // load sets from relays
-    yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
-        let filters = createFiltersFromAddressPointers(pointers);
-        let count = 0;
-        log(`Requesting from ${relay}`, pointers);
-        return request([relay], filters, id).pipe(completeOnEOSE(), tap({
-            next: () => count++,
-            complete: () => log(`Completed ${relay}, loaded ${count} events`),
-        }));
-    })).pipe(mergeAll());
-}
-/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
-export class UserSetsLoader extends Loader {
-    log = logger.extend("UserSetsLoader");
-    /** An array of relays to always fetch from */
-    extraRelays;
-    constructor(request, opts) {
-        let options = opts || {};
-        super((source) => source.pipe(
-        // load first from cache
-        bufferTime(options?.bufferTime ?? 1000), 
-        // ignore empty buffers
-        filter((buffer) => buffer.length > 0), 
-        // only load from each relay once
-        distinctRelaysBatch((p) => p.kind + ":" + p.pubkey, options.refreshTimeout ?? 120_000), 
-        // deduplicate address pointers
-        map(consolidateAddressPointers), 
-        // check cache, relays, lookup relays in that order
-        generatorSequence((pointers) => cacheFirstSequence(request, pointers, this.log, options), 
-        // there will always be more events, never complete
-        false)));
-    }
-}

package/dist/operators/__tests__/distinct-relays.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/operators/__tests__/distinct-relays.test.js

@@ -1,75 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { Subject } from "rxjs";
-import { distinctRelays } from "../distinct-relays.js";
-describe("distinctRelays", () => {
-    it("should filter out messages with same relay within timeout window", () => {
-        const fn = vi.fn();
-        const source$ = new Subject();
-        source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
-        const message = {
-            id: "123",
-            relays: ["relay1", "relay2"],
-        };
-        // Send message with two relays
-        source$.next(message);
-        expect(fn).toHaveBeenCalledTimes(1);
-        expect(fn).toHaveBeenCalledWith(message);
-        // send message again
-        source$.next({ ...message });
-        // should not call again
-        expect(fn).toHaveBeenCalledTimes(1);
-    });
-    it("should only remove duplicate relays in timeout window", () => {
-        const fn = vi.fn();
-        const source$ = new Subject();
-        source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
-        const message = {
-            id: "123",
-            relays: ["relay1", "relay2"],
-        };
-        // Send message with two relays
-        source$.next(message);
-        expect(fn).toHaveBeenCalledTimes(1);
-        expect(fn).toHaveBeenCalledWith(message);
-        // send message again
-        source$.next({ id: "123", relays: ["relay1", "relay3"] });
-        // should not call again
-        expect(fn).toHaveBeenCalledTimes(2);
-        expect(fn).toHaveBeenCalledWith({ id: "123", relays: ["relay3"] });
-    });
-    it("should filter out duplicate messages without relays in timeout", () => {
-        const fn = vi.fn();
-        const source$ = new Subject();
-        source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
-        const message = { id: "123" };
-        // Send message with two relays
-        source$.next(message);
-        expect(fn).toHaveBeenCalledTimes(1);
-        expect(fn).toHaveBeenCalledWith(message);
-        // send message again
-        source$.next({ ...message });
-        // should not call again
-        expect(fn).toHaveBeenCalledTimes(1);
-    });
-    it("should treat messages with relays severalty then messages without", () => {
-        const fn = vi.fn();
-        const source$ = new Subject();
-        source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
-        const withRelays = {
-            id: "123",
-            relays: ["relay1", "relay2"],
-        };
-        const withoutRelays = {
-            id: "123",
-        };
-        // Send message with two relays
-        source$.next(withoutRelays);
-        expect(fn).toHaveBeenCalledTimes(1);
-        expect(fn).toHaveBeenCalledWith(withoutRelays);
-        // send message with relays
-        source$.next(withRelays);
-        // should not call again
-        expect(fn).toHaveBeenCalledTimes(2);
-        expect(fn).toHaveBeenCalledWith(withRelays);
-    });
-});

package/dist/operators/__tests__/generator-sequence.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/operators/__tests__/generator-sequence.test.js

@@ -1,38 +0,0 @@
-import { expect, it } from "vitest";
-import { lastValueFrom, of, toArray } from "rxjs";
-import { TestScheduler } from "rxjs/testing";
-import { generatorSequence } from "../generator-sequence.js";
-let testScheduler = new TestScheduler((actual, expected) => {
-    expect(actual).toEqual(expected);
-});
-it("should work with normal generator functions", () => {
-    testScheduler.run(({ expectObservable }) => {
-        function* normalGenerator(value) {
-            yield of(value + 1);
-            yield of(value + 2);
-            yield of(value + 3);
-            yield value + 4;
-        }
-        const source$ = of(1).pipe(generatorSequence(normalGenerator));
-        // Define expected marble diagram
-        const expectedMarble = "(abcd|)";
-        const expectedValues = {
-            a: 2,
-            b: 3,
-            c: 4,
-            d: 5,
-        };
-        expectObservable(source$).toBe(expectedMarble, expectedValues);
-    });
-});
-it("should work with async generator functions", async () => {
-    async function* asyncGenerator(value) {
-        yield of(`${value}-1`);
-        yield of(`${value}-2`);
-        yield of(`${value}-3`);
-        yield `${value}-4`;
-    }
-    const source$ = of("test").pipe(generatorSequence(asyncGenerator));
-    const expectedValues = ["test-1", "test-2", "test-3", "test-4"];
-    expect(await lastValueFrom(source$.pipe(toArray()))).toEqual(expectedValues);
-});

package/dist/operators/generator-sequence.d.ts

@@ -1,3 +0,0 @@
-import { Observable, OperatorFunction } from "rxjs";
-/** Keeps retrying a value until the generator returns */
-export declare function generatorSequence<Input, Result>(createGenerator: (value: Input) => Generator<Observable<Result> | Result, void, Result[] | undefined> | AsyncGenerator<Observable<Result> | Result, void, Result[] | undefined>, shouldComplete?: boolean): OperatorFunction<Input, Result>;

package/dist/operators/generator-sequence.js

@@ -1,53 +0,0 @@
-import { isObservable, Observable } from "rxjs";
-/** Keeps retrying a value until the generator returns */
-export function generatorSequence(createGenerator, shouldComplete = true) {
-    return (source) => {
-        return new Observable((observer) => {
-            return source.subscribe((value) => {
-                const generator = createGenerator(value);
-                const nextSequence = (prevResults) => {
-                    const p = generator.next(prevResults);
-                    const handleResult = (result) => {
-                        // generator complete, exit
-                        if (result.done) {
-                            if (shouldComplete)
-                                observer.complete();
-                            return;
-                        }
-                        const results = [];
-                        if (isObservable(result.value)) {
-                            result.value.subscribe({
-                                next: (v) => {
-                                    // track results and pass along values
-                                    results.push(v);
-                                    observer.next(v);
-                                },
-                                error: (err) => {
-                                    observer.error(err);
-                                },
-                                complete: () => {
-                                    // run next step
-                                    nextSequence(results);
-                                },
-                            });
-                        }
-                        else {
-                            results.push(result.value);
-                            observer.next(result.value);
-                            nextSequence(results);
-                        }
-                    };
-                    // if its an async generator, wait for the promise
-                    if (p instanceof Promise)
-                        p.then(handleResult, (err) => {
-                            observer.error(err);
-                        });
-                    else
-                        handleResult(p);
-                };
-                // start running steps
-                nextSequence();
-            });
-        });
-    };
-}