applesauce-loaders 3.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/cache.d.ts +1 -1
- package/dist/helpers/cache.js +12 -6
- package/dist/loaders/address-loader.d.ts +3 -3
- package/dist/loaders/address-loader.js +4 -4
- package/dist/loaders/event-loader.d.ts +3 -3
- package/dist/loaders/event-loader.js +4 -4
- package/dist/loaders/social-graph.d.ts +2 -2
- package/dist/loaders/tag-value-loader.d.ts +3 -3
- package/dist/loaders/tag-value-loader.js +4 -4
- package/dist/loaders/timeline-loader.d.ts +61 -12
- package/dist/loaders/timeline-loader.js +267 -38
- package/dist/loaders/user-lists-loader.d.ts +2 -2
- package/package.json +8 -8
package/dist/helpers/cache.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Observable } from "rxjs";
|
|
2
1
|
import { Filter, NostrEvent } from "nostr-tools";
|
|
2
|
+
import { Observable } from "rxjs";
|
|
3
3
|
import { CacheRequest } from "../types.js";
|
|
4
4
|
/** Calls the cache request and converts the reponse into an observable */
|
|
5
5
|
export declare function unwrapCacheRequest(request: CacheRequest, filters: Filter[]): Observable<NostrEvent>;
|
package/dist/helpers/cache.js
CHANGED
|
@@ -1,18 +1,24 @@
|
|
|
1
|
-
import { from, isObservable, of, switchMap, tap } from "rxjs";
|
|
2
1
|
import { markFromCache } from "applesauce-core/helpers";
|
|
2
|
+
import { from, isObservable, of, switchMap, tap } from "rxjs";
|
|
3
3
|
/** Calls the cache request and converts the reponse into an observable */
|
|
4
4
|
export function unwrapCacheRequest(request, filters) {
|
|
5
5
|
const result = request(filters);
|
|
6
6
|
if (isObservable(result))
|
|
7
7
|
return result;
|
|
8
|
-
else if (result instanceof Promise)
|
|
9
|
-
return from(result).pipe(switchMap((v) => (Array.isArray(v) ? from(v) : of(v))));
|
|
10
|
-
|
|
8
|
+
else if (result instanceof Promise) {
|
|
9
|
+
return from(result).pipe(switchMap((v) => (Array.isArray(v) ? from(v) : of(v))), tap((e) => markFromCache(e)));
|
|
10
|
+
}
|
|
11
|
+
else if (Array.isArray(result)) {
|
|
12
|
+
for (const event of result)
|
|
13
|
+
markFromCache(event);
|
|
11
14
|
return from(result);
|
|
12
|
-
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
markFromCache(result);
|
|
13
18
|
return of(result);
|
|
19
|
+
}
|
|
14
20
|
}
|
|
15
21
|
/** Calls a cache request method with filters and marks all returned events as being from the cache */
|
|
16
22
|
export function makeCacheRequest(request, filters) {
|
|
17
|
-
return unwrapCacheRequest(request, filters)
|
|
23
|
+
return unwrapCacheRequest(request, filters);
|
|
18
24
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { filterDuplicateEvents } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
3
|
import { Observable } from "rxjs";
|
|
4
4
|
import { CacheRequest, NostrRequest, UpstreamPool } from "../types.js";
|
|
@@ -33,8 +33,8 @@ export type AddressLoaderOptions = Partial<{
|
|
|
33
33
|
bufferTime: number;
|
|
34
34
|
/** Max buffer size ( default 200 ) */
|
|
35
35
|
bufferSize: number;
|
|
36
|
-
/** An event store used to deduplicate events */
|
|
37
|
-
eventStore
|
|
36
|
+
/** An event store used to deduplicate events. Set to null to disable deduplication */
|
|
37
|
+
eventStore?: Parameters<typeof filterDuplicateEvents>[0] | null;
|
|
38
38
|
/** A method used to load events from a local cache */
|
|
39
39
|
cacheRequest: CacheRequest;
|
|
40
40
|
/** Whether to follow relay hints ( default true ) */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EventMemory, filterDuplicateEvents } from "applesauce-core";
|
|
2
2
|
import { createReplaceableAddress, getReplaceableAddress, getReplaceableIdentifier, isReplaceable, mergeRelaySets, } from "applesauce-core/helpers";
|
|
3
|
-
import { bufferTime, catchError, EMPTY } from "rxjs";
|
|
3
|
+
import { bufferTime, catchError, EMPTY, identity } from "rxjs";
|
|
4
4
|
import { createFiltersFromAddressPointers, isLoadableAddressPointer } from "../helpers/address-pointer.js";
|
|
5
5
|
import { makeCacheRequest } from "../helpers/cache.js";
|
|
6
6
|
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
@@ -116,6 +116,6 @@ export function createAddressLoader(pool, opts) {
|
|
|
116
116
|
(pointer, event) => event.kind === pointer.kind &&
|
|
117
117
|
event.pubkey === pointer.pubkey &&
|
|
118
118
|
(pointer.identifier ? getReplaceableIdentifier(event) === pointer.identifier : true),
|
|
119
|
-
// Pass all events through the store if
|
|
120
|
-
opts?.eventStore
|
|
119
|
+
// Pass all events through the store if provided, or use EventMemory for deduplication by default
|
|
120
|
+
opts?.eventStore === null ? identity : filterDuplicateEvents(opts?.eventStore || new EventMemory()));
|
|
121
121
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { filterDuplicateEvents } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
3
|
import { EventPointer } from "nostr-tools/nip19";
|
|
4
4
|
import { Observable } from "rxjs";
|
|
@@ -23,8 +23,8 @@ export type EventPointerLoaderOptions = Partial<{
|
|
|
23
23
|
bufferTime: number;
|
|
24
24
|
/** Max buffer size ( default 200 ) */
|
|
25
25
|
bufferSize: number;
|
|
26
|
-
/** An event store used to deduplicate events */
|
|
27
|
-
eventStore
|
|
26
|
+
/** An event store used to deduplicate events. Set to null to disable deduplication */
|
|
27
|
+
eventStore?: Parameters<typeof filterDuplicateEvents>[0] | null;
|
|
28
28
|
/** A method used to load events from a local cache */
|
|
29
29
|
cacheRequest: CacheRequest;
|
|
30
30
|
/** Whether to follow relay hints ( default true ) */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { bufferTime, catchError, EMPTY, merge, tap } from "rxjs";
|
|
1
|
+
import { EventMemory, filterDuplicateEvents } from "applesauce-core";
|
|
2
|
+
import { bufferTime, catchError, EMPTY, identity, merge, tap } from "rxjs";
|
|
3
3
|
import { makeCacheRequest } from "../helpers/cache.js";
|
|
4
4
|
import { consolidateEventPointers } from "../helpers/event-pointer.js";
|
|
5
5
|
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
@@ -85,6 +85,6 @@ export function createEventLoader(pool, opts) {
|
|
|
85
85
|
opts?.extraRelays ? relaysEventsLoader(request, opts.extraRelays) : undefined),
|
|
86
86
|
// Filter resutls based on requests
|
|
87
87
|
(pointer, event) => event.id === pointer.id,
|
|
88
|
-
// Pass all events through the store if
|
|
89
|
-
opts?.eventStore
|
|
88
|
+
// Pass all events through the store if provided, or use EventMemory for deduplication by default
|
|
89
|
+
opts?.eventStore === null ? identity : filterDuplicateEvents(opts?.eventStore || new EventMemory()));
|
|
90
90
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
3
|
import { ProfilePointer } from "nostr-tools/nip19";
|
|
4
4
|
import { Observable } from "rxjs";
|
|
@@ -9,7 +9,7 @@ export type SocialGraphLoader = (user: ProfilePointer & {
|
|
|
9
9
|
}) => Observable<NostrEvent>;
|
|
10
10
|
export type SocialGraphLoaderOptions = Partial<{
|
|
11
11
|
/** An event store to send all the events to */
|
|
12
|
-
eventStore
|
|
12
|
+
eventStore?: Parameters<typeof mapEventsToStore>[0];
|
|
13
13
|
/** The number of parallel requests to make (default 300) */
|
|
14
14
|
parallel: number;
|
|
15
15
|
/** Extra relays to load from */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { filterDuplicateEvents } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
3
|
import { Observable } from "rxjs";
|
|
4
4
|
import { CacheRequest, NostrRequest, UpstreamPool } from "../types.js";
|
|
@@ -25,8 +25,8 @@ export type TagValueLoaderOptions = {
|
|
|
25
25
|
cacheRequest?: CacheRequest;
|
|
26
26
|
/** An array of relays to always fetch from */
|
|
27
27
|
extraRelays?: string[] | Observable<string[]>;
|
|
28
|
-
/** An event store used to deduplicate events */
|
|
29
|
-
eventStore?:
|
|
28
|
+
/** An event store used to deduplicate events. Set to null to disable deduplication */
|
|
29
|
+
eventStore?: Parameters<typeof filterDuplicateEvents>[0] | null;
|
|
30
30
|
};
|
|
31
31
|
export type TagValueLoader = (pointer: TagValuePointer) => Observable<NostrEvent>;
|
|
32
32
|
/** Creates a loader that gets tag values from the cache */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { EventMemory, filterDuplicateEvents } from "applesauce-core";
|
|
2
2
|
import { mergeRelaySets } from "applesauce-core/helpers";
|
|
3
|
-
import { bufferTime, EMPTY, merge } from "rxjs";
|
|
3
|
+
import { bufferTime, EMPTY, identity, merge } from "rxjs";
|
|
4
4
|
import { unique } from "../helpers/array.js";
|
|
5
5
|
import { makeCacheRequest } from "../helpers/cache.js";
|
|
6
6
|
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
@@ -70,6 +70,6 @@ export function createTagValueLoader(pool, tagName, opts) {
|
|
|
70
70
|
},
|
|
71
71
|
// Filter results based on requests
|
|
72
72
|
(pointer, event) => event.tags.some((tag) => tag[0] === tagName && tag[1] === pointer.value),
|
|
73
|
-
// Pass all events through the store if
|
|
74
|
-
opts?.eventStore
|
|
73
|
+
// Pass all events through the store if provided, or use EventMemory for deduplication by default
|
|
74
|
+
opts?.eventStore === null ? identity : filterDuplicateEvents(opts?.eventStore || new EventMemory()));
|
|
75
75
|
}
|
|
@@ -1,24 +1,73 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { FilterMap, OutboxMap, ProfilePointer } from "applesauce-core/helpers";
|
|
3
|
+
import { Filter, NostrEvent } from "nostr-tools";
|
|
4
|
+
import { Observable, OperatorFunction } from "rxjs";
|
|
5
|
+
import { CacheRequest, TimelessFilter, UpstreamPool } from "../types.js";
|
|
5
6
|
/** A loader that optionally takes a timestamp to load till and returns a stream of events */
|
|
6
|
-
export type TimelineLoader = (since?: number) => Observable<NostrEvent>;
|
|
7
|
+
export type TimelineLoader = (since?: number | TimelineWindow) => Observable<NostrEvent>;
|
|
8
|
+
/**
|
|
9
|
+
* The current `since` value of the timeline loader
|
|
10
|
+
* `undefined` is used to initialize the loader to "now"
|
|
11
|
+
* -Infinity (for since) and Infinity (for until) should be used to force loading the next block of events
|
|
12
|
+
*/
|
|
13
|
+
export type TimelineWindow = {
|
|
14
|
+
since?: number;
|
|
15
|
+
until?: number;
|
|
16
|
+
};
|
|
7
17
|
/** Common options for timeline loaders */
|
|
8
18
|
export type CommonTimelineLoaderOptions = Partial<{
|
|
9
19
|
limit: number;
|
|
20
|
+
/** Logger to extend */
|
|
21
|
+
logger?: debug.Debugger;
|
|
10
22
|
}>;
|
|
23
|
+
/**
|
|
24
|
+
* Watches for changes in the window and loads blocks of events going backward
|
|
25
|
+
* NOTE: this operator is stateful
|
|
26
|
+
*/
|
|
27
|
+
export declare function loadBackwardBlocks(request: (until?: number) => Observable<NostrEvent>, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
28
|
+
/**
|
|
29
|
+
* Watches for changes in the window and loads blocks of events going forward
|
|
30
|
+
* NOTE: this operator is stateful
|
|
31
|
+
*/
|
|
32
|
+
export declare function loadForwardBlocks(request: (since?: number) => Observable<NostrEvent>, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
11
33
|
/** A loader that loads blocks of events until none are returned or the since timestamp is reached */
|
|
12
|
-
export declare function
|
|
13
|
-
/**
|
|
14
|
-
export declare function
|
|
15
|
-
/**
|
|
16
|
-
export declare function
|
|
34
|
+
export declare function loadBlocksForTimelineWindow(request: (base: Filter) => Observable<NostrEvent>, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
35
|
+
/** Loads timeline blocs from cache using a cache request */
|
|
36
|
+
export declare function loadBlocksFromCache(request: CacheRequest, filters: TimelessFilter[] | TimelessFilter, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
37
|
+
/** Loads timeline blocs from relays using a pool or request method */
|
|
38
|
+
export declare function loadBlocksFromRelays(pool: UpstreamPool, relays: string[], filters: TimelessFilter[] | TimelessFilter, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
39
|
+
/** Loads timeline blocs from relay set using a pool or request method */
|
|
40
|
+
export declare function loadBlocksFromRelay(pool: UpstreamPool, relay: string, filters: TimelessFilter[] | TimelessFilter, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
41
|
+
/** Loads timeline blocks from a map of relays and filters */
|
|
42
|
+
export declare function loadBlocksFromFilterMap(pool: UpstreamPool, relayMap: FilterMap | Observable<FilterMap>, opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
43
|
+
/** Loads timeline blocks from an {@link OutboxMap} and {@link Filter} or a function that projects users to a filter */
|
|
44
|
+
export declare function loadBlocksFromOutboxMap(pool: UpstreamPool, outboxes: OutboxMap | Observable<OutboxMap>, filter: TimelessFilter | ((users: ProfilePointer[]) => TimelessFilter | TimelessFilter[]), opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
45
|
+
/** Loads timeline blocks from a {@link CacheRequest} using a {@link OutboxMap} and filters */
|
|
46
|
+
export declare function loadBlocksFromOutboxMapCache(cache: CacheRequest, outboxes: OutboxMap | Observable<OutboxMap>, filter: TimelessFilter | ((users: ProfilePointer[]) => TimelessFilter | TimelessFilter[]), opts?: CommonTimelineLoaderOptions): OperatorFunction<TimelineWindow, NostrEvent>;
|
|
47
|
+
/** Options for the create timeline loader methods */
|
|
17
48
|
export type TimelineLoaderOptions = Partial<{
|
|
18
49
|
/** A method used to load the timeline from the cache */
|
|
19
50
|
cache: CacheRequest;
|
|
20
51
|
/** An event store to pass all the events to */
|
|
21
|
-
eventStore
|
|
52
|
+
eventStore?: Parameters<typeof mapEventsToStore>[0];
|
|
22
53
|
}> & CommonTimelineLoaderOptions;
|
|
23
|
-
/**
|
|
54
|
+
/** @deprecated Use the {@link loadBlocksFromCache} operator instead */
|
|
55
|
+
export declare function cacheTimelineLoader(request: CacheRequest, filters: TimelessFilter[] | TimelessFilter, opts?: CommonTimelineLoaderOptions): TimelineLoader;
|
|
56
|
+
/** @deprecated Use the {@link createTimelineLoader} operator instead */
|
|
57
|
+
export declare function relaysTimelineLoader(pool: UpstreamPool, relays: string[], filters: TimelessFilter[] | TimelessFilter, opts?: CommonTimelineLoaderOptions): TimelineLoader;
|
|
58
|
+
/**
|
|
59
|
+
* Creates a {@link TimelineLoader} that loads events from multiple relays and a cache
|
|
60
|
+
* @param pool - The upstream pool to use
|
|
61
|
+
* @param relays - The relays to load from
|
|
62
|
+
* @param filters - The filters to use
|
|
63
|
+
* @param opts - The options for the timeline loader
|
|
64
|
+
*/
|
|
24
65
|
export declare function createTimelineLoader(pool: UpstreamPool, relays: string[], filters: TimelessFilter[] | TimelessFilter, opts?: TimelineLoaderOptions): TimelineLoader;
|
|
66
|
+
/**
|
|
67
|
+
* Creates a {@link TimelineLoader} that loads events for a {@link OutboxMap} or an observable of {@link OutboxMap}
|
|
68
|
+
* @param pool - The upstream pool to use
|
|
69
|
+
* @param outboxMap - An {@link OutboxMap} or an observable of {@link OutboxMap}
|
|
70
|
+
* @param filter - A function to create filters for a set of users
|
|
71
|
+
* @param opts - The options for the timeline loader
|
|
72
|
+
*/
|
|
73
|
+
export declare function createOutboxTimelineLoader(pool: UpstreamPool, outboxes: OutboxMap | Observable<OutboxMap>, filter: TimelessFilter | ((users: ProfilePointer[]) => TimelessFilter | TimelessFilter[]), opts?: TimelineLoaderOptions): TimelineLoader;
|
|
@@ -1,50 +1,279 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { logger as baseLogger, EventMemory, filterDuplicateEvents } from "applesauce-core";
|
|
2
|
+
import { createFilterMap, isFilterEqual, mergeFilters, } from "applesauce-core/helpers";
|
|
3
|
+
import { nanoid } from "nanoid";
|
|
4
|
+
import { BehaviorSubject, distinctUntilChanged, EMPTY, filter, finalize, identity, isObservable, map, merge, mergeMap, Observable, of, share, switchMap, tap, } from "rxjs";
|
|
3
5
|
import { makeCacheRequest } from "../helpers/cache.js";
|
|
4
6
|
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Watches for changes in the window and loads blocks of events going backward
|
|
9
|
+
* NOTE: this operator is stateful
|
|
10
|
+
*/
|
|
11
|
+
export function loadBackwardBlocks(request, opts) {
|
|
12
|
+
return (source) => {
|
|
13
|
+
const log = opts?.logger?.extend("backward").extend(nanoid(8));
|
|
14
|
+
let cursor = undefined;
|
|
15
|
+
let loading = false;
|
|
16
|
+
let complete = false;
|
|
17
|
+
return source.pipe(filter(({ since, until }) => {
|
|
18
|
+
// Once complete, prevent further requests
|
|
19
|
+
if (complete)
|
|
20
|
+
return false;
|
|
21
|
+
// If max is unset, initialize to until
|
|
22
|
+
if (cursor === undefined && until !== undefined && Number.isFinite(until))
|
|
23
|
+
cursor = until;
|
|
24
|
+
// Skip loads if since is still in range
|
|
25
|
+
if (
|
|
26
|
+
// Ignore undefined since values
|
|
27
|
+
since === undefined ||
|
|
28
|
+
// If number is finite and in the loaded range, skip
|
|
29
|
+
(Number.isFinite(since) && cursor !== undefined && since <= cursor))
|
|
30
|
+
return false;
|
|
31
|
+
// Don't load blocks in parallel
|
|
32
|
+
if (loading)
|
|
33
|
+
return false;
|
|
34
|
+
return true;
|
|
35
|
+
}),
|
|
36
|
+
// NOTE: use mergeMap here to ensure old requests continue to load
|
|
37
|
+
mergeMap(() => {
|
|
38
|
+
// Set loading lock
|
|
39
|
+
loading = true;
|
|
40
|
+
// Count returned events so complete set
|
|
41
|
+
let count = 0;
|
|
42
|
+
log?.(`Loading block since:${cursor}`);
|
|
43
|
+
// Request the next block of events
|
|
44
|
+
return request(cursor).pipe(tap((event) => {
|
|
45
|
+
count++;
|
|
46
|
+
// Track the minimum created_at seen from the events
|
|
47
|
+
cursor = Math.min(event.created_at, cursor ?? Infinity);
|
|
48
|
+
}), finalize(() => {
|
|
49
|
+
loading = false;
|
|
50
|
+
complete = count === 0;
|
|
51
|
+
log?.(`Found ${count} events`);
|
|
52
|
+
if (complete)
|
|
53
|
+
log?.("Complete");
|
|
54
|
+
}));
|
|
19
55
|
}));
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Watches for changes in the window and loads blocks of events going forward
|
|
60
|
+
* NOTE: this operator is stateful
|
|
61
|
+
*/
|
|
62
|
+
export function loadForwardBlocks(request, opts) {
|
|
63
|
+
return (source) => {
|
|
64
|
+
const log = opts?.logger?.extend("forward").extend(nanoid(8));
|
|
65
|
+
let cursor = undefined;
|
|
66
|
+
let loading = false;
|
|
67
|
+
let complete = false;
|
|
68
|
+
return source.pipe(filter(({ since, until }) => {
|
|
69
|
+
// Once complete, prevent further requests
|
|
70
|
+
if (complete)
|
|
71
|
+
return false;
|
|
72
|
+
// If min is unset, initialize to since
|
|
73
|
+
if (cursor === undefined && since !== undefined && Number.isFinite(since))
|
|
74
|
+
cursor = since;
|
|
75
|
+
// Skip loads if until is still in range
|
|
76
|
+
if (
|
|
77
|
+
// Ignore undefined until values
|
|
78
|
+
until === undefined ||
|
|
79
|
+
// If number is finite and in the loaded range, skip
|
|
80
|
+
(Number.isFinite(until) && cursor !== undefined && until <= cursor))
|
|
81
|
+
return false;
|
|
82
|
+
// Don't load blocks in parallel
|
|
83
|
+
if (loading)
|
|
84
|
+
return false;
|
|
85
|
+
return true;
|
|
86
|
+
}),
|
|
87
|
+
// NOTE: use mergeMap here to ensure old requests continue to load
|
|
88
|
+
mergeMap(() => {
|
|
89
|
+
// Set loading lock
|
|
90
|
+
loading = true;
|
|
91
|
+
// Count returned events so complete set
|
|
92
|
+
let count = 0;
|
|
93
|
+
log?.(`Loading block until:${cursor}`);
|
|
94
|
+
// Request the next block of events
|
|
95
|
+
return request(cursor).pipe(tap((event) => {
|
|
96
|
+
count++;
|
|
97
|
+
// Track the maximum created_at seen from the events
|
|
98
|
+
cursor = Math.max(event.created_at, cursor ?? -Infinity);
|
|
99
|
+
}), finalize(() => {
|
|
100
|
+
loading = false;
|
|
101
|
+
complete = count === 0;
|
|
102
|
+
log?.(`Found ${count} events`);
|
|
103
|
+
if (complete)
|
|
104
|
+
log?.("Complete");
|
|
105
|
+
}));
|
|
30
106
|
}));
|
|
31
107
|
};
|
|
32
108
|
}
|
|
33
|
-
/**
|
|
34
|
-
export function
|
|
35
|
-
return
|
|
109
|
+
/** A loader that loads blocks of events until none are returned or the since timestamp is reached */
|
|
110
|
+
export function loadBlocksForTimelineWindow(request, opts) {
|
|
111
|
+
return (source) => merge(source.pipe(loadBackwardBlocks((until) => request({ until }), opts)), source.pipe(loadForwardBlocks((since) => request({ since }), opts)));
|
|
36
112
|
}
|
|
37
|
-
/**
|
|
38
|
-
export function
|
|
39
|
-
|
|
40
|
-
|
|
113
|
+
/** Loads timeline blocs from cache using a cache request */
|
|
114
|
+
export function loadBlocksFromCache(request, filters, opts) {
|
|
115
|
+
if (!Array.isArray(filters))
|
|
116
|
+
filters = [filters];
|
|
117
|
+
const logger = opts?.logger?.extend("cache");
|
|
118
|
+
const loader = (base) => makeCacheRequest(request,
|
|
119
|
+
// Create filters from filters, base, and optional limit
|
|
120
|
+
filters.map((f) => (opts?.limit ? mergeFilters(f, base, { limit: opts.limit }) : mergeFilters(f, base))));
|
|
121
|
+
return (source) => source.pipe(loadBlocksForTimelineWindow(loader, { ...opts, logger }));
|
|
41
122
|
}
|
|
42
|
-
/**
|
|
43
|
-
export function
|
|
123
|
+
/** Loads timeline blocs from relays using a pool or request method */
|
|
124
|
+
export function loadBlocksFromRelays(pool, relays, filters, opts) {
|
|
125
|
+
if (!Array.isArray(filters))
|
|
126
|
+
filters = [filters];
|
|
127
|
+
const logger = opts?.logger?.extend("relays");
|
|
128
|
+
const request = wrapUpstreamPool(pool);
|
|
129
|
+
const loader = (base) => request(relays,
|
|
130
|
+
// Create filters from filters, base, and optional limit
|
|
131
|
+
filters.map((f) => (opts?.limit ? mergeFilters(f, base, { limit: opts.limit }) : mergeFilters(f, base))));
|
|
132
|
+
return (source) => source.pipe(loadBlocksForTimelineWindow(loader, { ...opts, logger }));
|
|
133
|
+
}
|
|
134
|
+
/** Loads timeline blocs from relay set using a pool or request method */
|
|
135
|
+
export function loadBlocksFromRelay(pool, relay, filters, opts) {
|
|
44
136
|
if (!Array.isArray(filters))
|
|
45
137
|
filters = [filters];
|
|
138
|
+
const logger = opts?.logger?.extend(relay);
|
|
46
139
|
const request = wrapUpstreamPool(pool);
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
140
|
+
const loader = (base) => request([relay],
|
|
141
|
+
// Create filters from filters, base, and optional limit
|
|
142
|
+
filters.map((f) => (opts?.limit ? mergeFilters(f, base, { limit: opts.limit }) : mergeFilters(f, base))));
|
|
143
|
+
return (source) => source.pipe(loadBlocksForTimelineWindow(loader, { ...opts, logger }));
|
|
144
|
+
}
|
|
145
|
+
/** Loads timeline blocks from a map of relays and filters */
|
|
146
|
+
export function loadBlocksFromFilterMap(pool, relayMap, opts) {
|
|
147
|
+
return (source) => {
|
|
148
|
+
const map$ = isObservable(relayMap) ? relayMap : of(relayMap);
|
|
149
|
+
const cache = new Map();
|
|
150
|
+
const loaders$ = map$.pipe(map((relayMap) => Object.entries(relayMap).map(([relay, filters]) => {
|
|
151
|
+
const existing = cache.get(relay);
|
|
152
|
+
if (existing && isFilterEqual(existing.filters, filters))
|
|
153
|
+
return existing.loader;
|
|
154
|
+
// Create a new loader for this relay + filter set
|
|
155
|
+
const loader = source.pipe(loadBlocksFromRelay(pool, relay, filters, opts));
|
|
156
|
+
cache.set(relay, { filters, loader });
|
|
157
|
+
return loader;
|
|
158
|
+
})));
|
|
159
|
+
return loaders$.pipe(switchMap((loaders) => merge(...loaders)));
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
/** Loads timeline blocks from an {@link OutboxMap} and {@link Filter} or a function that projects users to a filter */
|
|
163
|
+
export function loadBlocksFromOutboxMap(pool, outboxes, filter, opts) {
|
|
164
|
+
const outboxes$ = isObservable(outboxes) ? outboxes : of(outboxes);
|
|
165
|
+
// Project outbox map to filter map
|
|
166
|
+
const filterMap$ = outboxes$.pipe(map((outboxes) => {
|
|
167
|
+
// Filter is a dynamic method that returns a filter
|
|
168
|
+
if (typeof filter === "function")
|
|
169
|
+
return Object.fromEntries(Object.entries(outboxes).map(([relay, users]) => [relay, filter(users)]));
|
|
170
|
+
// Filter is just a filter
|
|
171
|
+
return createFilterMap(outboxes, filter);
|
|
172
|
+
}));
|
|
173
|
+
// Load blocks from the filter map
|
|
174
|
+
return loadBlocksFromFilterMap(pool, filterMap$, opts);
|
|
175
|
+
}
|
|
176
|
+
/** Loads timeline blocks from a {@link CacheRequest} using a {@link OutboxMap} and filters */
|
|
177
|
+
export function loadBlocksFromOutboxMapCache(cache, outboxes, filter, opts) {
|
|
178
|
+
const outboxes$ = isObservable(outboxes) ? outboxes : of(outboxes);
|
|
179
|
+
// Project outboxes to filters
|
|
180
|
+
const filters$ = outboxes$.pipe(map((outboxes) => {
|
|
181
|
+
// Get all pubkeys from all relays
|
|
182
|
+
const pubkeys = new Set();
|
|
183
|
+
for (const users of Object.values(outboxes)) {
|
|
184
|
+
for (const user of users) {
|
|
185
|
+
pubkeys.add(user.pubkey);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
// Create filters for all the pubkeys
|
|
189
|
+
if (typeof filter === "function") {
|
|
190
|
+
const users = Array.from(pubkeys).map((pubkey) => ({ pubkey }));
|
|
191
|
+
return filter(users);
|
|
192
|
+
}
|
|
193
|
+
else {
|
|
194
|
+
return mergeFilters(filter, { authors: Array.from(pubkeys) });
|
|
195
|
+
}
|
|
196
|
+
}),
|
|
197
|
+
// Only create a new loader if the filters change
|
|
198
|
+
distinctUntilChanged((a, b) => isFilterEqual(a, b)));
|
|
199
|
+
// Load blocks from the filter map
|
|
200
|
+
return (source) => filters$.pipe(
|
|
201
|
+
// Every time the filters change, create a new cache loader instance
|
|
202
|
+
switchMap((filters) => source.pipe(loadBlocksFromCache(cache, filters, opts))));
|
|
203
|
+
}
|
|
204
|
+
/** Internal logic for function based timeline loaders */
|
|
205
|
+
function wrapTimelineLoader(window$, loader$, eventStore) {
|
|
206
|
+
const singleton$ = loader$.pipe(
|
|
207
|
+
// Pass all events through the store if provided
|
|
208
|
+
eventStore === null ? identity : filterDuplicateEvents(eventStore || new EventMemory()),
|
|
209
|
+
// Ensure a single subscription to the requests
|
|
210
|
+
share());
|
|
211
|
+
return (since) => {
|
|
212
|
+
// Return the loader so it can be subscribed to
|
|
213
|
+
return new Observable((observer) => {
|
|
214
|
+
const sub = singleton$.subscribe(observer);
|
|
215
|
+
// Once subscribed, update the window
|
|
216
|
+
if (typeof since === "object" && since !== undefined) {
|
|
217
|
+
// Pass new window to the loader
|
|
218
|
+
window$.next(since);
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
// Only update window when this request is subscribed to (prevents window getting lost before subscription)
|
|
222
|
+
window$.next({
|
|
223
|
+
// Default to -Infinity to force loading the next block of events (legacy behavior)
|
|
224
|
+
since: since ?? -Infinity,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
return () => sub.unsubscribe();
|
|
228
|
+
});
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
/** @deprecated Use the {@link loadBlocksFromCache} operator instead */
|
|
232
|
+
export function cacheTimelineLoader(request, filters, opts) {
|
|
233
|
+
const window$ = new BehaviorSubject({});
|
|
234
|
+
return wrapTimelineLoader(window$, window$.pipe(loadBlocksFromCache(request, filters, opts)));
|
|
235
|
+
}
|
|
236
|
+
/** @deprecated Use the {@link createTimelineLoader} operator instead */
|
|
237
|
+
export function relaysTimelineLoader(pool, relays, filters, opts) {
|
|
238
|
+
const window$ = new BehaviorSubject({});
|
|
239
|
+
return wrapTimelineLoader(window$, window$.pipe(loadBlocksFromRelays(pool, relays, filters, opts)));
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Creates a {@link TimelineLoader} that loads events from multiple relays and a cache
|
|
243
|
+
* @param pool - The upstream pool to use
|
|
244
|
+
* @param relays - The relays to load from
|
|
245
|
+
* @param filters - The filters to use
|
|
246
|
+
* @param opts - The options for the timeline loader
|
|
247
|
+
*/
|
|
248
|
+
export function createTimelineLoader(pool, relays, filters, opts) {
|
|
249
|
+
const logger = (opts?.logger ?? baseLogger).extend("timeline").extend(nanoid(4));
|
|
250
|
+
const window$ = new BehaviorSubject({});
|
|
251
|
+
// Create cache and relays loaders
|
|
252
|
+
const cache$ = opts?.cache ? window$.pipe(loadBlocksFromCache(opts.cache, filters, { ...opts, logger })) : EMPTY;
|
|
253
|
+
// Load blocks from relays
|
|
254
|
+
const relays$ = window$.pipe(loadBlocksFromRelays(pool, relays, filters, { ...opts, logger }));
|
|
255
|
+
// Merge the cache and relays loaders
|
|
256
|
+
const loader$ = merge(cache$, relays$);
|
|
257
|
+
return wrapTimelineLoader(window$, loader$, opts?.eventStore);
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Creates a {@link TimelineLoader} that loads events for a {@link OutboxMap} or an observable of {@link OutboxMap}
|
|
261
|
+
* @param pool - The upstream pool to use
|
|
262
|
+
* @param outboxMap - An {@link OutboxMap} or an observable of {@link OutboxMap}
|
|
263
|
+
* @param filter - A function to create filters for a set of users
|
|
264
|
+
* @param opts - The options for the timeline loader
|
|
265
|
+
*/
|
|
266
|
+
export function createOutboxTimelineLoader(pool, outboxes, filter, opts) {
|
|
267
|
+
const logger = (opts?.logger ?? baseLogger).extend("outbox-timeline").extend(nanoid(4));
|
|
268
|
+
const window$ = new BehaviorSubject({});
|
|
269
|
+
// An observable of a cache loader instance for all users
|
|
270
|
+
const cache$ = opts?.cache
|
|
271
|
+
? window$.pipe(loadBlocksFromOutboxMapCache(opts?.cache, outboxes, filter, { ...opts, logger }))
|
|
272
|
+
: EMPTY;
|
|
273
|
+
// Load blocks from relays using outboxes
|
|
274
|
+
const relays$ = window$.pipe(loadBlocksFromOutboxMap(pool, outboxes, filter, { ...opts, logger }));
|
|
275
|
+
// Merge the cache and relays loaders
|
|
276
|
+
const loader$ = merge(cache$, relays$);
|
|
277
|
+
// Wrap the loader in a timeline loader function
|
|
278
|
+
return wrapTimelineLoader(window$, loader$, opts?.eventStore);
|
|
50
279
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
3
|
import { ProfilePointer } from "nostr-tools/nip19";
|
|
4
4
|
import { Observable } from "rxjs";
|
|
@@ -17,7 +17,7 @@ export type UserListsLoaderOptions = Partial<{
|
|
|
17
17
|
/** An array of extra relay to load from */
|
|
18
18
|
extraRelays?: string[] | Observable<string[]>;
|
|
19
19
|
/** An event store used to deduplicate events */
|
|
20
|
-
eventStore
|
|
20
|
+
eventStore?: Parameters<typeof mapEventsToStore>[0];
|
|
21
21
|
}>;
|
|
22
22
|
/**
|
|
23
23
|
* A special address loader that can request addressable events without specifying the identifier
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "applesauce-loaders",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"description": "A collection of observable based loaders built on rx-nostr",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -12,8 +12,7 @@
|
|
|
12
12
|
"author": "hzrd149",
|
|
13
13
|
"license": "MIT",
|
|
14
14
|
"files": [
|
|
15
|
-
"dist"
|
|
16
|
-
"applesauce"
|
|
15
|
+
"dist"
|
|
17
16
|
],
|
|
18
17
|
"exports": {
|
|
19
18
|
".": {
|
|
@@ -53,17 +52,17 @@
|
|
|
53
52
|
}
|
|
54
53
|
},
|
|
55
54
|
"dependencies": {
|
|
56
|
-
"applesauce-core": "^
|
|
55
|
+
"applesauce-core": "^4.2.0",
|
|
57
56
|
"nanoid": "^5.0.9",
|
|
58
|
-
"nostr-tools": "~2.
|
|
57
|
+
"nostr-tools": "~2.17",
|
|
59
58
|
"rxjs": "^7.8.1"
|
|
60
59
|
},
|
|
61
60
|
"devDependencies": {
|
|
62
61
|
"@hirez_io/observer-spy": "^2.2.0",
|
|
63
|
-
"applesauce-signers": "^
|
|
62
|
+
"applesauce-signers": "^4.2.0",
|
|
63
|
+
"rimraf": "^6.0.1",
|
|
64
64
|
"typescript": "^5.8.3",
|
|
65
|
-
"vitest": "^3.2.
|
|
66
|
-
"vitest-nostr": "^0.4.1",
|
|
65
|
+
"vitest": "^3.2.4",
|
|
67
66
|
"vitest-websocket-mock": "^0.5.0"
|
|
68
67
|
},
|
|
69
68
|
"funding": {
|
|
@@ -71,6 +70,7 @@
|
|
|
71
70
|
"url": "lightning:nostrudel@geyser.fund"
|
|
72
71
|
},
|
|
73
72
|
"scripts": {
|
|
73
|
+
"prebuild": "rimraf dist",
|
|
74
74
|
"build": "tsc",
|
|
75
75
|
"watch:build": "tsc --watch > /dev/null",
|
|
76
76
|
"test": "vitest run --passWithNoTests",
|