applesauce-loaders 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -65
- package/dist/helpers/address-pointer.d.ts +0 -14
- package/dist/helpers/address-pointer.js +3 -51
- package/dist/helpers/cache.d.ts +7 -0
- package/dist/helpers/cache.js +18 -0
- package/dist/helpers/event-pointer.js +10 -10
- package/dist/helpers/index.d.ts +5 -0
- package/dist/helpers/index.js +5 -0
- package/dist/helpers/loaders.d.ts +8 -0
- package/dist/helpers/loaders.js +42 -0
- package/dist/helpers/pointer.d.ts +1 -0
- package/dist/helpers/pointer.js +1 -0
- package/dist/helpers/upstream.d.ts +7 -0
- package/dist/helpers/upstream.js +13 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/loaders/address-loader.d.ts +48 -0
- package/dist/loaders/address-loader.js +121 -0
- package/dist/loaders/event-loader.d.ts +36 -0
- package/dist/loaders/event-loader.js +90 -0
- package/dist/loaders/index.d.ts +8 -8
- package/dist/loaders/index.js +8 -8
- package/dist/loaders/reactions-loader.d.ts +12 -0
- package/dist/loaders/reactions-loader.js +18 -0
- package/dist/loaders/social-graph.d.ts +21 -0
- package/dist/loaders/social-graph.js +50 -0
- package/dist/loaders/tag-value-loader.d.ts +17 -19
- package/dist/loaders/tag-value-loader.js +71 -72
- package/dist/loaders/timeline-loader.d.ts +23 -21
- package/dist/loaders/timeline-loader.js +49 -55
- package/dist/loaders/user-lists-loader.d.ts +26 -0
- package/dist/loaders/user-lists-loader.js +33 -0
- package/dist/loaders/zaps-loader.d.ts +12 -0
- package/dist/loaders/zaps-loader.js +18 -0
- package/dist/operators/complete-on-eose.d.ts +4 -1
- package/dist/operators/complete-on-eose.js +4 -1
- package/dist/operators/generator.d.ts +13 -0
- package/dist/operators/generator.js +72 -0
- package/dist/operators/index.d.ts +1 -1
- package/dist/operators/index.js +1 -1
- package/dist/types.d.ts +14 -0
- package/package.json +6 -4
- package/dist/helpers/__tests__/address-pointer.test.js +0 -19
- package/dist/loaders/__tests__/dns-identity-loader.test.d.ts +0 -1
- package/dist/loaders/__tests__/dns-identity-loader.test.js +0 -59
- package/dist/loaders/__tests__/relay-timeline-loader.test.d.ts +0 -1
- package/dist/loaders/__tests__/relay-timeline-loader.test.js +0 -26
- package/dist/loaders/cache-timeline-loader.d.ts +0 -22
- package/dist/loaders/cache-timeline-loader.js +0 -61
- package/dist/loaders/loader.d.ts +0 -22
- package/dist/loaders/loader.js +0 -22
- package/dist/loaders/relay-timeline-loader.d.ts +0 -23
- package/dist/loaders/relay-timeline-loader.js +0 -71
- package/dist/loaders/replaceable-loader.d.ts +0 -27
- package/dist/loaders/replaceable-loader.js +0 -104
- package/dist/loaders/single-event-loader.d.ts +0 -32
- package/dist/loaders/single-event-loader.js +0 -74
- package/dist/loaders/user-sets-loader.d.ts +0 -33
- package/dist/loaders/user-sets-loader.js +0 -59
- package/dist/operators/__tests__/distinct-relays.test.d.ts +0 -1
- package/dist/operators/__tests__/distinct-relays.test.js +0 -75
- package/dist/operators/__tests__/generator-sequence.test.d.ts +0 -1
- package/dist/operators/__tests__/generator-sequence.test.js +0 -38
- package/dist/operators/generator-sequence.d.ts +0 -3
- package/dist/operators/generator-sequence.js +0 -53
- /package/dist/{helpers/__tests__/address-pointer.test.d.ts → types.js} +0 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { createReplaceableAddress, getReplaceableAddress, getReplaceableIdentifier, isReplaceable, mergeRelaySets, } from "applesauce-core/helpers";
|
|
3
|
+
import { bufferTime, catchError, EMPTY } from "rxjs";
|
|
4
|
+
import { createFiltersFromAddressPointers, isLoadableAddressPointer } from "../helpers/address-pointer.js";
|
|
5
|
+
import { makeCacheRequest } from "../helpers/cache.js";
|
|
6
|
+
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
7
|
+
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
8
|
+
import { wrapGeneratorFunction } from "../operators/generator.js";
|
|
9
|
+
/**
|
|
10
|
+
* Loads address pointers from an async cache
|
|
11
|
+
* @note ignores pointers with force=true
|
|
12
|
+
*/
|
|
13
|
+
export function cacheAddressPointersLoader(request) {
|
|
14
|
+
return (pointers) => {
|
|
15
|
+
pointers = pointers.filter((p) => p.cache !== false);
|
|
16
|
+
// Skip if there are no pointers to load from cache
|
|
17
|
+
if (pointers.length === 0)
|
|
18
|
+
return EMPTY;
|
|
19
|
+
return makeCacheRequest(request, createFiltersFromAddressPointers(pointers));
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/** Loads address pointers from the relay hints */
|
|
23
|
+
export function relayHintsAddressPointersLoader(request) {
|
|
24
|
+
return (pointers) => {
|
|
25
|
+
const relays = mergeRelaySets(...pointers.map((p) => p.relays));
|
|
26
|
+
if (relays.length === 0)
|
|
27
|
+
return EMPTY;
|
|
28
|
+
const filters = createFiltersFromAddressPointers(pointers);
|
|
29
|
+
return request(relays, filters);
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/** Loads address pointers from an array of relays */
|
|
33
|
+
export function relaysAddressPointersLoader(request, relays) {
|
|
34
|
+
return (pointers) => unwrap(relays, (relays) => {
|
|
35
|
+
if (relays.length === 0)
|
|
36
|
+
return EMPTY;
|
|
37
|
+
const filters = createFiltersFromAddressPointers(pointers);
|
|
38
|
+
return request(relays, filters);
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/** Creates a loader that loads all event pointers based on their relays */
|
|
42
|
+
export function addressPointerLoadingSequence(...loaders) {
|
|
43
|
+
return wrapGeneratorFunction(function* (pointers) {
|
|
44
|
+
// Filter out invalid pointers and consolidate
|
|
45
|
+
pointers = consolidateAddressPointers(pointers.filter(isLoadableAddressPointer));
|
|
46
|
+
// Skip if there are no pointers
|
|
47
|
+
if (pointers.length === 0)
|
|
48
|
+
return;
|
|
49
|
+
// Keep track of remaining pointers to load
|
|
50
|
+
let remaining = Array.from(pointers);
|
|
51
|
+
for (const loader of loaders) {
|
|
52
|
+
if (loader === undefined)
|
|
53
|
+
continue;
|
|
54
|
+
const results = yield loader(remaining).pipe(
|
|
55
|
+
// If the loader throws an error, skip it
|
|
56
|
+
catchError(() => EMPTY));
|
|
57
|
+
// Get set of addresses loaded
|
|
58
|
+
const addresses = new Set(results.filter((e) => isReplaceable(e.kind)).map((event) => getReplaceableAddress(event)));
|
|
59
|
+
// Remove the pointers that were loaded
|
|
60
|
+
remaining = remaining.filter((p) => !addresses.has(createReplaceableAddress(p.kind, p.pubkey, p.identifier)));
|
|
61
|
+
// If there are no remaining pointers, complete
|
|
62
|
+
if (remaining.length === 0)
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
/** deep clone a loadable pointer to ensure its safe to modify */
|
|
68
|
+
function cloneLoadablePointer(pointer) {
|
|
69
|
+
const clone = { ...pointer };
|
|
70
|
+
if (pointer.relays)
|
|
71
|
+
clone.relays = [...pointer.relays];
|
|
72
|
+
return clone;
|
|
73
|
+
}
|
|
74
|
+
/** deduplicates an array of address pointers and merges their relays array */
|
|
75
|
+
export function consolidateAddressPointers(pointers) {
|
|
76
|
+
const byAddress = new Map();
|
|
77
|
+
for (const pointer of pointers) {
|
|
78
|
+
const addr = createReplaceableAddress(pointer.kind, pointer.pubkey, pointer.identifier);
|
|
79
|
+
if (byAddress.has(addr)) {
|
|
80
|
+
// duplicate, merge pointers
|
|
81
|
+
const current = byAddress.get(addr);
|
|
82
|
+
// merge relays
|
|
83
|
+
if (pointer.relays && pointer.relays.length > 0) {
|
|
84
|
+
if (current.relays)
|
|
85
|
+
current.relays = mergeRelaySets(current.relays, pointer.relays);
|
|
86
|
+
else
|
|
87
|
+
current.relays = pointer.relays;
|
|
88
|
+
}
|
|
89
|
+
// merge cache flag
|
|
90
|
+
if (pointer.cache === false)
|
|
91
|
+
current.cache = false;
|
|
92
|
+
}
|
|
93
|
+
else
|
|
94
|
+
byAddress.set(addr, cloneLoadablePointer(pointer));
|
|
95
|
+
}
|
|
96
|
+
// return consolidated pointers
|
|
97
|
+
return Array.from(byAddress.values());
|
|
98
|
+
}
|
|
99
|
+
/** Create a pre-built address pointer loader that supports batching, caching, and lookup relays */
|
|
100
|
+
export function createAddressLoader(pool, opts) {
|
|
101
|
+
const request = wrapUpstreamPool(pool);
|
|
102
|
+
return batchLoader(
|
|
103
|
+
// buffer requests by time or size
|
|
104
|
+
bufferTime(opts?.bufferTime ?? 1000, undefined, opts?.bufferSize ?? 200),
|
|
105
|
+
// Create a loader for batching
|
|
106
|
+
addressPointerLoadingSequence(
|
|
107
|
+
// Step 1. load from cache if available
|
|
108
|
+
opts?.cacheRequest ? cacheAddressPointersLoader(opts.cacheRequest) : undefined,
|
|
109
|
+
// Step 2. load from relay hints on pointers
|
|
110
|
+
opts?.followRelayHints !== false ? relayHintsAddressPointersLoader(request) : undefined,
|
|
111
|
+
// Step 3. load from extra relays
|
|
112
|
+
opts?.extraRelays ? relaysAddressPointersLoader(request, opts.extraRelays) : undefined,
|
|
113
|
+
// Step 4. load from lookup relays
|
|
114
|
+
opts?.lookupRelays ? relaysAddressPointersLoader(request, opts.lookupRelays) : undefined),
|
|
115
|
+
// Filter resutls based on requests
|
|
116
|
+
(pointer, event) => event.kind === pointer.kind &&
|
|
117
|
+
event.pubkey === pointer.pubkey &&
|
|
118
|
+
(pointer.identifier ? getReplaceableIdentifier(event) === pointer.identifier : true),
|
|
119
|
+
// Pass all events through the store if defined
|
|
120
|
+
opts?.eventStore && mapEventsToStore(opts.eventStore));
|
|
121
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { IEventStore } from "applesauce-core";
|
|
2
|
+
import { NostrEvent } from "nostr-tools";
|
|
3
|
+
import { EventPointer } from "nostr-tools/nip19";
|
|
4
|
+
import { Observable } from "rxjs";
|
|
5
|
+
import { CacheRequest, NostrRequest, UpstreamPool } from "../types.js";
|
|
6
|
+
export type LoadableEventPointer = EventPointer & {
|
|
7
|
+
cache?: boolean;
|
|
8
|
+
};
|
|
9
|
+
export type EventPointerLoader = (pointer: LoadableEventPointer) => Observable<NostrEvent>;
|
|
10
|
+
export type createEventLoader = (pointers: LoadableEventPointer[]) => Observable<NostrEvent>;
|
|
11
|
+
/** Creates a loader that gets a single event from the cache */
|
|
12
|
+
export declare function cacheEventsLoader(request: CacheRequest): createEventLoader;
|
|
13
|
+
/** Creates a loader that gets an array of events from a list of relays */
|
|
14
|
+
export declare function relaysEventsLoader(request: NostrRequest, relays: string[] | Observable<string[]>): createEventLoader;
|
|
15
|
+
/** Creates a loader that gets an array of events from a single relay */
|
|
16
|
+
export declare function relayEventsLoader(request: NostrRequest, relay: string): createEventLoader;
|
|
17
|
+
/** Creates a loader that creates a new loader for each relay hint and uses them to load events */
|
|
18
|
+
export declare function relayHintsEventsLoader(request: NostrRequest, upstream?: (request: NostrRequest, relay: string) => createEventLoader): createEventLoader;
|
|
19
|
+
/** Creates a loader that tries to load events from a list of loaders in order */
|
|
20
|
+
export declare function eventLoadingSequence(...loaders: (createEventLoader | undefined)[]): createEventLoader;
|
|
21
|
+
export type EventPointerLoaderOptions = Partial<{
|
|
22
|
+
/** Time interval to buffer requests in ms ( default 1000 ) */
|
|
23
|
+
bufferTime: number;
|
|
24
|
+
/** Max buffer size ( default 200 ) */
|
|
25
|
+
bufferSize: number;
|
|
26
|
+
/** An event store used to deduplicate events */
|
|
27
|
+
eventStore: IEventStore;
|
|
28
|
+
/** A method used to load events from a local cache */
|
|
29
|
+
cacheRequest: CacheRequest;
|
|
30
|
+
/** Whether to follow relay hints ( default true ) */
|
|
31
|
+
followRelayHints: boolean;
|
|
32
|
+
/** An array of relays to always fetch from */
|
|
33
|
+
extraRelays: string[] | Observable<string[]>;
|
|
34
|
+
}>;
|
|
35
|
+
/** Create a pre-built address pointer loader that supports batching, caching, and lookup relays */
|
|
36
|
+
export declare function createEventLoader(pool: UpstreamPool, opts?: EventPointerLoaderOptions): EventPointerLoader;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { bufferTime, catchError, EMPTY, merge, tap } from "rxjs";
|
|
3
|
+
import { makeCacheRequest } from "../helpers/cache.js";
|
|
4
|
+
import { consolidateEventPointers } from "../helpers/event-pointer.js";
|
|
5
|
+
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
6
|
+
import { groupByRelay } from "../helpers/pointer.js";
|
|
7
|
+
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
8
|
+
import { wrapGeneratorFunction } from "../operators/generator.js";
|
|
9
|
+
/** Creates a loader that gets a single event from the cache */
|
|
10
|
+
export function cacheEventsLoader(request) {
|
|
11
|
+
return (pointers) => {
|
|
12
|
+
pointers = pointers.filter((p) => p.cache !== false);
|
|
13
|
+
if (pointers.length === 0)
|
|
14
|
+
return EMPTY;
|
|
15
|
+
return makeCacheRequest(request, [{ ids: pointers.map((p) => p.id) }]);
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/** Creates a loader that gets an array of events from a list of relays */
|
|
19
|
+
export function relaysEventsLoader(request, relays) {
|
|
20
|
+
return (pointers) => unwrap(relays, (relays) => request(relays, [{ ids: pointers.map((p) => p.id) }]));
|
|
21
|
+
}
|
|
22
|
+
/** Creates a loader that gets an array of events from a single relay */
|
|
23
|
+
export function relayEventsLoader(request, relay) {
|
|
24
|
+
return relaysEventsLoader(request, [relay]);
|
|
25
|
+
}
|
|
26
|
+
/** Creates a loader that creates a new loader for each relay hint and uses them to load events */
|
|
27
|
+
export function relayHintsEventsLoader(request, upstream = relayEventsLoader) {
|
|
28
|
+
const loaders = new Map();
|
|
29
|
+
// Get or create a new loader for each relay
|
|
30
|
+
const getLoader = (relay) => {
|
|
31
|
+
let loader = loaders.get(relay);
|
|
32
|
+
if (!loader) {
|
|
33
|
+
loader = upstream(request, relay);
|
|
34
|
+
loaders.set(relay, loader);
|
|
35
|
+
}
|
|
36
|
+
return loader;
|
|
37
|
+
};
|
|
38
|
+
return (pointers) => {
|
|
39
|
+
// Group pointers by thier relay hints
|
|
40
|
+
const byRelay = groupByRelay(pointers);
|
|
41
|
+
// Request the pointers from each relay
|
|
42
|
+
return merge(...Array.from(byRelay).map(([relay, pointers]) => getLoader(relay)(pointers)));
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** Creates a loader that tries to load events from a list of loaders in order */
|
|
46
|
+
export function eventLoadingSequence(...loaders) {
|
|
47
|
+
return wrapGeneratorFunction(function* (pointers) {
|
|
48
|
+
// Filter out invalid pointers and consolidate
|
|
49
|
+
pointers = consolidateEventPointers(pointers);
|
|
50
|
+
// Skip if there are no pointers
|
|
51
|
+
if (pointers.length === 0)
|
|
52
|
+
return;
|
|
53
|
+
const found = new Set();
|
|
54
|
+
let remaining = Array.from(pointers);
|
|
55
|
+
for (const loader of loaders) {
|
|
56
|
+
if (loader === undefined)
|
|
57
|
+
continue;
|
|
58
|
+
yield loader(remaining).pipe(
|
|
59
|
+
// If the loader throws an error, skip it
|
|
60
|
+
catchError(() => EMPTY),
|
|
61
|
+
// Record events that where found
|
|
62
|
+
tap((e) => found.add(e.id)));
|
|
63
|
+
// Remove poniters that where found
|
|
64
|
+
remaining = remaining.filter((p) => !found.has(p.id));
|
|
65
|
+
// If there are no remaining pointers, complete
|
|
66
|
+
if (remaining.length === 0)
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
/** Create a pre-built address pointer loader that supports batching, caching, and lookup relays */
|
|
72
|
+
export function createEventLoader(pool, opts) {
|
|
73
|
+
const request = wrapUpstreamPool(pool);
|
|
74
|
+
return batchLoader(
|
|
75
|
+
// Create batching sequence
|
|
76
|
+
// buffer requests by time or size
|
|
77
|
+
bufferTime(opts?.bufferTime ?? 1000, undefined, opts?.bufferSize ?? 200),
|
|
78
|
+
// Create a loader for batching
|
|
79
|
+
eventLoadingSequence(
|
|
80
|
+
// Step 1. load from cache if available
|
|
81
|
+
opts?.cacheRequest ? cacheEventsLoader(opts.cacheRequest) : undefined,
|
|
82
|
+
// Step 2. load from relay hints on pointers
|
|
83
|
+
opts?.followRelayHints !== false ? relayHintsEventsLoader(request) : undefined,
|
|
84
|
+
// Step 3. load from extra relays
|
|
85
|
+
opts?.extraRelays ? relaysEventsLoader(request, opts.extraRelays) : undefined),
|
|
86
|
+
// Filter resutls based on requests
|
|
87
|
+
(pointer, event) => event.id === pointer.id,
|
|
88
|
+
// Pass all events through the store if defined
|
|
89
|
+
opts?.eventStore && mapEventsToStore(opts.eventStore));
|
|
90
|
+
}
|
package/dist/loaders/index.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from "./loader.js";
|
|
2
|
-
export * from "./replaceable-loader.js";
|
|
3
|
-
export * from "./single-event-loader.js";
|
|
4
|
-
export * from "./user-sets-loader.js";
|
|
5
|
-
export * from "./timeline-loader.js";
|
|
6
|
-
export * from "./relay-timeline-loader.js";
|
|
7
|
-
export * from "./cache-timeline-loader.js";
|
|
8
|
-
export * from "./tag-value-loader.js";
|
|
1
|
+
export * from "./address-loader.js";
|
|
9
2
|
export * from "./dns-identity-loader.js";
|
|
3
|
+
export * from "./event-loader.js";
|
|
4
|
+
export * from "./reactions-loader.js";
|
|
5
|
+
export * from "./social-graph.js";
|
|
6
|
+
export * from "./tag-value-loader.js";
|
|
7
|
+
export * from "./timeline-loader.js";
|
|
8
|
+
export * from "./user-lists-loader.js";
|
|
9
|
+
export * from "./zaps-loader.js";
|
package/dist/loaders/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
export * from "./loader.js";
|
|
2
|
-
export * from "./replaceable-loader.js";
|
|
3
|
-
export * from "./single-event-loader.js";
|
|
4
|
-
export * from "./user-sets-loader.js";
|
|
5
|
-
export * from "./timeline-loader.js";
|
|
6
|
-
export * from "./relay-timeline-loader.js";
|
|
7
|
-
export * from "./cache-timeline-loader.js";
|
|
8
|
-
export * from "./tag-value-loader.js";
|
|
1
|
+
export * from "./address-loader.js";
|
|
9
2
|
export * from "./dns-identity-loader.js";
|
|
3
|
+
export * from "./event-loader.js";
|
|
4
|
+
export * from "./reactions-loader.js";
|
|
5
|
+
export * from "./social-graph.js";
|
|
6
|
+
export * from "./tag-value-loader.js";
|
|
7
|
+
export * from "./timeline-loader.js";
|
|
8
|
+
export * from "./user-lists-loader.js";
|
|
9
|
+
export * from "./zaps-loader.js";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { NostrEvent } from "nostr-tools";
|
|
2
|
+
import { Observable } from "rxjs";
|
|
3
|
+
import { UpstreamPool } from "../types.js";
|
|
4
|
+
import { TagValueLoaderOptions } from "./tag-value-loader.js";
|
|
5
|
+
/** A loader that takes an event and returns zaps */
|
|
6
|
+
export type ReactionsLoader = (event: NostrEvent, relays?: string[]) => Observable<NostrEvent>;
|
|
7
|
+
export type ReactionsLoaderOptions = Omit<TagValueLoaderOptions, "kinds"> & {
|
|
8
|
+
/** Whether to request reactions from the relays the event was seen on ( default true ) */
|
|
9
|
+
useSeenRelays?: boolean;
|
|
10
|
+
};
|
|
11
|
+
/** Creates a loader that loads reaction events for a given event */
|
|
12
|
+
export declare function createReactionsLoader(pool: UpstreamPool, opts?: ReactionsLoaderOptions): ReactionsLoader;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { getReplaceableAddress, getSeenRelays, isReplaceable, mergeRelaySets } from "applesauce-core/helpers";
|
|
2
|
+
import { kinds } from "nostr-tools";
|
|
3
|
+
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
4
|
+
import { createTagValueLoader } from "./tag-value-loader.js";
|
|
5
|
+
/** Creates a loader that loads reaction events for a given event */
|
|
6
|
+
export function createReactionsLoader(pool, opts) {
|
|
7
|
+
const request = wrapUpstreamPool(pool);
|
|
8
|
+
const eventLoader = createTagValueLoader(request, "e", { ...opts, kinds: [kinds.Reaction] });
|
|
9
|
+
const addressableLoader = createTagValueLoader(request, "a", { ...opts, kinds: [kinds.Reaction] });
|
|
10
|
+
// Return diffrent loaders depending on if the event is addressable
|
|
11
|
+
return (event, relays) => {
|
|
12
|
+
if (opts?.useSeenRelays ?? true)
|
|
13
|
+
relays = mergeRelaySets(relays, getSeenRelays(event));
|
|
14
|
+
return isReplaceable(event.kind)
|
|
15
|
+
? addressableLoader({ value: getReplaceableAddress(event), relays })
|
|
16
|
+
: eventLoader({ value: event.id, relays });
|
|
17
|
+
};
|
|
18
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { IEventStore } from "applesauce-core";
|
|
2
|
+
import { NostrEvent } from "nostr-tools";
|
|
3
|
+
import { ProfilePointer } from "nostr-tools/nip19";
|
|
4
|
+
import { Observable } from "rxjs";
|
|
5
|
+
import { AddressPointerLoader } from "./address-loader.js";
|
|
6
|
+
/** A loader that loads the social graph of a user out to a set distance */
|
|
7
|
+
export type SocialGraphLoader = (user: ProfilePointer & {
|
|
8
|
+
distance: number;
|
|
9
|
+
}) => Observable<NostrEvent>;
|
|
10
|
+
export type SocialGraphLoaderOptions = Partial<{
|
|
11
|
+
/** An event store to send all the events to */
|
|
12
|
+
eventStore: IEventStore;
|
|
13
|
+
/** The number of parallel requests to make (default 300) */
|
|
14
|
+
parallel: number;
|
|
15
|
+
/** Extra relays to load from */
|
|
16
|
+
extraRelays?: string[] | Observable<string[]>;
|
|
17
|
+
/** Whether to follow relay hints in contact events */
|
|
18
|
+
hints?: boolean;
|
|
19
|
+
}>;
|
|
20
|
+
/** Create a social graph loader */
|
|
21
|
+
export declare function createSocialGraphLoader(addressLoader: AddressPointerLoader, opts?: SocialGraphLoaderOptions): SocialGraphLoader;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { getProfilePointersFromList, mergeRelaySets } from "applesauce-core/helpers";
|
|
3
|
+
import { kinds } from "nostr-tools";
|
|
4
|
+
import { firstValueFrom, identity, isObservable, lastValueFrom, toArray } from "rxjs";
|
|
5
|
+
import { wrapGeneratorFunction } from "../operators/generator.js";
|
|
6
|
+
/** Create a social graph loader */
|
|
7
|
+
export function createSocialGraphLoader(addressLoader, opts) {
|
|
8
|
+
return wrapGeneratorFunction(async function* (user) {
|
|
9
|
+
const seen = new Set();
|
|
10
|
+
const queue = [user];
|
|
11
|
+
// Maximum parallel requests (default to 300)
|
|
12
|
+
const maxParallel = opts?.parallel ?? 300;
|
|
13
|
+
// get the relays to load from
|
|
14
|
+
const relays = mergeRelaySets(user.relays, isObservable(opts?.extraRelays) ? await firstValueFrom(opts?.extraRelays) : opts?.extraRelays);
|
|
15
|
+
// Keep loading while the queue has items
|
|
16
|
+
while (queue.length > 0) {
|
|
17
|
+
// Process up to maxParallel items at once
|
|
18
|
+
const batch = queue.splice(0, maxParallel);
|
|
19
|
+
const promises = batch.map(async (pointer) => {
|
|
20
|
+
const address = {
|
|
21
|
+
kind: kinds.Contacts,
|
|
22
|
+
pubkey: pointer.pubkey,
|
|
23
|
+
relays: opts?.hints ? mergeRelaySets(pointer.relays, relays) : relays,
|
|
24
|
+
};
|
|
25
|
+
// load the contacts events
|
|
26
|
+
const events = await lastValueFrom(addressLoader(address).pipe(
|
|
27
|
+
// Pass all events to the store if set
|
|
28
|
+
opts?.eventStore ? mapEventsToStore(opts.eventStore) : identity,
|
|
29
|
+
// Conver to array
|
|
30
|
+
toArray()));
|
|
31
|
+
if (events.length === 0)
|
|
32
|
+
return;
|
|
33
|
+
const contacts = getProfilePointersFromList(events[events.length - 1]);
|
|
34
|
+
// if the distance is greater than 0, add the contacts to the queue
|
|
35
|
+
if (pointer.distance > 0) {
|
|
36
|
+
for (const contact of contacts) {
|
|
37
|
+
// Dont add any contacts that have already been seen
|
|
38
|
+
if (seen.has(contact.pubkey))
|
|
39
|
+
continue;
|
|
40
|
+
seen.add(contact.pubkey);
|
|
41
|
+
// Add to queue
|
|
42
|
+
queue.push({ ...contact, distance: pointer.distance - 1 });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
// Wait for all parallel operations to complete
|
|
47
|
+
await Promise.all(promises);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { IEventStore } from "applesauce-core";
|
|
2
2
|
import { NostrEvent } from "nostr-tools";
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import { Observable } from "rxjs";
|
|
4
|
+
import { CacheRequest, NostrRequest, UpstreamPool } from "../types.js";
|
|
5
|
+
export type TagValuePointer = {
|
|
5
6
|
/** The value of the tag to load */
|
|
6
7
|
value: string;
|
|
7
8
|
/** The relays to load from */
|
|
@@ -10,13 +11,10 @@ export type TabValuePointer = {
|
|
|
10
11
|
force?: boolean;
|
|
11
12
|
};
|
|
12
13
|
export type TagValueLoaderOptions = {
|
|
13
|
-
/**
|
|
14
|
-
name?: string;
|
|
15
|
-
/**
|
|
16
|
-
* Time interval to buffer requests in ms
|
|
17
|
-
* @default 1000
|
|
18
|
-
*/
|
|
14
|
+
/** Time interval to buffer requests in ms ( default 1000 ) */
|
|
19
15
|
bufferTime?: number;
|
|
16
|
+
/** Max buffer size ( default 200 ) */
|
|
17
|
+
bufferSize?: number;
|
|
20
18
|
/** Restrict queries to specific kinds */
|
|
21
19
|
kinds?: number[];
|
|
22
20
|
/** Restrict queries to specific authors */
|
|
@@ -26,14 +24,14 @@ export type TagValueLoaderOptions = {
|
|
|
26
24
|
/** Method used to load from the cache */
|
|
27
25
|
cacheRequest?: CacheRequest;
|
|
28
26
|
/** An array of relays to always fetch from */
|
|
29
|
-
extraRelays?: string[]
|
|
27
|
+
extraRelays?: string[] | Observable<string[]>;
|
|
28
|
+
/** An event store used to deduplicate events */
|
|
29
|
+
eventStore?: IEventStore;
|
|
30
30
|
};
|
|
31
|
-
export
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
constructor(request: NostrRequest, tagName: string, opts?: TagValueLoaderOptions);
|
|
39
|
-
}
|
|
31
|
+
export type TagValueLoader = (pointer: TagValuePointer) => Observable<NostrEvent>;
|
|
32
|
+
/** Creates a loader that gets tag values from the cache */
|
|
33
|
+
export declare function cacheTagValueLoader(request: CacheRequest, tagName: string, opts?: TagValueLoaderOptions): (pointers: TagValuePointer[]) => Observable<NostrEvent>;
|
|
34
|
+
/** Creates a loader that gets tag values from relays */
|
|
35
|
+
export declare function relaysTagValueLoader(request: NostrRequest, tagName: string, opts?: TagValueLoaderOptions): (pointers: TagValuePointer[]) => Observable<NostrEvent>;
|
|
36
|
+
/** Create a pre-built tag value loader that supports batching, caching, and relay hints */
|
|
37
|
+
export declare function createTagValueLoader(pool: UpstreamPool, tagName: string, opts?: TagValueLoaderOptions): TagValueLoader;
|
|
@@ -1,76 +1,75 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { bufferTime,
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { mergeRelaySets } from "applesauce-core/helpers";
|
|
3
|
+
import { bufferTime, EMPTY, merge } from "rxjs";
|
|
4
4
|
import { unique } from "../helpers/array.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
5
|
+
import { makeCacheRequest } from "../helpers/cache.js";
|
|
6
|
+
import { batchLoader, unwrap } from "../helpers/loaders.js";
|
|
7
|
+
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
8
|
+
/** Creates a loader that gets tag values from the cache */
|
|
9
|
+
export function cacheTagValueLoader(request, tagName, opts) {
|
|
10
|
+
return (pointers) => {
|
|
11
|
+
const baseFilter = {};
|
|
12
|
+
if (opts?.kinds)
|
|
13
|
+
baseFilter.kinds = opts.kinds;
|
|
14
|
+
if (opts?.authors)
|
|
15
|
+
baseFilter.authors = opts.authors;
|
|
16
|
+
if (opts?.since)
|
|
17
|
+
baseFilter.since = opts.since;
|
|
16
18
|
const filterTag = `#${tagName}`;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
// map for relay already exists, add the tag value
|
|
44
|
-
const filter = map[relay][0];
|
|
45
|
-
filter[filterTag].push(pointer.value);
|
|
46
|
-
}
|
|
19
|
+
const filter = { ...baseFilter, [filterTag]: unique(pointers.map((p) => p.value)) };
|
|
20
|
+
return makeCacheRequest(request, [filter]);
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/** Creates a loader that gets tag values from relays */
|
|
24
|
+
export function relaysTagValueLoader(request, tagName, opts) {
|
|
25
|
+
const filterTag = `#${tagName}`;
|
|
26
|
+
return (pointers) => unwrap(opts?.extraRelays, (extraRelays) => {
|
|
27
|
+
const baseFilter = {};
|
|
28
|
+
if (opts?.kinds)
|
|
29
|
+
baseFilter.kinds = opts.kinds;
|
|
30
|
+
if (opts?.authors)
|
|
31
|
+
baseFilter.authors = opts.authors;
|
|
32
|
+
if (opts?.since)
|
|
33
|
+
baseFilter.since = opts.since;
|
|
34
|
+
// build request map for relays
|
|
35
|
+
const requestMap = pointers.reduce((map, pointer) => {
|
|
36
|
+
const relays = mergeRelaySets(pointer.relays, extraRelays);
|
|
37
|
+
for (const relay of relays) {
|
|
38
|
+
if (!map[relay]) {
|
|
39
|
+
// create new filter for relay
|
|
40
|
+
map[relay] = { ...baseFilter, [filterTag]: [pointer.value] };
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
// map for relay already exists, add the tag value
|
|
44
|
+
map[relay][filterTag].push(pointer.value);
|
|
47
45
|
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
46
|
+
}
|
|
47
|
+
return map;
|
|
48
|
+
}, {});
|
|
49
|
+
const requests = Object.entries(requestMap).map(([relay, filter]) => request([relay], [filter]));
|
|
50
|
+
return merge(...requests);
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/** Create a pre-built tag value loader that supports batching, caching, and relay hints */
|
|
54
|
+
export function createTagValueLoader(pool, tagName, opts) {
|
|
55
|
+
const request = wrapUpstreamPool(pool);
|
|
56
|
+
return batchLoader(
|
|
57
|
+
// buffer requests by time or size
|
|
58
|
+
bufferTime(opts?.bufferTime ?? 1000, undefined, opts?.bufferSize ?? 200),
|
|
59
|
+
// Create a loader for batching
|
|
60
|
+
(pointers) => {
|
|
61
|
+
// Skip if there are no pointers
|
|
62
|
+
if (pointers.length === 0)
|
|
63
|
+
return EMPTY;
|
|
64
|
+
// Load from cache and relays in parallel
|
|
65
|
+
return merge(
|
|
66
|
+
// load from cache if available
|
|
67
|
+
opts?.cacheRequest ? cacheTagValueLoader(opts.cacheRequest, tagName, opts)(pointers) : [],
|
|
68
|
+
// load from relays
|
|
69
|
+
relaysTagValueLoader(request, tagName, opts)(pointers));
|
|
70
|
+
},
|
|
71
|
+
// Filter results based on requests
|
|
72
|
+
(pointer, event) => event.tags.some((tag) => tag[0] === tagName && tag[1] === pointer.value),
|
|
73
|
+
// Pass all events through the store if defined
|
|
74
|
+
opts?.eventStore && mapEventsToStore(opts?.eventStore));
|
|
76
75
|
}
|