applesauce-loaders 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -65
- package/dist/helpers/address-pointer.d.ts +2 -2
- package/dist/helpers/address-pointer.js +8 -10
- package/dist/helpers/cache.d.ts +9 -0
- package/dist/helpers/cache.js +22 -0
- package/dist/helpers/event-pointer.js +10 -10
- package/dist/helpers/index.d.ts +5 -0
- package/dist/helpers/index.js +5 -0
- package/dist/helpers/loaders.d.ts +3 -0
- package/dist/helpers/loaders.js +27 -0
- package/dist/helpers/pointer.d.ts +1 -0
- package/dist/helpers/pointer.js +1 -0
- package/dist/helpers/upstream.d.ts +7 -0
- package/dist/helpers/upstream.js +13 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/loaders/address-loader.d.ts +37 -0
- package/dist/loaders/address-loader.js +94 -0
- package/dist/loaders/event-loader.d.ts +33 -0
- package/dist/loaders/event-loader.js +92 -0
- package/dist/loaders/index.d.ts +8 -8
- package/dist/loaders/index.js +8 -8
- package/dist/loaders/reactions-loader.d.ts +12 -0
- package/dist/loaders/reactions-loader.js +18 -0
- package/dist/loaders/social-graph.d.ts +21 -0
- package/dist/loaders/social-graph.js +50 -0
- package/dist/loaders/tag-value-loader.d.ts +16 -18
- package/dist/loaders/tag-value-loader.js +72 -71
- package/dist/loaders/timeline-loader.d.ts +23 -21
- package/dist/loaders/timeline-loader.js +49 -55
- package/dist/loaders/user-lists-loader.d.ts +26 -0
- package/dist/loaders/user-lists-loader.js +33 -0
- package/dist/loaders/zaps-loader.d.ts +12 -0
- package/dist/loaders/zaps-loader.js +18 -0
- package/dist/operators/complete-on-eose.d.ts +4 -1
- package/dist/operators/complete-on-eose.js +4 -1
- package/dist/operators/generator.d.ts +13 -0
- package/dist/operators/generator.js +72 -0
- package/dist/operators/index.d.ts +1 -1
- package/dist/operators/index.js +1 -1
- package/dist/types.d.ts +14 -0
- package/package.json +6 -4
- package/dist/helpers/__tests__/address-pointer.test.js +0 -19
- package/dist/loaders/__tests__/dns-identity-loader.test.d.ts +0 -1
- package/dist/loaders/__tests__/dns-identity-loader.test.js +0 -59
- package/dist/loaders/__tests__/relay-timeline-loader.test.d.ts +0 -1
- package/dist/loaders/__tests__/relay-timeline-loader.test.js +0 -26
- package/dist/loaders/cache-timeline-loader.d.ts +0 -22
- package/dist/loaders/cache-timeline-loader.js +0 -61
- package/dist/loaders/loader.d.ts +0 -22
- package/dist/loaders/loader.js +0 -22
- package/dist/loaders/relay-timeline-loader.d.ts +0 -23
- package/dist/loaders/relay-timeline-loader.js +0 -71
- package/dist/loaders/replaceable-loader.d.ts +0 -27
- package/dist/loaders/replaceable-loader.js +0 -104
- package/dist/loaders/single-event-loader.d.ts +0 -32
- package/dist/loaders/single-event-loader.js +0 -74
- package/dist/loaders/user-sets-loader.d.ts +0 -33
- package/dist/loaders/user-sets-loader.js +0 -59
- package/dist/operators/__tests__/distinct-relays.test.d.ts +0 -1
- package/dist/operators/__tests__/distinct-relays.test.js +0 -75
- package/dist/operators/__tests__/generator-sequence.test.d.ts +0 -1
- package/dist/operators/__tests__/generator-sequence.test.js +0 -38
- package/dist/operators/generator-sequence.d.ts +0 -3
- package/dist/operators/generator-sequence.js +0 -53
- /package/dist/{helpers/__tests__/address-pointer.test.d.ts → types.js} +0 -0
package/README.md
CHANGED
|
@@ -1,93 +1,232 @@
|
|
|
1
1
|
# applesauce-loaders
|
|
2
2
|
|
|
3
|
-
A collection of
|
|
3
|
+
A collection of functional loading methods to make common event loading patterns easier.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[Documentation](https://hzrd149.github.io/applesauce/loaders/package.html) [typedoc](https://hzrd149.github.io/applesauce/typedoc/modules/applesauce-loaders.html)
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
## Address Loader
|
|
8
|
+
|
|
9
|
+
The Address Loader is a specialized loader for fetching Nostr replaceable events by their address (kind, pubkey, and optional identifier). It provides an efficient way to batch and deduplicate requests, cache results, and handle relay hints.
|
|
8
10
|
|
|
9
11
|
```ts
|
|
10
|
-
import {
|
|
12
|
+
import { createAddressLoader } from "applesauce-loaders/loaders";
|
|
11
13
|
import { EventStore } from "applesauce-core";
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
export const eventStore = new EventStore();
|
|
15
|
-
|
|
16
|
-
// Create a method to let the loaders use nostr-tools relay pool
|
|
17
|
-
function nostrRequest(relays: string[], filters: Filter[]) {
|
|
18
|
-
return new Observable((observer) => {
|
|
19
|
-
const sub = pool.subscribe(filters, {
|
|
20
|
-
onevent: (event) => observer.next(event),
|
|
21
|
-
oneose: () => {
|
|
22
|
-
sub.close();
|
|
23
|
-
observer.complete();
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
return () => sub.close();
|
|
28
|
-
});
|
|
29
|
-
}
|
|
14
|
+
import { RelayPool } from "applesauce-relay";
|
|
30
15
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
return new Observable((observer) => {
|
|
34
|
-
const sub = cacheRelay.subscribe(filters, {
|
|
35
|
-
onevent: (event) => observer.next(event),
|
|
36
|
-
oneose: () => {
|
|
37
|
-
sub.close();
|
|
38
|
-
observer.complete();
|
|
39
|
-
},
|
|
40
|
-
});
|
|
41
|
-
});
|
|
42
|
-
}
|
|
16
|
+
const eventStore = new EventStore();
|
|
17
|
+
const pool = new RelayPool();
|
|
43
18
|
|
|
44
|
-
|
|
19
|
+
// Create an address loader (do this once at the app level)
|
|
20
|
+
const addressLoader = createAddressLoader(pool, {
|
|
21
|
+
// Pass all events to the event store to deduplicate them
|
|
22
|
+
eventStore,
|
|
23
|
+
// Optional configuration options
|
|
45
24
|
bufferTime: 1000,
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
// lookup relays are used as a fallback if the event cant be found
|
|
49
|
-
lookupRelays: ["wss://purplepag.es/"],
|
|
25
|
+
followRelayHints: true,
|
|
26
|
+
extraRelays: ["wss://relay.example.com"],
|
|
50
27
|
});
|
|
51
28
|
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
29
|
+
// Load a profile (kind 0)
|
|
30
|
+
addressLoader({
|
|
31
|
+
kind: 0,
|
|
32
|
+
pubkey: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d",
|
|
33
|
+
relays: ["wss://relay.example.com"],
|
|
34
|
+
}).subscribe((event) => {
|
|
35
|
+
// Handle the loaded event
|
|
36
|
+
console.log(event);
|
|
56
37
|
});
|
|
57
38
|
|
|
58
|
-
//
|
|
59
|
-
|
|
60
|
-
kind:
|
|
39
|
+
// Load a contact list (kind 3)
|
|
40
|
+
addressLoader({
|
|
41
|
+
kind: 3,
|
|
61
42
|
pubkey: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d",
|
|
62
|
-
relays: ["wss://
|
|
43
|
+
relays: ["wss://relay.example.com"],
|
|
44
|
+
}).subscribe((event) => {
|
|
45
|
+
// Handle the loaded event
|
|
46
|
+
console.log(event);
|
|
63
47
|
});
|
|
64
48
|
|
|
65
|
-
//
|
|
66
|
-
|
|
49
|
+
// Load a parameterized replaceable event
|
|
50
|
+
addressLoader({
|
|
67
51
|
kind: 30000,
|
|
68
52
|
pubkey: "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d",
|
|
69
53
|
identifier: "list of bad people",
|
|
70
|
-
relays: ["wss://
|
|
54
|
+
relays: ["wss://relay.example.com"],
|
|
55
|
+
}).subscribe((event) => {
|
|
56
|
+
// Handle the loaded event
|
|
57
|
+
console.log(event);
|
|
71
58
|
});
|
|
59
|
+
```
|
|
72
60
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
61
|
+
## Event Loader
|
|
62
|
+
|
|
63
|
+
The Event Loader is a specialized loader for fetching Nostr events by their IDs. It provides an efficient way to batch and deduplicate requests, cache results, and handle relay hints.
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import { createEventLoader } from "applesauce-loaders/loaders";
|
|
67
|
+
|
|
68
|
+
// Create an event loader (do this once at the app level)
|
|
69
|
+
const eventLoader = createEventLoader(pool, {
|
|
70
|
+
// Pass all events to the event store to deduplicate them
|
|
71
|
+
eventStore,
|
|
72
|
+
// Optional configuration options
|
|
73
|
+
bufferTime: 1000,
|
|
74
|
+
followRelayHints: true,
|
|
75
|
+
extraRelays: ["wss://relay.example.com"],
|
|
77
76
|
});
|
|
78
77
|
|
|
79
|
-
//
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
78
|
+
// Load an event by ID
|
|
79
|
+
eventLoader({
|
|
80
|
+
id: "2650f6292166624f45795248edb9ca136c276a3d10a0d8f4efd2b8b23eb2d5fc",
|
|
81
|
+
relays: ["wss://relay.example.com"],
|
|
82
|
+
}).subscribe((event) => {
|
|
83
|
+
// Handle the loaded event
|
|
84
|
+
console.log(event);
|
|
84
85
|
});
|
|
85
86
|
|
|
86
|
-
//
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
87
|
+
// Load from extra relays
|
|
88
|
+
eventLoader({
|
|
89
|
+
id: "2650f6292166624f45795248edb9ca136c276a3d10a0d8f4efd2b8b23eb2d5fc",
|
|
90
|
+
relays: ["wss://relay.example.com"],
|
|
91
|
+
}).subscribe((event) => {
|
|
92
|
+
// Handle the loaded event
|
|
93
|
+
console.log(event);
|
|
94
|
+
});
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Timeline Loader
|
|
98
|
+
|
|
99
|
+
The Timeline Loader is designed for fetching paginated Nostr events in chronological order. It maintains state between calls, allowing you to efficiently load timeline events in blocks until you reach a specific timestamp or exhaust available events.
|
|
100
|
+
|
|
101
|
+
```ts
|
|
102
|
+
import { createTimelineLoader } from "applesauce-loaders/loaders";
|
|
103
|
+
|
|
104
|
+
// Create a timeline loader
|
|
105
|
+
const timelineLoader = createTimelineLoader(
|
|
106
|
+
pool,
|
|
107
|
+
["wss://relay.example.com"],
|
|
108
|
+
{ kinds: [1] }, // Load text notes
|
|
109
|
+
{ eventStore },
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
// Initial load - gets the most recent events
|
|
113
|
+
timelineLoader().subscribe((event) => {
|
|
114
|
+
console.log("Loaded event:", event);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Later, load older events by calling the loader again
|
|
118
|
+
// Each call continues from where the previous one left off
|
|
119
|
+
timelineLoader().subscribe((event) => {
|
|
120
|
+
console.log("Loaded older event:", event);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
// Load events until a specific timestamp
|
|
124
|
+
const oneWeekAgo = Math.floor(Date.now() / 1000) - 7 * 24 * 60 * 60;
|
|
125
|
+
timelineLoader(oneWeekAgo).subscribe((event) => {
|
|
126
|
+
console.log("Event from last week:", event);
|
|
92
127
|
});
|
|
93
128
|
```
|
|
129
|
+
|
|
130
|
+
## Loading from cache
|
|
131
|
+
|
|
132
|
+
All loaders support a `cacheRequest` option to load events from a local cache.
|
|
133
|
+
|
|
134
|
+
```ts
|
|
135
|
+
import { NostrEvent, Filter } from "nostr-tools";
|
|
136
|
+
import { createEventLoader } from "applesauce-loaders/loaders";
|
|
137
|
+
|
|
138
|
+
// Custom method for loading events from a database
|
|
139
|
+
async function cacheRequest(filters: Filter[]): Promise<NostrEvent[]> {
|
|
140
|
+
return await cacheDatabase.getEvents(filters);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const eventLoader = createEventLoader(pool, {
|
|
144
|
+
// Pass all events to the event store to deduplicate them
|
|
145
|
+
eventStore,
|
|
146
|
+
// Pass a custom cache method
|
|
147
|
+
cacheRequest,
|
|
148
|
+
// Optional configuration options
|
|
149
|
+
bufferTime: 1000,
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
// Because no relays are specified, the event will be loaded from the cache
|
|
153
|
+
eventLoader({
|
|
154
|
+
id: "2650f6292166624f45795248edb9ca136c276a3d10a0d8f4efd2b8b23eb2d5fc",
|
|
155
|
+
}).subscribe((event) => {
|
|
156
|
+
// Handle the loaded event
|
|
157
|
+
console.log(event);
|
|
158
|
+
});
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Configuration Options
|
|
162
|
+
|
|
163
|
+
All loaders accept these common configuration options:
|
|
164
|
+
|
|
165
|
+
### Address Loader Options
|
|
166
|
+
|
|
167
|
+
- `bufferTime`: Time interval to buffer requests in ms (default 1000)
|
|
168
|
+
- `bufferSize`: Max buffer size (default 200)
|
|
169
|
+
- `eventStore`: An event store used to deduplicate events
|
|
170
|
+
- `cacheRequest`: A method used to load events from a local cache
|
|
171
|
+
- `followRelayHints`: Whether to follow relay hints (default true)
|
|
172
|
+
- `lookupRelays`: Fallback lookup relays to check when event can't be found
|
|
173
|
+
- `extraRelays`: An array of relays to always fetch from
|
|
174
|
+
|
|
175
|
+
### Event Loader Options
|
|
176
|
+
|
|
177
|
+
- `bufferTime`: Time interval to buffer requests in ms (default 1000)
|
|
178
|
+
- `bufferSize`: Max buffer size (default 200)
|
|
179
|
+
- `eventStore`: An event store used to deduplicate events
|
|
180
|
+
- `cacheRequest`: A method used to load events from a local cache
|
|
181
|
+
- `followRelayHints`: Whether to follow relay hints (default true)
|
|
182
|
+
- `extraRelays`: An array of relays to always fetch from
|
|
183
|
+
|
|
184
|
+
### Timeline Loader Options
|
|
185
|
+
|
|
186
|
+
- `limit`: Maximum number of events to request per filter
|
|
187
|
+
- `cache`: A method used to load events from a local cache
|
|
188
|
+
- `eventStore`: An event store to pass all events to
|
|
189
|
+
|
|
190
|
+
## Working with Relay Pools
|
|
191
|
+
|
|
192
|
+
All loaders require a request method for loading Nostr events from relays. You can provide this in multiple ways:
|
|
193
|
+
|
|
194
|
+
### Using a RelayPool instance
|
|
195
|
+
|
|
196
|
+
The simplest approach is to pass a RelayPool instance directly:
|
|
197
|
+
|
|
198
|
+
```ts
|
|
199
|
+
import { createAddressLoader, createEventLoader } from "applesauce-loaders/loaders";
|
|
200
|
+
import { RelayPool } from "applesauce-relay";
|
|
201
|
+
|
|
202
|
+
const pool = new RelayPool();
|
|
203
|
+
const addressLoader = createAddressLoader(pool, { eventStore });
|
|
204
|
+
const eventLoader = createEventLoader(pool, { eventStore });
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Using a custom request method
|
|
208
|
+
|
|
209
|
+
You can also provide a custom request method, such as one from nostr-tools:
|
|
210
|
+
|
|
211
|
+
```ts
|
|
212
|
+
import { createEventLoader } from "applesauce-loaders/loaders";
|
|
213
|
+
import { SimplePool } from "nostr-tools";
|
|
214
|
+
import { Observable } from "rxjs";
|
|
215
|
+
|
|
216
|
+
const pool = SimplePool();
|
|
217
|
+
|
|
218
|
+
// Create a custom request function using nostr-tools
|
|
219
|
+
function customRequest(relays, filters) {
|
|
220
|
+
return new Observable((observer) => {
|
|
221
|
+
const sub = pool.subscribeMany(relays, filters, {
|
|
222
|
+
onevent: (event) => observer.next(event),
|
|
223
|
+
eose: () => observer.complete(),
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
return () => sub.close();
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// Create event loader with custom request
|
|
231
|
+
const eventLoader = createEventLoader(customRequest, options);
|
|
232
|
+
```
|
|
@@ -8,7 +8,7 @@ export type LoadableAddressPointer = {
|
|
|
8
8
|
identifier?: string;
|
|
9
9
|
/** Relays to load from */
|
|
10
10
|
relays?: string[];
|
|
11
|
-
/**
|
|
11
|
+
/** Ignore all forms of caching */
|
|
12
12
|
force?: boolean;
|
|
13
13
|
};
|
|
14
14
|
/** Converts an array of address pointers to a filter */
|
|
@@ -23,7 +23,7 @@ export declare function groupAddressPointersByKind(pointers: AddressPointerWitho
|
|
|
23
23
|
export declare function groupAddressPointersByPubkey(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]>;
|
|
24
24
|
/** Groups address pointers by kind or pubkey depending on which is most optimal */
|
|
25
25
|
export declare function groupAddressPointersByPubkeyOrKind(pointers: AddressPointerWithoutD[]): Map<string, AddressPointerWithoutD[]> | Map<number, AddressPointerWithoutD[]>;
|
|
26
|
+
/** @deprecated use mergeRelaySets instead */
|
|
26
27
|
export declare function getRelaysFromPointers(pointers: AddressPointerWithoutD[]): Set<string>;
|
|
27
|
-
export declare function getAddressPointerId<T extends AddressPointerWithoutD>(pointer: T): string;
|
|
28
28
|
/** deduplicates an array of address pointers and merges their relays array */
|
|
29
29
|
export declare function consolidateAddressPointers(pointers: LoadableAddressPointer[]): LoadableAddressPointer[];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createReplaceableAddress, mergeRelaySets } from "applesauce-core/helpers";
|
|
2
2
|
import { isAddressableKind, isReplaceableKind } from "nostr-tools/kinds";
|
|
3
3
|
import { unique } from "./array.js";
|
|
4
4
|
/** Converts an array of address pointers to a filter */
|
|
@@ -62,6 +62,7 @@ export function groupAddressPointersByPubkeyOrKind(pointers) {
|
|
|
62
62
|
const pubkeys = new Set(pointers.map((p) => p.pubkey));
|
|
63
63
|
return pubkeys.size < kinds.size ? groupAddressPointersByPubkey(pointers) : groupAddressPointersByKind(pointers);
|
|
64
64
|
}
|
|
65
|
+
/** @deprecated use mergeRelaySets instead */
|
|
65
66
|
export function getRelaysFromPointers(pointers) {
|
|
66
67
|
const relays = new Set();
|
|
67
68
|
for (const pointer of pointers) {
|
|
@@ -73,9 +74,6 @@ export function getRelaysFromPointers(pointers) {
|
|
|
73
74
|
}
|
|
74
75
|
return relays;
|
|
75
76
|
}
|
|
76
|
-
export function getAddressPointerId(pointer) {
|
|
77
|
-
return getReplaceableUID(pointer.kind, pointer.pubkey, pointer.identifier);
|
|
78
|
-
}
|
|
79
77
|
/** deep clone a loadable pointer to ensure its safe to modify */
|
|
80
78
|
function cloneLoadablePointer(pointer) {
|
|
81
79
|
const clone = { ...pointer };
|
|
@@ -85,12 +83,12 @@ function cloneLoadablePointer(pointer) {
|
|
|
85
83
|
}
|
|
86
84
|
/** deduplicates an array of address pointers and merges their relays array */
|
|
87
85
|
export function consolidateAddressPointers(pointers) {
|
|
88
|
-
const
|
|
86
|
+
const byAddress = new Map();
|
|
89
87
|
for (const pointer of pointers) {
|
|
90
|
-
const
|
|
91
|
-
if (
|
|
88
|
+
const addr = createReplaceableAddress(pointer.kind, pointer.pubkey, pointer.identifier);
|
|
89
|
+
if (byAddress.has(addr)) {
|
|
92
90
|
// duplicate, merge pointers
|
|
93
|
-
const current =
|
|
91
|
+
const current = byAddress.get(addr);
|
|
94
92
|
// merge relays
|
|
95
93
|
if (pointer.relays) {
|
|
96
94
|
if (current.relays)
|
|
@@ -104,8 +102,8 @@ export function consolidateAddressPointers(pointers) {
|
|
|
104
102
|
}
|
|
105
103
|
}
|
|
106
104
|
else
|
|
107
|
-
|
|
105
|
+
byAddress.set(addr, cloneLoadablePointer(pointer));
|
|
108
106
|
}
|
|
109
107
|
// return consolidated pointers
|
|
110
|
-
return Array.from(
|
|
108
|
+
return Array.from(byAddress.values());
|
|
111
109
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Observable } from "rxjs";
|
|
2
|
+
import { Filter, NostrEvent } from "nostr-tools";
|
|
3
|
+
import { CacheRequest, FilterRequest } from "../types.js";
|
|
4
|
+
/** Calls the cache request and converts the reponse into an observable */
|
|
5
|
+
export declare function unwrapCacheRequest(request: CacheRequest, filters: Filter[]): Observable<NostrEvent>;
|
|
6
|
+
/** Calls a cache request method with filters and marks all returned events as being from the cache */
|
|
7
|
+
export declare function makeCacheRequest(request: CacheRequest, filters: Filter[]): Observable<NostrEvent>;
|
|
8
|
+
/** Wraps a cache request method and returns a FilterRequest */
|
|
9
|
+
export declare function wrapCacheRequest(request: CacheRequest): FilterRequest;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { from, isObservable, of, switchMap, tap } from "rxjs";
|
|
2
|
+
import { markFromCache } from "applesauce-core/helpers";
|
|
3
|
+
/** Calls the cache request and converts the reponse into an observable */
|
|
4
|
+
export function unwrapCacheRequest(request, filters) {
|
|
5
|
+
const result = request(filters);
|
|
6
|
+
if (isObservable(result))
|
|
7
|
+
return result;
|
|
8
|
+
else if (result instanceof Promise)
|
|
9
|
+
return from(result).pipe(switchMap((v) => (Array.isArray(v) ? from(v) : of(v))));
|
|
10
|
+
else if (Array.isArray(result))
|
|
11
|
+
return from(result);
|
|
12
|
+
else
|
|
13
|
+
return of(result);
|
|
14
|
+
}
|
|
15
|
+
/** Calls a cache request method with filters and marks all returned events as being from the cache */
|
|
16
|
+
export function makeCacheRequest(request, filters) {
|
|
17
|
+
return unwrapCacheRequest(request, filters).pipe(tap((e) => markFromCache(e)));
|
|
18
|
+
}
|
|
19
|
+
/** Wraps a cache request method and returns a FilterRequest */
|
|
20
|
+
export function wrapCacheRequest(request) {
|
|
21
|
+
return (filters) => makeCacheRequest(request, filters);
|
|
22
|
+
}
|
|
@@ -3,17 +3,17 @@ export function consolidateEventPointers(pointers) {
|
|
|
3
3
|
let ids = new Map();
|
|
4
4
|
for (let pointer of pointers) {
|
|
5
5
|
let existing = ids.get(pointer.id);
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
if (
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
6
|
+
// merge relays
|
|
7
|
+
if (existing && pointer.relays) {
|
|
8
|
+
if (!existing.relays)
|
|
9
|
+
existing.relays = [...pointer.relays];
|
|
10
|
+
// TODO: maybe use mergeRelaySets here if its performant enough
|
|
11
|
+
else
|
|
12
|
+
existing.relays = [...existing.relays, ...pointer.relays.filter((r) => !existing.relays.includes(r))];
|
|
13
|
+
}
|
|
14
|
+
else if (!existing) {
|
|
15
|
+
ids.set(pointer.id, { ...pointer });
|
|
14
16
|
}
|
|
15
|
-
else
|
|
16
|
-
ids.set(pointer.id, pointer);
|
|
17
17
|
}
|
|
18
18
|
return Array.from(ids.values());
|
|
19
19
|
}
|
package/dist/helpers/index.d.ts
CHANGED
package/dist/helpers/index.js
CHANGED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { MonoTypeOperatorFunction, Observable, OperatorFunction } from "rxjs";
|
|
2
|
+
/** Creates a loader that takes a single value and batches the requests to an upstream loader */
|
|
3
|
+
export declare function batchLoader<Input extends unknown = unknown, Output extends unknown = unknown>(buffer: OperatorFunction<Input, Input[]>, upstream: (input: Input[]) => Observable<Output>, matcher: (input: Input, output: Output) => boolean, output?: MonoTypeOperatorFunction<Output>): (value: Input) => Observable<Output>;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { filter, identity, map, mergeAll, Observable, share, Subject, take, } from "rxjs";
|
|
2
|
+
/** Creates a loader that takes a single value and batches the requests to an upstream loader */
|
|
3
|
+
export function batchLoader(buffer, upstream, matcher, output) {
|
|
4
|
+
const queue = new Subject();
|
|
5
|
+
const requests = queue.pipe(buffer,
|
|
6
|
+
// ignore empty buffers
|
|
7
|
+
filter((buffer) => buffer.length > 0),
|
|
8
|
+
// create a new upstream request for each buffer and make sure only one is created
|
|
9
|
+
map((v) => upstream(v).pipe(share())),
|
|
10
|
+
// Make sure that only a single upstream buffer is created
|
|
11
|
+
share());
|
|
12
|
+
return (input) => new Observable((observer) => {
|
|
13
|
+
// Add the pointer to the queue when observable is subscribed
|
|
14
|
+
setTimeout(() => queue.next(input), 0);
|
|
15
|
+
return requests
|
|
16
|
+
.pipe(
|
|
17
|
+
// wait for the next batch to run
|
|
18
|
+
take(1),
|
|
19
|
+
// subscribe to it
|
|
20
|
+
mergeAll(),
|
|
21
|
+
// filter the results for the requested input
|
|
22
|
+
filter((output) => matcher(input, output)),
|
|
23
|
+
// Extra output operations
|
|
24
|
+
output ?? identity)
|
|
25
|
+
.subscribe(observer);
|
|
26
|
+
});
|
|
27
|
+
}
|
package/dist/helpers/pointer.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { mergeRelaySets } from "applesauce-core/helpers";
|
|
2
|
+
/** Takes an array of objects with a `relays` property and returns a map of relays to the objects */
|
|
2
3
|
export function groupByRelay(pointers, extraRelays) {
|
|
3
4
|
let byRelay = new Map();
|
|
4
5
|
for (const pointer of pointers) {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { Observable } from "rxjs";
|
|
2
|
+
import { NostrRequest, UpstreamPool } from "../types.js";
|
|
3
|
+
import { Filter, NostrEvent } from "nostr-tools";
|
|
4
|
+
/** Makes a nostr request on the upstream pool */
|
|
5
|
+
export declare function makeUpstreamRequest(pool: UpstreamPool, relays: string[], filters: Filter[]): Observable<NostrEvent>;
|
|
6
|
+
/** Wraps an upstream pool and returns a NostrRequest */
|
|
7
|
+
export declare function wrapUpstreamPool(pool: UpstreamPool): NostrRequest;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Makes a nostr request on the upstream pool */
|
|
2
|
+
export function makeUpstreamRequest(pool, relays, filters) {
|
|
3
|
+
if (typeof pool === "function")
|
|
4
|
+
return pool(relays, filters);
|
|
5
|
+
else if (typeof pool === "object" && "request" in pool)
|
|
6
|
+
return pool.request(relays, filters);
|
|
7
|
+
else
|
|
8
|
+
throw new Error("Invalid upstream pool");
|
|
9
|
+
}
|
|
10
|
+
/** Wraps an upstream pool and returns a NostrRequest */
|
|
11
|
+
export function wrapUpstreamPool(pool) {
|
|
12
|
+
return (relays, filters) => makeUpstreamRequest(pool, relays, filters);
|
|
13
|
+
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { IEventStore } from "applesauce-core";
|
|
2
|
+
import { NostrEvent } from "nostr-tools";
|
|
3
|
+
import { Observable } from "rxjs";
|
|
4
|
+
import { LoadableAddressPointer } from "../helpers/address-pointer.js";
|
|
5
|
+
import { CacheRequest, NostrRequest, UpstreamPool } from "../types.js";
|
|
6
|
+
/** A method that takes address pointers and returns an observable of events */
|
|
7
|
+
export type AddressPointersLoader = (pointers: LoadableAddressPointer[]) => Observable<NostrEvent>;
|
|
8
|
+
export type AddressPointerLoader = (pointer: LoadableAddressPointer) => Observable<NostrEvent>;
|
|
9
|
+
/**
|
|
10
|
+
* Loads address pointers from an async cache
|
|
11
|
+
* @note ignores pointers with force=true
|
|
12
|
+
*/
|
|
13
|
+
export declare function cacheAddressPointersLoader(request: CacheRequest): AddressPointersLoader;
|
|
14
|
+
/** Loads address pointers from the relay hints */
|
|
15
|
+
export declare function relayHintsAddressPointersLoader(request: NostrRequest): AddressPointersLoader;
|
|
16
|
+
/** Loads address pointers from an array of relays */
|
|
17
|
+
export declare function relaysAddressPointersLoader(request: NostrRequest, relays: Observable<string[]> | string[]): AddressPointersLoader;
|
|
18
|
+
/** Creates a loader that loads all event pointers based on their relays */
|
|
19
|
+
export declare function addressPointerLoadingSequence(...loaders: (AddressPointersLoader | undefined)[]): AddressPointersLoader;
|
|
20
|
+
export type AddressLoaderOptions = Partial<{
|
|
21
|
+
/** Time interval to buffer requests in ms ( default 1000 ) */
|
|
22
|
+
bufferTime: number;
|
|
23
|
+
/** Max buffer size ( default 200 ) */
|
|
24
|
+
bufferSize: number;
|
|
25
|
+
/** An event store used to deduplicate events */
|
|
26
|
+
eventStore: IEventStore;
|
|
27
|
+
/** A method used to load events from a local cache */
|
|
28
|
+
cacheRequest: CacheRequest;
|
|
29
|
+
/** Whether to follow relay hints ( default true ) */
|
|
30
|
+
followRelayHints: boolean;
|
|
31
|
+
/** Fallback lookup relays to check when event cant be found */
|
|
32
|
+
lookupRelays: string[] | Observable<string[]>;
|
|
33
|
+
/** An array of relays to always fetch from */
|
|
34
|
+
extraRelays: string[] | Observable<string[]>;
|
|
35
|
+
}>;
|
|
36
|
+
/** Create a pre-built address pointer loader that supports batching, caching, and lookup relays */
|
|
37
|
+
export declare function createAddressLoader(pool: UpstreamPool, opts?: AddressLoaderOptions): AddressPointerLoader;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { mapEventsToStore } from "applesauce-core";
|
|
2
|
+
import { createReplaceableAddress, getReplaceableAddress, getReplaceableIdentifier, isReplaceable, mergeRelaySets, } from "applesauce-core/helpers";
|
|
3
|
+
import { bufferTime, catchError, EMPTY, filter, isObservable, map, of, pipe, switchMap, take } from "rxjs";
|
|
4
|
+
import { consolidateAddressPointers, createFiltersFromAddressPointers, isLoadableAddressPointer, } from "../helpers/address-pointer.js";
|
|
5
|
+
import { makeCacheRequest, wrapCacheRequest } from "../helpers/cache.js";
|
|
6
|
+
import { batchLoader } from "../helpers/loaders.js";
|
|
7
|
+
import { wrapGeneratorFunction } from "../operators/generator.js";
|
|
8
|
+
import { wrapUpstreamPool } from "../helpers/upstream.js";
|
|
9
|
+
/**
|
|
10
|
+
* Loads address pointers from an async cache
|
|
11
|
+
* @note ignores pointers with force=true
|
|
12
|
+
*/
|
|
13
|
+
export function cacheAddressPointersLoader(request) {
|
|
14
|
+
return (pointers) => makeCacheRequest(request, createFiltersFromAddressPointers(pointers
|
|
15
|
+
// Ignore pointers that want to skip cache
|
|
16
|
+
.filter((p) => p.force !== true)));
|
|
17
|
+
}
|
|
18
|
+
/** Loads address pointers from the relay hints */
|
|
19
|
+
export function relayHintsAddressPointersLoader(request) {
|
|
20
|
+
return (pointers) => {
|
|
21
|
+
const relays = mergeRelaySets(...pointers.map((p) => p.relays));
|
|
22
|
+
if (relays.length === 0)
|
|
23
|
+
return EMPTY;
|
|
24
|
+
const filters = createFiltersFromAddressPointers(pointers);
|
|
25
|
+
return request(relays, filters);
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/** Loads address pointers from an array of relays */
|
|
29
|
+
export function relaysAddressPointersLoader(request, relays) {
|
|
30
|
+
return (pointers) =>
|
|
31
|
+
// Resolve the relays as an observable
|
|
32
|
+
(isObservable(relays) ? relays : of(relays)).pipe(
|
|
33
|
+
// Only take the first value
|
|
34
|
+
take(1),
|
|
35
|
+
// Make the request
|
|
36
|
+
switchMap((relays) => {
|
|
37
|
+
if (relays.length === 0)
|
|
38
|
+
return EMPTY;
|
|
39
|
+
const filters = createFiltersFromAddressPointers(pointers);
|
|
40
|
+
return request(relays, filters);
|
|
41
|
+
}));
|
|
42
|
+
}
|
|
43
|
+
/** Creates a loader that loads all event pointers based on their relays */
|
|
44
|
+
export function addressPointerLoadingSequence(...loaders) {
|
|
45
|
+
return wrapGeneratorFunction(function* (pointers) {
|
|
46
|
+
let remaining = Array.from(pointers);
|
|
47
|
+
for (const loader of loaders) {
|
|
48
|
+
if (loader === undefined)
|
|
49
|
+
continue;
|
|
50
|
+
const results = yield loader(remaining).pipe(
|
|
51
|
+
// If the loader throws an error, skip it
|
|
52
|
+
catchError(() => EMPTY));
|
|
53
|
+
// Get set of addresses loaded
|
|
54
|
+
const addresses = new Set(results.filter((e) => isReplaceable(e.kind)).map((event) => getReplaceableAddress(event)));
|
|
55
|
+
// Remove the pointers that were loaded
|
|
56
|
+
remaining = remaining.filter((p) => !addresses.has(createReplaceableAddress(p.kind, p.pubkey, p.identifier)) || p.force === true);
|
|
57
|
+
// If there are no remaining pointers, complete
|
|
58
|
+
if (remaining.length === 0)
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
/** Create a pre-built address pointer loader that supports batching, caching, and lookup relays */
|
|
64
|
+
export function createAddressLoader(pool, opts) {
|
|
65
|
+
const request = wrapUpstreamPool(pool);
|
|
66
|
+
const cacheRequest = opts?.cacheRequest ? wrapCacheRequest(opts.cacheRequest) : undefined;
|
|
67
|
+
return batchLoader(
|
|
68
|
+
// Create batching sequence
|
|
69
|
+
pipe(
|
|
70
|
+
// filter out invalid pointers
|
|
71
|
+
filter(isLoadableAddressPointer),
|
|
72
|
+
// buffer requests by time or size
|
|
73
|
+
bufferTime(opts?.bufferTime ?? 1000, undefined, opts?.bufferSize ?? 200),
|
|
74
|
+
// Ingore empty buffers
|
|
75
|
+
filter((b) => b.length > 0),
|
|
76
|
+
// consolidate buffered pointers
|
|
77
|
+
map(consolidateAddressPointers)),
|
|
78
|
+
// Create a loader for batching
|
|
79
|
+
addressPointerLoadingSequence(
|
|
80
|
+
// Step 1. load from cache if available
|
|
81
|
+
cacheRequest ? cacheAddressPointersLoader(cacheRequest) : undefined,
|
|
82
|
+
// Step 2. load from relay hints on pointers
|
|
83
|
+
opts?.followRelayHints !== false ? relayHintsAddressPointersLoader(request) : undefined,
|
|
84
|
+
// Step 3. load from extra relays
|
|
85
|
+
opts?.extraRelays ? relaysAddressPointersLoader(request, opts.extraRelays) : undefined,
|
|
86
|
+
// Step 4. load from lookup relays
|
|
87
|
+
opts?.lookupRelays ? relaysAddressPointersLoader(request, opts.lookupRelays) : undefined),
|
|
88
|
+
// Filter resutls based on requests
|
|
89
|
+
(pointer, event) => event.kind === pointer.kind &&
|
|
90
|
+
event.pubkey === pointer.pubkey &&
|
|
91
|
+
(pointer.identifier ? getReplaceableIdentifier(event) === pointer.identifier : true),
|
|
92
|
+
// Pass all events through the store if defined
|
|
93
|
+
opts?.eventStore && mapEventsToStore(opts.eventStore));
|
|
94
|
+
}
|