applesauce-loaders 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helpers/__tests__/address-pointer.test.d.ts +1 -0
- package/dist/helpers/__tests__/address-pointer.test.js +19 -0
- package/dist/helpers/address-pointer.d.ts +15 -2
- package/dist/helpers/address-pointer.js +48 -4
- package/dist/helpers/array.d.ts +0 -2
- package/dist/helpers/array.js +0 -17
- package/dist/helpers/dns-identity.d.ts +40 -0
- package/dist/helpers/dns-identity.js +50 -0
- package/dist/helpers/event-pointer.d.ts +5 -0
- package/dist/helpers/event-pointer.js +19 -0
- package/dist/helpers/index.d.ts +1 -0
- package/dist/helpers/index.js +1 -0
- package/dist/helpers/pointer.d.ts +11 -0
- package/dist/helpers/pointer.js +47 -0
- package/dist/helpers/rx-nostr.d.ts +2 -0
- package/dist/helpers/rx-nostr.js +5 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/loaders/__tests__/dns-identity-loader.test.d.ts +1 -0
- package/dist/loaders/__tests__/dns-identity-loader.test.js +59 -0
- package/dist/loaders/__tests__/relay-timeline-loader.test.d.ts +1 -0
- package/dist/loaders/__tests__/relay-timeline-loader.test.js +26 -0
- package/dist/loaders/__tests__/request-loader.test.d.ts +1 -0
- package/dist/loaders/__tests__/request-loader.test.js +37 -0
- package/dist/loaders/cache-timeline-loader.d.ts +22 -0
- package/dist/loaders/cache-timeline-loader.js +61 -0
- package/dist/loaders/dns-identity-loader.d.ts +25 -0
- package/dist/loaders/dns-identity-loader.js +66 -0
- package/dist/loaders/index.d.ts +8 -1
- package/dist/loaders/index.js +8 -1
- package/dist/loaders/loader.d.ts +14 -8
- package/dist/loaders/loader.js +7 -2
- package/dist/loaders/relay-timeline-loader.d.ts +24 -0
- package/dist/loaders/relay-timeline-loader.js +70 -0
- package/dist/loaders/replaceable-loader.d.ts +7 -15
- package/dist/loaders/replaceable-loader.js +49 -106
- package/dist/loaders/request-loader.d.ts +28 -0
- package/dist/loaders/request-loader.js +42 -0
- package/dist/loaders/single-event-loader.d.ts +26 -0
- package/dist/loaders/single-event-loader.js +76 -0
- package/dist/loaders/tag-value-loader.d.ts +33 -0
- package/dist/loaders/tag-value-loader.js +75 -0
- package/dist/loaders/timeline-loader.d.ts +22 -0
- package/dist/loaders/timeline-loader.js +56 -0
- package/dist/loaders/user-sets-loader.d.ts +31 -0
- package/dist/loaders/user-sets-loader.js +66 -0
- package/dist/operators/__tests__/distinct-relays.test.d.ts +1 -0
- package/dist/operators/__tests__/distinct-relays.test.js +75 -0
- package/dist/operators/__tests__/generator-sequence.test.d.ts +1 -0
- package/dist/operators/__tests__/generator-sequence.test.js +38 -0
- package/dist/operators/distinct-relays.d.ts +4 -0
- package/dist/operators/distinct-relays.js +14 -0
- package/dist/operators/distinct-timeout.d.ts +3 -0
- package/dist/operators/distinct-timeout.js +15 -0
- package/dist/operators/generator-sequence.d.ts +1 -1
- package/dist/operators/generator-sequence.js +42 -34
- package/dist/operators/index.d.ts +2 -3
- package/dist/operators/index.js +2 -3
- package/package.json +26 -7
- package/dist/loaders/single-relay-replaceable-loader.d.ts +0 -14
- package/dist/loaders/single-relay-replaceable-loader.js +0 -51
- package/dist/operators/address-pointers-request.d.ts +0 -5
- package/dist/operators/address-pointers-request.js +0 -25
- package/dist/operators/max-filters.d.ts +0 -4
- package/dist/operators/max-filters.js +0 -8
- package/dist/operators/relay-request.d.ts +0 -4
- package/dist/operators/relay-request.js +0 -9
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { bufferTime, filter, from, map, mergeAll, tap } from "rxjs";
|
|
2
|
+
import { createRxOneshotReq } from "rx-nostr";
|
|
3
|
+
import { markFromCache } from "applesauce-core/helpers";
|
|
4
|
+
import { logger } from "applesauce-core";
|
|
5
|
+
import { nanoid } from "nanoid";
|
|
6
|
+
import { Loader } from "./loader.js";
|
|
7
|
+
import { generatorSequence } from "../operators/generator-sequence.js";
|
|
8
|
+
import { distinctRelaysBatch } from "../operators/distinct-relays.js";
|
|
9
|
+
import { groupByRelay } from "../helpers/pointer.js";
|
|
10
|
+
import { consolidateEventPointers } from "../helpers/event-pointer.js";
|
|
11
|
+
function* cacheFirstSequence(rxNostr, pointers, opts, log) {
|
|
12
|
+
let remaining = [...pointers];
|
|
13
|
+
const id = nanoid(8);
|
|
14
|
+
log = log.extend(id);
|
|
15
|
+
const loaded = (packets) => {
|
|
16
|
+
const ids = new Set(packets.map((p) => p.event.id));
|
|
17
|
+
remaining = remaining.filter((p) => !ids.has(p.id));
|
|
18
|
+
};
|
|
19
|
+
if (opts?.cacheRequest) {
|
|
20
|
+
let filter = { ids: remaining.map((e) => e.id) };
|
|
21
|
+
const results = yield opts.cacheRequest([filter]).pipe(
|
|
22
|
+
// mark the event as from the cache
|
|
23
|
+
tap((event) => markFromCache(event)),
|
|
24
|
+
// convert to event packets
|
|
25
|
+
map((e) => ({ event: e, from: "", subId: "single-event-loader", type: "EVENT" })));
|
|
26
|
+
if (results.length > 0) {
|
|
27
|
+
log(`Loaded ${results.length} events from cache`);
|
|
28
|
+
loaded(results);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
// exit early if all pointers are loaded
|
|
32
|
+
if (remaining.length === 0)
|
|
33
|
+
return;
|
|
34
|
+
let byRelay = groupByRelay(remaining, "default");
|
|
35
|
+
// load remaining pointers from the relays
|
|
36
|
+
let results = yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
|
|
37
|
+
let filter = { ids: pointers.map((e) => e.id) };
|
|
38
|
+
let count = 0;
|
|
39
|
+
const req = createRxOneshotReq({ filters: [filter], rxReqId: id });
|
|
40
|
+
log(`Requesting from ${relay}`, filter.ids);
|
|
41
|
+
let sub$;
|
|
42
|
+
// don't specify relay if this is the "default" relay
|
|
43
|
+
if (relay === "default")
|
|
44
|
+
sub$ = rxNostr.use(req);
|
|
45
|
+
else
|
|
46
|
+
sub$ = rxNostr.use(req, { on: { relays: [relay] } });
|
|
47
|
+
return sub$.pipe(tap({
|
|
48
|
+
next: () => count++,
|
|
49
|
+
complete: () => log(`Completed ${relay}, loaded ${count} events`),
|
|
50
|
+
}));
|
|
51
|
+
})).pipe(mergeAll());
|
|
52
|
+
loaded(results);
|
|
53
|
+
if (remaining.length > 0) {
|
|
54
|
+
// failed to find remaining
|
|
55
|
+
log("Failed to load", remaining.map((p) => p.id));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
export class SingleEventLoader extends Loader {
|
|
59
|
+
log = logger.extend("SingleEventLoader");
|
|
60
|
+
constructor(rxNostr, opts) {
|
|
61
|
+
let options = opts || {};
|
|
62
|
+
super((source) => source.pipe(
|
|
63
|
+
// load first from cache
|
|
64
|
+
bufferTime(opts?.bufferTime ?? 1000),
|
|
65
|
+
// ignore empty buffers
|
|
66
|
+
filter((buffer) => buffer.length > 0),
|
|
67
|
+
// only request events from relays once
|
|
68
|
+
distinctRelaysBatch((p) => p.id, options.refreshTimeout ?? 60_000),
|
|
69
|
+
// ensure there is only one of each event pointer
|
|
70
|
+
map(consolidateEventPointers),
|
|
71
|
+
// run the loader sequence
|
|
72
|
+
generatorSequence((pointers) => cacheFirstSequence(rxNostr, pointers, options, this.log),
|
|
73
|
+
// there will always be more events, never complete
|
|
74
|
+
false)));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { EventPacket, RxNostr } from "rx-nostr";
|
|
2
|
+
import { logger } from "applesauce-core";
|
|
3
|
+
import { CacheRequest, Loader } from "./loader.js";
|
|
4
|
+
export type TabValuePointer = {
|
|
5
|
+
/** The value of the tag to load */
|
|
6
|
+
value: string;
|
|
7
|
+
/** The relays to load from */
|
|
8
|
+
relays?: string[];
|
|
9
|
+
/** bypass the cache */
|
|
10
|
+
force?: boolean;
|
|
11
|
+
};
|
|
12
|
+
export type TagValueLoaderOptions = {
|
|
13
|
+
/** the name of this loader (for debugging) */
|
|
14
|
+
name?: string;
|
|
15
|
+
/**
|
|
16
|
+
* Time interval to buffer requests in ms
|
|
17
|
+
* @default 1000
|
|
18
|
+
*/
|
|
19
|
+
bufferTime?: number;
|
|
20
|
+
/** Restrict queries to specific kinds */
|
|
21
|
+
kinds?: number[];
|
|
22
|
+
/** Restrict queries to specific authors */
|
|
23
|
+
authors?: string[];
|
|
24
|
+
/** Restrict queries since */
|
|
25
|
+
since?: number;
|
|
26
|
+
/** Method used to load from the cache */
|
|
27
|
+
cacheRequest?: CacheRequest;
|
|
28
|
+
};
|
|
29
|
+
export declare class TagValueLoader extends Loader<TabValuePointer, EventPacket> {
|
|
30
|
+
name: string;
|
|
31
|
+
protected log: typeof logger;
|
|
32
|
+
constructor(rxNostr: RxNostr, tagName: string, opts?: TagValueLoaderOptions);
|
|
33
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { createRxOneshotReq } from "rx-nostr";
|
|
2
|
+
import { bufferTime, filter, map, merge, mergeMap, tap } from "rxjs";
|
|
3
|
+
import { markFromCache } from "applesauce-core/helpers";
|
|
4
|
+
import { logger } from "applesauce-core";
|
|
5
|
+
import { Loader } from "./loader.js";
|
|
6
|
+
import { distinctRelaysBatch } from "../operators/distinct-relays.js";
|
|
7
|
+
import { getDefaultReadRelays } from "../helpers/rx-nostr.js";
|
|
8
|
+
import { unique } from "../helpers/array.js";
|
|
9
|
+
export class TagValueLoader extends Loader {
|
|
10
|
+
name;
|
|
11
|
+
log = logger.extend("TagValueLoader");
|
|
12
|
+
constructor(rxNostr, tagName, opts) {
|
|
13
|
+
const filterTag = `#${tagName}`;
|
|
14
|
+
super((source) => source.pipe(
|
|
15
|
+
// batch the pointers
|
|
16
|
+
bufferTime(opts?.bufferTime ?? 1000),
|
|
17
|
+
// filter out empty batches
|
|
18
|
+
filter((pointers) => pointers.length > 0),
|
|
19
|
+
// only request from each relay once
|
|
20
|
+
distinctRelaysBatch((m) => m.value),
|
|
21
|
+
// batch pointers into requests
|
|
22
|
+
mergeMap((pointers) => {
|
|
23
|
+
const baseFilter = {};
|
|
24
|
+
if (opts?.kinds)
|
|
25
|
+
baseFilter.kinds = opts.kinds;
|
|
26
|
+
if (opts?.since)
|
|
27
|
+
baseFilter.since = opts.since;
|
|
28
|
+
if (opts?.authors)
|
|
29
|
+
baseFilter.authors = opts.authors;
|
|
30
|
+
// build request map for relays
|
|
31
|
+
const requestMap = pointers.reduce((map, pointer) => {
|
|
32
|
+
const relays = pointer.relays ?? getDefaultReadRelays(rxNostr);
|
|
33
|
+
for (const relay of relays) {
|
|
34
|
+
if (!map[relay]) {
|
|
35
|
+
// create new filter for relay
|
|
36
|
+
const filter = { ...baseFilter, [filterTag]: [pointer.value] };
|
|
37
|
+
map[relay] = [filter];
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
// map for relay already exists, add the tag value
|
|
41
|
+
const filter = map[relay][0];
|
|
42
|
+
filter[filterTag].push(pointer.value);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return map;
|
|
46
|
+
}, {});
|
|
47
|
+
let fromCache = 0;
|
|
48
|
+
const cacheRequest = opts
|
|
49
|
+
?.cacheRequest?.([{ ...baseFilter, [filterTag]: unique(pointers.map((p) => p.value)) }])
|
|
50
|
+
.pipe(
|
|
51
|
+
// mark the event as from the cache
|
|
52
|
+
tap({
|
|
53
|
+
next: (event) => {
|
|
54
|
+
markFromCache(event);
|
|
55
|
+
fromCache++;
|
|
56
|
+
},
|
|
57
|
+
complete: () => {
|
|
58
|
+
if (fromCache > 0)
|
|
59
|
+
this.log(`Loaded ${fromCache} from cache`);
|
|
60
|
+
},
|
|
61
|
+
}),
|
|
62
|
+
// convert to event packets
|
|
63
|
+
map((e) => ({ event: e, from: "", subId: "replaceable-loader", type: "EVENT" })));
|
|
64
|
+
const requests = Object.entries(requestMap).map(([relay, filters]) => {
|
|
65
|
+
const req = createRxOneshotReq({ filters });
|
|
66
|
+
return rxNostr.use(req, { on: { relays: [relay] } });
|
|
67
|
+
});
|
|
68
|
+
this.log(`Requesting ${pointers.length} tag values from ${requests.length} relays`);
|
|
69
|
+
return cacheRequest ? merge(cacheRequest, ...requests) : merge(...requests);
|
|
70
|
+
})));
|
|
71
|
+
// create a unique logger for this instance
|
|
72
|
+
this.name = opts?.name ?? "";
|
|
73
|
+
this.log = this.log.extend(opts?.kinds ? `${this.name} ${filterTag} (${opts?.kinds?.join(",")})` : `${this.name} ${filterTag}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { EventPacket, RxNostr } from "rx-nostr";
|
|
2
|
+
import { BehaviorSubject } from "rxjs";
|
|
3
|
+
import { logger } from "applesauce-core";
|
|
4
|
+
import { RelayTimelineLoader, TimelessFilter } from "./relay-timeline-loader.js";
|
|
5
|
+
import { CacheRequest, Loader, RelayFilterMap } from "./loader.js";
|
|
6
|
+
import { CacheTimelineLoader } from "./cache-timeline-loader.js";
|
|
7
|
+
export type TimelineLoaderOptions = {
|
|
8
|
+
limit?: number;
|
|
9
|
+
cacheRequest?: CacheRequest;
|
|
10
|
+
};
|
|
11
|
+
/** A multi-relay timeline loader that can be used to load a timeline from multiple relays */
|
|
12
|
+
export declare class TimelineLoader extends Loader<number | undefined, EventPacket> {
|
|
13
|
+
id: string;
|
|
14
|
+
loading$: BehaviorSubject<boolean>;
|
|
15
|
+
get loading(): boolean;
|
|
16
|
+
requests: RelayFilterMap<TimelessFilter>;
|
|
17
|
+
protected log: typeof logger;
|
|
18
|
+
protected cache?: CacheTimelineLoader;
|
|
19
|
+
protected loaders: Map<string, RelayTimelineLoader>;
|
|
20
|
+
constructor(rxNostr: RxNostr, requests: RelayFilterMap<TimelessFilter>, opts?: TimelineLoaderOptions);
|
|
21
|
+
static simpleFilterMap(relays: string[], filters: TimelessFilter[]): RelayFilterMap<TimelessFilter>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { BehaviorSubject, combineLatest, connect, merge, tap } from "rxjs";
|
|
2
|
+
import { logger } from "applesauce-core";
|
|
3
|
+
import { mergeFilters } from "applesauce-core/helpers";
|
|
4
|
+
import { nanoid } from "nanoid";
|
|
5
|
+
import { RelayTimelineLoader } from "./relay-timeline-loader.js";
|
|
6
|
+
import { Loader } from "./loader.js";
|
|
7
|
+
import { CacheTimelineLoader } from "./cache-timeline-loader.js";
|
|
8
|
+
/** A multi-relay timeline loader that can be used to load a timeline from multiple relays */
|
|
9
|
+
export class TimelineLoader extends Loader {
|
|
10
|
+
id = nanoid(8);
|
|
11
|
+
loading$ = new BehaviorSubject(false);
|
|
12
|
+
get loading() {
|
|
13
|
+
return this.loading$.value;
|
|
14
|
+
}
|
|
15
|
+
requests;
|
|
16
|
+
log = logger.extend("TimelineLoader");
|
|
17
|
+
cache;
|
|
18
|
+
loaders;
|
|
19
|
+
constructor(rxNostr, requests, opts) {
|
|
20
|
+
const loaders = new Map();
|
|
21
|
+
// create cache loader
|
|
22
|
+
const cache = opts?.cacheRequest
|
|
23
|
+
? new CacheTimelineLoader(opts.cacheRequest, [mergeFilters(...Object.values(requests).flat())], opts)
|
|
24
|
+
: undefined;
|
|
25
|
+
// create loaders
|
|
26
|
+
for (const [relay, filters] of Object.entries(requests)) {
|
|
27
|
+
loaders.set(relay, new RelayTimelineLoader(rxNostr, relay, filters, opts));
|
|
28
|
+
}
|
|
29
|
+
const allLoaders = cache ? [cache, ...loaders.values()] : Array.from(loaders.values());
|
|
30
|
+
super((source) => {
|
|
31
|
+
// observable that triggers the loaders based on cursor
|
|
32
|
+
const trigger$ = source.pipe(tap((cursor) => {
|
|
33
|
+
for (const loader of allLoaders) {
|
|
34
|
+
// load the next page if cursor is past loader cursor
|
|
35
|
+
if (!cursor || !Number.isFinite(cursor) || cursor <= loader.cursor)
|
|
36
|
+
loader.next();
|
|
37
|
+
}
|
|
38
|
+
}));
|
|
39
|
+
// observable that handles updating the loading state
|
|
40
|
+
const loading$ = combineLatest(allLoaders.map((l) => l.loading$)).pipe(
|
|
41
|
+
// set loading to true as long as one loader is still loading
|
|
42
|
+
tap((loading) => this.loading$.next(loading.some((v) => v === true))));
|
|
43
|
+
// observable that merges all the outputs of the loaders
|
|
44
|
+
const events$ = merge(...allLoaders.map((l) => l.observable));
|
|
45
|
+
// subscribe to all observables but only return the results of events$
|
|
46
|
+
return merge(trigger$, loading$, events$).pipe(connect((_shared$) => events$));
|
|
47
|
+
});
|
|
48
|
+
this.requests = requests;
|
|
49
|
+
this.cache = cache;
|
|
50
|
+
this.loaders = loaders;
|
|
51
|
+
this.log = this.log.extend(this.id);
|
|
52
|
+
}
|
|
53
|
+
static simpleFilterMap(relays, filters) {
|
|
54
|
+
return relays.reduce((map, relay) => ({ ...map, [relay]: filters }), {});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { EventPacket, RxNostr } from "rx-nostr";
|
|
2
|
+
import { logger } from "applesauce-core";
|
|
3
|
+
import { CacheRequest, Loader } from "./loader.js";
|
|
4
|
+
export type LoadableSetPointer = {
|
|
5
|
+
/** A replaceable kind >= 30000 & < 40000 */
|
|
6
|
+
kind: number;
|
|
7
|
+
pubkey: string;
|
|
8
|
+
/** Relays to load from */
|
|
9
|
+
relays?: string[];
|
|
10
|
+
/** Load the sets even if it has already been loaded */
|
|
11
|
+
force?: boolean;
|
|
12
|
+
};
|
|
13
|
+
export type UserSetsLoaderOptions = {
|
|
14
|
+
/**
|
|
15
|
+
* Time interval to buffer requests in ms
|
|
16
|
+
* @default 1000
|
|
17
|
+
*/
|
|
18
|
+
bufferTime?: number;
|
|
19
|
+
/** A method used to load events from a local cache */
|
|
20
|
+
cacheRequest?: CacheRequest;
|
|
21
|
+
/**
|
|
22
|
+
* How long the loader should wait before it allows an event pointer to be refreshed from a relay
|
|
23
|
+
* @default 120000
|
|
24
|
+
*/
|
|
25
|
+
refreshTimeout?: number;
|
|
26
|
+
};
|
|
27
|
+
/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
|
|
28
|
+
export declare class UserSetsLoader extends Loader<LoadableSetPointer, EventPacket> {
|
|
29
|
+
log: typeof logger;
|
|
30
|
+
constructor(rxNostr: RxNostr, opts?: UserSetsLoaderOptions);
|
|
31
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { tap, from, filter, map, mergeAll, bufferTime } from "rxjs";
|
|
2
|
+
import { createRxOneshotReq } from "rx-nostr";
|
|
3
|
+
import { markFromCache } from "applesauce-core/helpers";
|
|
4
|
+
import { logger } from "applesauce-core";
|
|
5
|
+
import { nanoid } from "nanoid";
|
|
6
|
+
import { Loader } from "./loader.js";
|
|
7
|
+
import { generatorSequence } from "../operators/generator-sequence.js";
|
|
8
|
+
import { consolidateAddressPointers, createFiltersFromAddressPointers } from "../helpers/address-pointer.js";
|
|
9
|
+
import { groupByRelay } from "../helpers/pointer.js";
|
|
10
|
+
import { distinctRelaysBatch } from "../operators/distinct-relays.js";
|
|
11
|
+
/** A generator that tries to load the address pointers from the cache first, then tries the relays */
|
|
12
|
+
function* cacheFirstSequence(rxNostr, pointers, log, opts) {
|
|
13
|
+
const id = nanoid(8);
|
|
14
|
+
log = log.extend(id);
|
|
15
|
+
// first attempt, load from cache relays
|
|
16
|
+
if (opts?.cacheRequest) {
|
|
17
|
+
log(`Checking cache`);
|
|
18
|
+
const filters = createFiltersFromAddressPointers(pointers);
|
|
19
|
+
const results = yield opts.cacheRequest(filters).pipe(
|
|
20
|
+
// mark the event as from the cache
|
|
21
|
+
tap((event) => markFromCache(event)),
|
|
22
|
+
// convert to event packets
|
|
23
|
+
map((e) => ({ event: e, from: "", subId: "user-sets-loader", type: "EVENT" })));
|
|
24
|
+
if (results.length > 0) {
|
|
25
|
+
log(`Loaded ${results.length} events from cache`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
let byRelay = groupByRelay(pointers, "default");
|
|
29
|
+
// load sets from relays
|
|
30
|
+
yield from(Array.from(byRelay.entries()).map(([relay, pointers]) => {
|
|
31
|
+
let filters = createFiltersFromAddressPointers(pointers);
|
|
32
|
+
let count = 0;
|
|
33
|
+
const req = createRxOneshotReq({ filters, rxReqId: id });
|
|
34
|
+
log(`Requesting from ${relay}`, pointers);
|
|
35
|
+
let sub$;
|
|
36
|
+
// don't specify relay if this is the "default" relay
|
|
37
|
+
if (relay === "default")
|
|
38
|
+
sub$ = rxNostr.use(req);
|
|
39
|
+
else
|
|
40
|
+
sub$ = rxNostr.use(req, { on: { relays: [relay] } });
|
|
41
|
+
return sub$.pipe(tap({
|
|
42
|
+
next: () => count++,
|
|
43
|
+
complete: () => log(`Completed ${relay}, loaded ${count} events`),
|
|
44
|
+
}));
|
|
45
|
+
})).pipe(mergeAll());
|
|
46
|
+
}
|
|
47
|
+
/** A loader that can be used to load users NIP-51 sets events ( kind >= 30000 < 40000) */
|
|
48
|
+
export class UserSetsLoader extends Loader {
|
|
49
|
+
log = logger.extend("UserSetsLoader");
|
|
50
|
+
constructor(rxNostr, opts) {
|
|
51
|
+
let options = opts || {};
|
|
52
|
+
super((source) => source.pipe(
|
|
53
|
+
// load first from cache
|
|
54
|
+
bufferTime(options?.bufferTime ?? 1000),
|
|
55
|
+
// ignore empty buffers
|
|
56
|
+
filter((buffer) => buffer.length > 0),
|
|
57
|
+
// only load from each relay once
|
|
58
|
+
distinctRelaysBatch((p) => p.kind + ":" + p.pubkey, options.refreshTimeout ?? 120_000),
|
|
59
|
+
// deduplicate address pointers
|
|
60
|
+
map(consolidateAddressPointers),
|
|
61
|
+
// check cache, relays, lookup relays in that order
|
|
62
|
+
generatorSequence((pointers) => cacheFirstSequence(rxNostr, pointers, this.log, options),
|
|
63
|
+
// there will always be more events, never complete
|
|
64
|
+
false)));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { Subject } from "rxjs";
|
|
3
|
+
import { distinctRelays } from "../distinct-relays.js";
|
|
4
|
+
describe("distinctRelays", () => {
|
|
5
|
+
it("should filter out messages with same relay within timeout window", () => {
|
|
6
|
+
const fn = vi.fn();
|
|
7
|
+
const source$ = new Subject();
|
|
8
|
+
source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
|
|
9
|
+
const message = {
|
|
10
|
+
id: "123",
|
|
11
|
+
relays: ["relay1", "relay2"],
|
|
12
|
+
};
|
|
13
|
+
// Send message with two relays
|
|
14
|
+
source$.next(message);
|
|
15
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
16
|
+
expect(fn).toHaveBeenCalledWith(message);
|
|
17
|
+
// send message again
|
|
18
|
+
source$.next({ ...message });
|
|
19
|
+
// should not call again
|
|
20
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
21
|
+
});
|
|
22
|
+
it("should only remove duplicate relays in timeout window", () => {
|
|
23
|
+
const fn = vi.fn();
|
|
24
|
+
const source$ = new Subject();
|
|
25
|
+
source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
|
|
26
|
+
const message = {
|
|
27
|
+
id: "123",
|
|
28
|
+
relays: ["relay1", "relay2"],
|
|
29
|
+
};
|
|
30
|
+
// Send message with two relays
|
|
31
|
+
source$.next(message);
|
|
32
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
33
|
+
expect(fn).toHaveBeenCalledWith(message);
|
|
34
|
+
// send message again
|
|
35
|
+
source$.next({ id: "123", relays: ["relay1", "relay3"] });
|
|
36
|
+
// should not call again
|
|
37
|
+
expect(fn).toHaveBeenCalledTimes(2);
|
|
38
|
+
expect(fn).toHaveBeenCalledWith({ id: "123", relays: ["relay3"] });
|
|
39
|
+
});
|
|
40
|
+
it("should filter out duplicate messages without relays in timeout", () => {
|
|
41
|
+
const fn = vi.fn();
|
|
42
|
+
const source$ = new Subject();
|
|
43
|
+
source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
|
|
44
|
+
const message = { id: "123" };
|
|
45
|
+
// Send message with two relays
|
|
46
|
+
source$.next(message);
|
|
47
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
48
|
+
expect(fn).toHaveBeenCalledWith(message);
|
|
49
|
+
// send message again
|
|
50
|
+
source$.next({ ...message });
|
|
51
|
+
// should not call again
|
|
52
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
53
|
+
});
|
|
54
|
+
it("should treat messages with relays severalty then messages without", () => {
|
|
55
|
+
const fn = vi.fn();
|
|
56
|
+
const source$ = new Subject();
|
|
57
|
+
source$.pipe(distinctRelays((msg) => msg.id, 1000)).subscribe(fn);
|
|
58
|
+
const withRelays = {
|
|
59
|
+
id: "123",
|
|
60
|
+
relays: ["relay1", "relay2"],
|
|
61
|
+
};
|
|
62
|
+
const withoutRelays = {
|
|
63
|
+
id: "123",
|
|
64
|
+
};
|
|
65
|
+
// Send message with two relays
|
|
66
|
+
source$.next(withoutRelays);
|
|
67
|
+
expect(fn).toHaveBeenCalledTimes(1);
|
|
68
|
+
expect(fn).toHaveBeenCalledWith(withoutRelays);
|
|
69
|
+
// send message with relays
|
|
70
|
+
source$.next(withRelays);
|
|
71
|
+
// should not call again
|
|
72
|
+
expect(fn).toHaveBeenCalledTimes(2);
|
|
73
|
+
expect(fn).toHaveBeenCalledWith(withRelays);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { expect, it } from "vitest";
|
|
2
|
+
import { lastValueFrom, of, toArray } from "rxjs";
|
|
3
|
+
import { TestScheduler } from "rxjs/testing";
|
|
4
|
+
import { generatorSequence } from "../generator-sequence.js";
|
|
5
|
+
let testScheduler = new TestScheduler((actual, expected) => {
|
|
6
|
+
expect(actual).toEqual(expected);
|
|
7
|
+
});
|
|
8
|
+
it("should work with normal generator functions", () => {
|
|
9
|
+
testScheduler.run(({ expectObservable }) => {
|
|
10
|
+
function* normalGenerator(value) {
|
|
11
|
+
yield of(value + 1);
|
|
12
|
+
yield of(value + 2);
|
|
13
|
+
yield of(value + 3);
|
|
14
|
+
yield value + 4;
|
|
15
|
+
}
|
|
16
|
+
const source$ = of(1).pipe(generatorSequence(normalGenerator));
|
|
17
|
+
// Define expected marble diagram
|
|
18
|
+
const expectedMarble = "(abcd|)";
|
|
19
|
+
const expectedValues = {
|
|
20
|
+
a: 2,
|
|
21
|
+
b: 3,
|
|
22
|
+
c: 4,
|
|
23
|
+
d: 5,
|
|
24
|
+
};
|
|
25
|
+
expectObservable(source$).toBe(expectedMarble, expectedValues);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
28
|
+
it("should work with async generator functions", async () => {
|
|
29
|
+
async function* asyncGenerator(value) {
|
|
30
|
+
yield of(`${value}-1`);
|
|
31
|
+
yield of(`${value}-2`);
|
|
32
|
+
yield of(`${value}-3`);
|
|
33
|
+
yield `${value}-4`;
|
|
34
|
+
}
|
|
35
|
+
const source$ = of("test").pipe(generatorSequence(asyncGenerator));
|
|
36
|
+
const expectedValues = ["test-1", "test-2", "test-3", "test-4"];
|
|
37
|
+
expect(await lastValueFrom(source$.pipe(toArray()))).toEqual(expectedValues);
|
|
38
|
+
});
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { OperatorFunction } from "rxjs";
|
|
2
|
+
import { MessageWithRelay } from "../helpers/pointer.js";
|
|
3
|
+
export declare function distinctRelays<T extends MessageWithRelay>(keyFn: (message: T) => string, timeout?: number): OperatorFunction<T, T>;
|
|
4
|
+
export declare function distinctRelaysBatch<T extends MessageWithRelay>(keyFn: (message: T) => string, timeout?: number): OperatorFunction<T[], T[]>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { filter, map } from "rxjs";
|
|
2
|
+
import { removePreviouslyUsedRelays } from "../helpers/pointer.js";
|
|
3
|
+
export function distinctRelays(keyFn, timeout = 60_000) {
|
|
4
|
+
return (source$) => {
|
|
5
|
+
const cache = new Map();
|
|
6
|
+
return source$.pipe(map((message) => removePreviouslyUsedRelays(message, keyFn, cache, timeout)), filter((message) => message !== null));
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
export function distinctRelaysBatch(keyFn, timeout = 60_000) {
|
|
10
|
+
return (source$) => {
|
|
11
|
+
const cache = new Map();
|
|
12
|
+
return source$.pipe(map((batch) => batch.map((m) => removePreviouslyUsedRelays(m, keyFn, cache, timeout)).filter((m) => m !== null)));
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { LRU } from "applesauce-core/helpers";
|
|
2
|
+
import { filter } from "rxjs";
|
|
3
|
+
/** Filters out duplicate values based on a key getter and a TTL */
|
|
4
|
+
export function distinctTimeout(keyFn, ttl = 1000) {
|
|
5
|
+
const seen = new LRU(undefined, ttl);
|
|
6
|
+
return (source) => source.pipe(filter((value) => {
|
|
7
|
+
const key = keyFn(value);
|
|
8
|
+
if (seen.has(key))
|
|
9
|
+
return false;
|
|
10
|
+
else {
|
|
11
|
+
seen.set(key, Date.now());
|
|
12
|
+
return true;
|
|
13
|
+
}
|
|
14
|
+
}));
|
|
15
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { Observable, OperatorFunction } from "rxjs";
|
|
2
2
|
/** Keeps retrying a value until the generator returns */
|
|
3
|
-
export declare function generatorSequence<
|
|
3
|
+
export declare function generatorSequence<Input, Result>(createGenerator: (value: Input) => Generator<Observable<Result> | Result, void, Result[] | undefined> | AsyncGenerator<Observable<Result> | Result, void, Result[] | undefined>, shouldComplete?: boolean): OperatorFunction<Input, Result>;
|
|
@@ -1,45 +1,53 @@
|
|
|
1
|
-
import { Observable } from "rxjs";
|
|
1
|
+
import { isObservable, Observable } from "rxjs";
|
|
2
2
|
/** Keeps retrying a value until the generator returns */
|
|
3
|
-
export function generatorSequence(createGenerator) {
|
|
3
|
+
export function generatorSequence(createGenerator, shouldComplete = true) {
|
|
4
4
|
return (source) => {
|
|
5
5
|
return new Observable((observer) => {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
const
|
|
11
|
-
const result = generator.next(prevResults);
|
|
6
|
+
return source.subscribe((value) => {
|
|
7
|
+
const generator = createGenerator(value);
|
|
8
|
+
const nextSequence = (prevResults) => {
|
|
9
|
+
const p = generator.next(prevResults);
|
|
10
|
+
const handleResult = (result) => {
|
|
12
11
|
// generator complete, exit
|
|
13
|
-
if (result.done)
|
|
12
|
+
if (result.done) {
|
|
13
|
+
if (shouldComplete)
|
|
14
|
+
observer.complete();
|
|
14
15
|
return;
|
|
16
|
+
}
|
|
15
17
|
const results = [];
|
|
16
|
-
result.value
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
18
|
+
if (isObservable(result.value)) {
|
|
19
|
+
result.value.subscribe({
|
|
20
|
+
next: (v) => {
|
|
21
|
+
// track results and pass along values
|
|
22
|
+
results.push(v);
|
|
23
|
+
observer.next(v);
|
|
24
|
+
},
|
|
25
|
+
error: (err) => {
|
|
26
|
+
observer.error(err);
|
|
27
|
+
},
|
|
28
|
+
complete: () => {
|
|
29
|
+
// run next step
|
|
30
|
+
nextSequence(results);
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
results.push(result.value);
|
|
36
|
+
observer.next(result.value);
|
|
37
|
+
nextSequence(results);
|
|
38
|
+
}
|
|
33
39
|
};
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
// if its an async generator, wait for the promise
|
|
41
|
+
if (p instanceof Promise)
|
|
42
|
+
p.then(handleResult, (err) => {
|
|
43
|
+
observer.error(err);
|
|
44
|
+
});
|
|
45
|
+
else
|
|
46
|
+
handleResult(p);
|
|
47
|
+
};
|
|
48
|
+
// start running steps
|
|
49
|
+
nextSequence();
|
|
41
50
|
});
|
|
42
|
-
return () => sub.unsubscribe();
|
|
43
51
|
});
|
|
44
52
|
};
|
|
45
53
|
}
|
package/dist/operators/index.js
CHANGED