@recursiv/sdk 0.7.63 → 0.7.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/feed-definition.d.ts +5 -1
- package/dist/feed-definition.js +5 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +6 -1
- package/dist/resources/curator.d.ts +163 -1
- package/dist/resources/curator.js +9 -0
- package/dist/resources/discovery.d.ts +159 -5
- package/dist/resources/discovery.js +360 -30
- package/dist/resources/feeds.d.ts +228 -0
- package/dist/resources/feeds.js +61 -0
- package/dist/resources/profiles.d.ts +4 -1
- package/dist/resources/profiles.js +4 -1
- package/dist/resources/protocols.d.ts +4 -5
- package/dist/resources/search.d.ts +3 -2
- package/dist/types.d.ts +3 -2
- package/llm.md +25 -0
- package/package.json +1 -1
|
@@ -42,6 +42,26 @@ export const DISCOVERY_SOURCE_TIMEOUT_MS = 15_000;
|
|
|
42
42
|
* "Minds posts didn't load" with Try again and which its Politics fallback (minds#1212) waits for.
|
|
43
43
|
*/
|
|
44
44
|
export const DISCOVERY_SOURCE_RETRIES = 2;
|
|
45
|
+
/**
|
|
46
|
+
* The longest the top of the feed holds the first page, counted from the page's start. The top's read and the reads of
|
|
47
|
+
* the posts its items need (Minds posts by id, a story's network posts) share this one deadline: a top that has not
|
|
48
|
+
* answered by then is left out, and an item whose post was not read by then shows only if the first hot page carries it.
|
|
49
|
+
*/
|
|
50
|
+
export const DISCOVERY_TOP_TIMEOUT_MS = 1500;
|
|
51
|
+
/** What the top's deadline answers when it passes first. */
|
|
52
|
+
const TOP_LATE = Symbol('top of the feed late');
|
|
53
|
+
/** Top items a session asks for unless its options say otherwise: the top block. */
|
|
54
|
+
export const DISCOVERY_TOP_LIMIT = 10;
|
|
55
|
+
/** Stories whose network posts a top's items are read from, at most. */
|
|
56
|
+
const TOP_MAX_STORY_READS = 3;
|
|
57
|
+
/** A hot page the seen set (or the top) leaves under this many posts walks on to the next hot page... */
|
|
58
|
+
export const DISCOVERY_SEEN_MIN_ITEMS = 5;
|
|
59
|
+
/** ...up to this many more pages in one read; then the page is what it is and the next read goes on from there. */
|
|
60
|
+
export const DISCOVERY_SEEN_WALK = 3;
|
|
61
|
+
/** A story whose lead was on screen this many times with no open is skipped (DiscoveryFeedSessionOptions.seenClusters). */
|
|
62
|
+
export const RADAR_REST_AFTER = 3;
|
|
63
|
+
/** A visitor's page-0 story rotates by bucket among this many best breaking stories (or best stories when none is breaking). */
|
|
64
|
+
export const RADAR_BUCKET_ROTATION = 5;
|
|
45
65
|
/** The page deadline a source's read ran past. */
|
|
46
66
|
class SourceTimedOut extends Error {
|
|
47
67
|
constructor() { super('Discovery source timed out'); this.name = 'SourceTimedOut'; }
|
|
@@ -168,6 +188,103 @@ function radarCluster(value) {
|
|
|
168
188
|
return { id: c.id, label: c.label.slice(0, 300), kind: c.kind, ...(theme ? { theme } : {}), outletCount: mainstream + independent, laneMix: { mainstream, independent }, sources,
|
|
169
189
|
velocity: { normalized: typeof normalized === 'number' && Number.isFinite(normalized) && normalized >= 0 ? normalized : 0 }, newestAt, ...(lead ? { lead } : {}), links, memberKeys };
|
|
170
190
|
}
|
|
191
|
+
/**
|
|
192
|
+
* The top's lead story as a cluster, for a page whose radar snapshot does not hold it (the radar read failed or timed
|
|
193
|
+
* out): its label, kind, theme, sources and freshness, with no links or member posts (the app opens the story by id).
|
|
194
|
+
*/
|
|
195
|
+
function storyCluster(value) {
|
|
196
|
+
if (!value || typeof value !== 'object')
|
|
197
|
+
return null;
|
|
198
|
+
const s = value;
|
|
199
|
+
if (typeof s.id !== 'string' || !/^[0-9a-f]{64}$/.test(s.id) || !nonEmpty(s.label) || typeof s.kind !== 'string' || !RADAR_KINDS.has(s.kind))
|
|
200
|
+
return null;
|
|
201
|
+
const newestAt = (s.freshness ?? {}).newestAt;
|
|
202
|
+
if (typeof newestAt !== 'string' || !Number.isFinite(Date.parse(newestAt)))
|
|
203
|
+
return null;
|
|
204
|
+
const sources = {};
|
|
205
|
+
if (s.sources && typeof s.sources === 'object')
|
|
206
|
+
for (const [name, count] of Object.entries(s.sources)) {
|
|
207
|
+
const n = countOf(count);
|
|
208
|
+
if (n !== null && /^[a-z]{1,32}$/.test(name))
|
|
209
|
+
sources[name] = n;
|
|
210
|
+
}
|
|
211
|
+
const theme = typeof s.theme === 'string' && /^[a-z][a-z0-9-]{0,31}$/.test(s.theme) ? s.theme : undefined;
|
|
212
|
+
return { id: s.id, label: s.label.slice(0, 300), kind: s.kind, ...(theme ? { theme } : {}), outletCount: sources.outlet ?? 0,
|
|
213
|
+
laneMix: { mainstream: 0, independent: 0 }, sources, velocity: { normalized: 0 }, newestAt, links: [], memberKeys: [] };
|
|
214
|
+
}
|
|
215
|
+
/** Page 0: the top breaking story, else the top story. Later pages: the next unused story that is not breaking. */
|
|
216
|
+
export function pickRadarCluster(clusters, page, options = {}) {
|
|
217
|
+
const rested = (cluster) => { const seen = options.seenClusters?.get(cluster.id); return !!seen && !seen.clicked && seen.n >= RADAR_REST_AFTER; };
|
|
218
|
+
const unused = clusters.filter(candidate => !options.used?.has(candidate.id) && !rested(candidate));
|
|
219
|
+
if (page !== 0)
|
|
220
|
+
return unused.find(candidate => candidate.kind !== 'breaking');
|
|
221
|
+
const breaking = unused.filter(candidate => candidate.kind === 'breaking');
|
|
222
|
+
if (options.bucket !== undefined && Number.isSafeInteger(options.bucket)) {
|
|
223
|
+
const turn = (breaking.length ? breaking : unused).slice(0, RADAR_BUCKET_ROTATION);
|
|
224
|
+
if (turn.length)
|
|
225
|
+
return turn[((options.bucket % turn.length) + turn.length) % turn.length];
|
|
226
|
+
}
|
|
227
|
+
return breaking[0] ?? unused[0];
|
|
228
|
+
}
|
|
229
|
+
const TOP_NETWORKS = new Set(['bluesky', 'activitypub', 'nostr']);
|
|
230
|
+
const idString = (value) => typeof value === 'string' && value.length > 0 && value.length <= 4096;
|
|
231
|
+
const counted = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
|
|
232
|
+
/** The refs an app's seen set may hold for a post: its id, or '<protocol>/<nativeId>' (an ActivityPub id also without https://). */
|
|
233
|
+
function seenRefsOf(protocol, id) {
|
|
234
|
+
if (protocol === 'minds')
|
|
235
|
+
return [id];
|
|
236
|
+
const ref = `${protocol}/${id}`;
|
|
237
|
+
return protocol === 'activitypub' && /^https:\/\//i.test(id) ? [ref, `${protocol}/${id.replace(/^https:\/\//i, '')}`] : [ref];
|
|
238
|
+
}
|
|
239
|
+
function topRow(value) {
|
|
240
|
+
if (!value || typeof value !== 'object')
|
|
241
|
+
return null;
|
|
242
|
+
const item = value;
|
|
243
|
+
const target = (item.target ?? {});
|
|
244
|
+
if (!idString(item.ref))
|
|
245
|
+
return null;
|
|
246
|
+
const explain = (item.explain ?? {});
|
|
247
|
+
const cluster = typeof explain.cluster === 'string' && /^[0-9a-f]{64}$/.test(explain.cluster) ? explain.cluster : null;
|
|
248
|
+
if (item.kind === 'post' && target.kind === 'post' && nonEmpty(target.id))
|
|
249
|
+
return { ref: item.ref, key: key('minds', target.id), kind: 'minds', id: target.id, cluster, seenRefs: seenRefsOf('minds', target.id) };
|
|
250
|
+
if (item.kind === 'network_post' && target.kind === 'network_post' && typeof target.protocol === 'string' && TOP_NETWORKS.has(target.protocol) && idString(target.nativeId)) {
|
|
251
|
+
return { ref: item.ref, key: key(target.protocol, target.nativeId), kind: 'social', id: target.nativeId, cluster, seenRefs: seenRefsOf(target.protocol, target.nativeId) };
|
|
252
|
+
}
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* One open-network lane entry as a post the session can show: the card's text, author, media and counts. The card
|
|
257
|
+
* carries no URL on the post's network, so neither does the post (`sourceUrl` and the author's `profileUrl` are empty):
|
|
258
|
+
* the app opens it by its ref, in the app.
|
|
259
|
+
*/
|
|
260
|
+
function laneItem(value, protocol) {
|
|
261
|
+
if (!value || typeof value !== 'object')
|
|
262
|
+
return null;
|
|
263
|
+
const entry = value;
|
|
264
|
+
const target = (entry.target ?? {});
|
|
265
|
+
const card = (entry.card ?? null);
|
|
266
|
+
const author = (card?.author ?? null);
|
|
267
|
+
if (entry.protocol !== protocol || !idString(entry.ref) || target.kind !== 'network_post' || target.protocol !== protocol || !idString(target.nativeId))
|
|
268
|
+
return null;
|
|
269
|
+
if (!card || typeof card.text !== 'string' || typeof card.createdAt !== 'string' || !author || !idString(author.nativeId) || !nonEmpty(author.handle))
|
|
270
|
+
return null;
|
|
271
|
+
const counts = (card.counts ?? {});
|
|
272
|
+
const sensitive = card.sensitive === true;
|
|
273
|
+
const media = [];
|
|
274
|
+
for (const raw of Array.isArray(card.media) ? card.media.slice(0, 8) : []) {
|
|
275
|
+
const m = (raw ?? {});
|
|
276
|
+
if ((m.type !== 'image' && m.type !== 'video') || !httpUrl(m.url))
|
|
277
|
+
continue;
|
|
278
|
+
media.push({ type: m.type, url: m.url, alt: typeof m.alt === 'string' ? m.alt : '', ...(httpUrl(m.poster) ? { poster: m.poster } : {}),
|
|
279
|
+
...(dimension(m.width) && dimension(m.height) ? { width: dimension(m.width), height: dimension(m.height) } : {}), ...(sensitive ? { sensitive: true } : {}) });
|
|
280
|
+
}
|
|
281
|
+
const post = { protocol, nativeId: target.nativeId, sourceUrl: '', ...(nonEmpty(target.readRef) ? { readRef: target.readRef } : {}),
|
|
282
|
+
author: { protocol, nativeId: author.nativeId, handle: author.handle, displayName: nonEmpty(author.displayName) ? author.displayName : author.handle, description: '',
|
|
283
|
+
profileUrl: '', avatarUrl: httpUrl(author.avatarUrl) ? author.avatarUrl : null, followersCount: null, followingCount: null, postsCount: null },
|
|
284
|
+
text: card.text, createdAt: card.createdAt, replyCount: counted(counts.replies), likeCount: counted(counts.likes), repostCount: counted(counts.reposts),
|
|
285
|
+
media, hasAdditionalContent: card.hasQuote === true, ...(sensitive ? { sensitive: true } : {}) };
|
|
286
|
+
return { key: key(protocol, target.nativeId), kind: 'social', post, top: { ref: entry.ref, position: 0, block: 'top', lane: protocol } };
|
|
287
|
+
}
|
|
171
288
|
const FALLOVER_VIA = new Set(['personal', 'popular', 'static']);
|
|
172
289
|
const filled = (value) => typeof value === 'string' && value.length > 0;
|
|
173
290
|
/**
|
|
@@ -234,7 +351,17 @@ export class DiscoveryFeedSession {
|
|
|
234
351
|
radarLoading = null;
|
|
235
352
|
/** Member reads per story, started as soon as the story is known and shared by whichever page shows it. */
|
|
236
353
|
memberReads = new Map();
|
|
237
|
-
|
|
354
|
+
/** The top of the feed (options.top with a reader), its read started with page 0's reads, and the keys it placed. */
|
|
355
|
+
top;
|
|
356
|
+
topLoading = null;
|
|
357
|
+
/** The story the top leads with, once its answer is in (null: it names none); undefined until then. */
|
|
358
|
+
topLead;
|
|
359
|
+
topKeys = new Set();
|
|
360
|
+
/** What the member already saw (options.seen): left out of the Minds hot pages. */
|
|
361
|
+
appSeen = null;
|
|
362
|
+
seenClusters;
|
|
363
|
+
bucket;
|
|
364
|
+
constructor(posts, protocols, networks, category, source, learning, recommend, radar, members, options = {}) {
|
|
238
365
|
this.posts = posts;
|
|
239
366
|
this.protocols = protocols;
|
|
240
367
|
this.category = category;
|
|
@@ -247,6 +374,19 @@ export class DiscoveryFeedSession {
|
|
|
247
374
|
const sources = [...new Set(networks.filter(n => n.capabilities.includes('public_trends')).map(n => n.protocol))].slice(0, 16);
|
|
248
375
|
this.streams = ['minds', ...sources].filter(protocol => !source || source === 'all' || source === protocol)
|
|
249
376
|
.map(protocol => ({ protocol, offset: 0, done: false, failed: false, used: [] }));
|
|
377
|
+
const { top, seen, seenClusters, bucket } = options;
|
|
378
|
+
if (top && typeof top.read === 'function' && nonEmpty(top.ref))
|
|
379
|
+
this.top = { ...top, read: top.read };
|
|
380
|
+
if (seen)
|
|
381
|
+
this.appSeen = new Set([...seen].filter((ref) => typeof ref === 'string').slice(0, 10_000));
|
|
382
|
+
this.seenClusters = seenClusters;
|
|
383
|
+
this.bucket = bucket;
|
|
384
|
+
}
|
|
385
|
+
/** The Minds hot pages leave out what the member saw or the top placed, and walk on past a page that leaves too little. */
|
|
386
|
+
get filtering() { return !!this.appSeen || this.topKeys.size > 0; }
|
|
387
|
+
/** A Minds hot item the page leaves out: shown on an earlier page, placed by the top, or in the member's seen set. */
|
|
388
|
+
hiddenFromHot(item) {
|
|
389
|
+
return this.seen.has(item.key) || this.topKeys.has(item.key) || (item.kind === 'minds' && !!this.appSeen?.has(item.post.id));
|
|
250
390
|
}
|
|
251
391
|
retryFailed() { for (const stream of this.streams) {
|
|
252
392
|
stream.failed = false;
|
|
@@ -272,20 +412,44 @@ export class DiscoveryFeedSession {
|
|
|
272
412
|
// opt-out rides in the params (ListPostsParams.retry), which a reader handed to a session forwards to read the
|
|
273
413
|
// right page at all: a second argument was dropped by the Minds app's own wrappers (lib/discoverFeedResource,
|
|
274
414
|
// lib/compassStatus withCompass, lib/watchSource), which pass the params alone (review of #4047).
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
stream.
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
415
|
+
// With a seen set or a top, a hot page leaves out what the member saw or the top placed; one left under
|
|
416
|
+
// DISCOVERY_SEEN_MIN_ITEMS walks on to the next hot page, up to DISCOVERY_SEEN_WALK more, so the feed never looks
|
|
417
|
+
// ended while the ranking goes on. Each kept post keeps its own window's layout. Without either, one page as before.
|
|
418
|
+
const kept = [];
|
|
419
|
+
const layouts = [];
|
|
420
|
+
let head;
|
|
421
|
+
let partial = false;
|
|
422
|
+
let filtered = false;
|
|
423
|
+
for (let walked = 0;; walked++) {
|
|
424
|
+
const response = await this.posts.list({ ...params, sort: 'hot', ...(stream.cursor ? { cursor: stream.cursor } : { offset: stream.offset }), retry: false });
|
|
425
|
+
stream.offset += response.meta.scanned ?? response.meta.limit;
|
|
426
|
+
const next = response.meta.next_cursor ?? undefined;
|
|
427
|
+
stream.done = !response.meta.has_more || !!next && (next === stream.cursor || stream.used.includes(next));
|
|
428
|
+
if (stream.cursor)
|
|
429
|
+
stream.used = [...stream.used, stream.cursor].slice(-32);
|
|
430
|
+
stream.cursor = next;
|
|
431
|
+
// A category feed that ended a stage says where to go next; kept on the stream and reported on every page from here.
|
|
432
|
+
const fallover = readFallover(response.meta.fallover);
|
|
433
|
+
if (fallover)
|
|
434
|
+
stream.fallover = fallover;
|
|
435
|
+
const items = response.data.map(post => ({ key: key('minds', post.id), kind: 'minds', post }));
|
|
436
|
+
const read = { layout: response.meta.discovery ?? ordinary, partial: response.meta.category_filter?.status === 'partial',
|
|
437
|
+
...(response.meta.compass_filter ? { compass: response.meta.compass_filter } : {}) };
|
|
438
|
+
if (!this.filtering)
|
|
439
|
+
return { items, ...read };
|
|
440
|
+
head ??= read;
|
|
441
|
+
partial ||= read.partial;
|
|
442
|
+
items.forEach((item, index) => {
|
|
443
|
+
if (this.hiddenFromHot(item) || kept.some(row => row.key === item.key)) {
|
|
444
|
+
filtered = true;
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
kept.push(item);
|
|
448
|
+
layouts.push(windowLayout(read.layout, index));
|
|
449
|
+
});
|
|
450
|
+
if (kept.length >= DISCOVERY_SEEN_MIN_ITEMS || !filtered || stream.done || walked >= DISCOVERY_SEEN_WALK)
|
|
451
|
+
return { items: kept, ...head, partial, layouts };
|
|
452
|
+
}
|
|
289
453
|
}
|
|
290
454
|
const validatePosts = (data) => {
|
|
291
455
|
if (data.coverage !== 'provider_ranked_window' || data.items.length > 25 || data.items.some(post => post.protocol !== stream.protocol || post.author.protocol !== stream.protocol))
|
|
@@ -308,7 +472,7 @@ export class DiscoveryFeedSession {
|
|
|
308
472
|
// not a round later, so the network's first page in a category is not empty by design.
|
|
309
473
|
const previews = data.items.flatMap(topic => !this.category && topic.preview ? [{ key: key(topic.preview.protocol, topic.preview.nativeId), kind: 'social', post: topic.preview }] : []);
|
|
310
474
|
if (previews.length || stream.done)
|
|
311
|
-
return { items: previews, layout: ordinary, partial };
|
|
475
|
+
return { items: previews, layout: ordinary, partial, ...(previews.length ? { previews: true } : {}) };
|
|
312
476
|
return this.walkTopics(stream, params, validatePosts, partial, started);
|
|
313
477
|
}
|
|
314
478
|
validatePosts(data);
|
|
@@ -377,10 +541,21 @@ export class DiscoveryFeedSession {
|
|
|
377
541
|
async readPage() {
|
|
378
542
|
// The radar is one small snapshot read: start it with the page, not after the slowest provider,
|
|
379
543
|
// and start the lead story's member reads the moment the snapshot names it.
|
|
544
|
+
// The top of the feed starts with them too, and holds the page at most DISCOVERY_TOP_TIMEOUT_MS from here (loadTop).
|
|
545
|
+
// Neither waits for the other: the radar's own pick reads its members as soon as the radar answers (unless the top
|
|
546
|
+
// already named its lead), and a lead the top names reads its own as soon as the top answers (readyTop). A lead that
|
|
547
|
+
// is the radar's pick shares the one read (memberPosts).
|
|
548
|
+
if (this.page === 0 && this.top && !this.topLoading)
|
|
549
|
+
this.topLoading = this.loadTop();
|
|
380
550
|
if (this.page === 0 && this.radar && !this.radarLoading) {
|
|
381
551
|
this.radarLoading = this.loadRadar();
|
|
382
|
-
void this.radarLoading.then(clusters => {
|
|
383
|
-
|
|
552
|
+
void this.radarLoading.then(clusters => {
|
|
553
|
+
if (this.topLead)
|
|
554
|
+
return;
|
|
555
|
+
const lead = this.pickCluster(clusters, 0);
|
|
556
|
+
if (lead)
|
|
557
|
+
void this.memberPosts(lead);
|
|
558
|
+
});
|
|
384
559
|
}
|
|
385
560
|
// A network that failed for a passing reason sat out the page after its failure; from this one it takes its turn
|
|
386
561
|
// again. It stays failed, and named in retryingSources, until a page asks it: with more than three networks the
|
|
@@ -422,6 +597,9 @@ export class DiscoveryFeedSession {
|
|
|
422
597
|
cancel();
|
|
423
598
|
}
|
|
424
599
|
}));
|
|
600
|
+
// Page 0 opens with the top of the feed when it answered: its items lead, its lanes stand in for the networks' first reads.
|
|
601
|
+
const top = this.page === 0 && this.topLoading ? await this.topLoading : null;
|
|
602
|
+
const topItems = top ? this.placeTop(top, selected, reads) : [];
|
|
425
603
|
// Round-robin before grouping: the largest network cannot occupy every slot.
|
|
426
604
|
// Each item carries the judgment for its own window of its source response,
|
|
427
605
|
// so one page can feature several modules instead of one per source batch.
|
|
@@ -432,18 +610,20 @@ export class DiscoveryFeedSession {
|
|
|
432
610
|
if (!item || this.seen.has(item.key))
|
|
433
611
|
continue;
|
|
434
612
|
this.seen.add(item.key);
|
|
435
|
-
mixed.push({ item, layout: windowLayout(read.layout, index) });
|
|
613
|
+
mixed.push({ item, layout: read.layouts?.[index] ?? windowLayout(read.layout, index) });
|
|
436
614
|
}
|
|
437
615
|
// Deduplication memory is bounded independently of the UI's virtual window.
|
|
438
616
|
while (this.seen.size > 4096)
|
|
439
617
|
this.seen.delete(this.seen.values().next().value);
|
|
440
|
-
const modules =
|
|
618
|
+
const modules = topItems.map((item, index) => this.topModule(this.page, index, item));
|
|
441
619
|
const page = this.page++;
|
|
620
|
+
const topCount = modules.length;
|
|
442
621
|
let remaining = mixed;
|
|
443
|
-
// The radar leads the first page with the top breaking story and returns
|
|
622
|
+
// The radar leads the first page with the top breaking story (the top's lead, when it names one) and returns
|
|
444
623
|
// every second page with the next one. Its member posts move under the
|
|
445
624
|
// story, on this page or on any later one, so the hot stream never repeats them.
|
|
446
|
-
|
|
625
|
+
// A top that names a story leads with it even when no post came with the top or the reads (Codex review of #4205).
|
|
626
|
+
const cluster = mixed.length || topCount || top?.lead ? await this.radarCluster(page, top) : null;
|
|
447
627
|
let clusterItems = [];
|
|
448
628
|
if (cluster) {
|
|
449
629
|
const members = new Set(cluster.memberKeys);
|
|
@@ -495,7 +675,7 @@ export class DiscoveryFeedSession {
|
|
|
495
675
|
// the opening modules so the page still leads with content.
|
|
496
676
|
const recommendation = modules.length ? await this.recommendation(page) : null;
|
|
497
677
|
if (recommendation)
|
|
498
|
-
modules.splice(Math.min(2, modules.length), 0, recommendation);
|
|
678
|
+
modules.splice(Math.min(topCount + 2, modules.length), 0, recommendation);
|
|
499
679
|
if (cluster)
|
|
500
680
|
modules.splice(page === 0 ? 0 : Math.min(2, modules.length), 0, this.radarModule(page, cluster, clusterItems));
|
|
501
681
|
const fallover = this.streams.find(stream => stream.protocol === 'minds')?.fallover;
|
|
@@ -507,12 +687,20 @@ export class DiscoveryFeedSession {
|
|
|
507
687
|
...(retrying.length ? { retryingSources: retrying } : {}), partialCategories: reads.some(read => read.partial),
|
|
508
688
|
...(fallover ? { fallover } : {}), ...(compass ? { compass } : {}) };
|
|
509
689
|
}
|
|
510
|
-
/**
|
|
511
|
-
|
|
690
|
+
/**
|
|
691
|
+
* Page 0: the top's lead when it names one, else the top breaking cluster, else the top cluster. Every second later
|
|
692
|
+
* page: the next unused story or viral cluster.
|
|
693
|
+
*/
|
|
694
|
+
async radarCluster(page, top = null) {
|
|
512
695
|
if (!this.radar)
|
|
513
696
|
return null;
|
|
514
697
|
if (page === 0)
|
|
515
698
|
this.clusters = await (this.radarLoading ?? this.loadRadar());
|
|
699
|
+
if (page === 0 && top?.lead) {
|
|
700
|
+
const lead = this.leadOf(this.clusters ?? [], top);
|
|
701
|
+
this.radarUsed.add(lead.id);
|
|
702
|
+
return this.learning && this.learning.choose(lead.kind === 'breaking' ? 'breaking' : 'story', page) === 'posts' ? null : lead;
|
|
703
|
+
}
|
|
516
704
|
if (!this.clusters?.length || (page !== 0 && page % 2 !== 0))
|
|
517
705
|
return null;
|
|
518
706
|
const cluster = this.pickCluster(this.clusters, page);
|
|
@@ -524,10 +712,139 @@ export class DiscoveryFeedSession {
|
|
|
524
712
|
return null;
|
|
525
713
|
return cluster;
|
|
526
714
|
}
|
|
527
|
-
/** Page 0: the top breaking cluster, else the top cluster. Later pages: the next unused story or viral cluster. */
|
|
715
|
+
/** Page 0: the top breaking cluster, else the top cluster. Later pages: the next unused story or viral cluster (pickRadarCluster). */
|
|
528
716
|
pickCluster(clusters, page) {
|
|
529
|
-
|
|
530
|
-
|
|
717
|
+
return pickRadarCluster(clusters, page, { used: this.radarUsed, seenClusters: this.seenClusters, bucket: this.bucket });
|
|
718
|
+
}
|
|
719
|
+
/** Page 0's story: the top's lead (from the radar snapshot when it holds it, so its links and members come along), else the client's pick. */
|
|
720
|
+
leadOf(clusters, top) {
|
|
721
|
+
if (top?.lead)
|
|
722
|
+
return clusters.find(candidate => candidate.id === top.lead.id) ?? top.lead;
|
|
723
|
+
return this.pickCluster(clusters, 0);
|
|
724
|
+
}
|
|
725
|
+
/** One top item as its own module, in the top's order: a video plays as a video module (unless the member's layouts say posts). */
|
|
726
|
+
topModule(page, index, item) {
|
|
727
|
+
const video = !!item.post.media?.some(media => media.type === 'video') && (this.learning?.choose('video', page) ?? 'video') === 'video';
|
|
728
|
+
return { id: `discovery-${page}-top-${index}-${item.key}`, presentation: video ? 'video' : 'posts', source: 'fallback', items: [item],
|
|
729
|
+
spec: { root: 'module', elements: { module: { type: video ? 'DiscoveryVideo' : 'DiscoveryPosts', props: { itemKeys: [item.key] }, children: [] } } } };
|
|
730
|
+
}
|
|
731
|
+
/**
|
|
732
|
+
* The top's read, then the posts its items need, all under one deadline of DISCOVERY_TOP_TIMEOUT_MS from the page's
|
|
733
|
+
* start: the reads of its posts get what is left of it, never a deadline of their own. Null when it failed, timed out
|
|
734
|
+
* or answered nothing usable: the page is then as without a top.
|
|
735
|
+
*/
|
|
736
|
+
async loadTop() {
|
|
737
|
+
const top = this.top;
|
|
738
|
+
let timer;
|
|
739
|
+
const deadline = new Promise(resolve => { timer = setTimeout(() => resolve(TOP_LATE), DISCOVERY_TOP_TIMEOUT_MS); });
|
|
740
|
+
try {
|
|
741
|
+
const limit = Number.isSafeInteger(top.limit) && top.limit >= 1 && top.limit <= 25 ? top.limit : DISCOVERY_TOP_LIMIT;
|
|
742
|
+
const answer = await Promise.race([Promise.resolve().then(() => top.read({ ref: top.ref, limit, ...(nonEmpty(top.session) ? { session: top.session } : {}) })), deadline]);
|
|
743
|
+
if (answer === TOP_LATE)
|
|
744
|
+
return null;
|
|
745
|
+
return await this.readyTop(answer, deadline);
|
|
746
|
+
}
|
|
747
|
+
catch {
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
finally {
|
|
751
|
+
clearTimeout(timer);
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Checks a top answer and reads what its items need before `deadline`: Minds posts by id, a story's network posts
|
|
756
|
+
* (lanes come with their cards). A lead it names starts reading its member posts at once.
|
|
757
|
+
*/
|
|
758
|
+
async readyTop(answer, deadline) {
|
|
759
|
+
if (!answer || typeof answer !== 'object')
|
|
760
|
+
return null;
|
|
761
|
+
const value = answer;
|
|
762
|
+
if (!Array.isArray(value.items))
|
|
763
|
+
return null;
|
|
764
|
+
// Only what this session reads: a Minds item needs its Minds stream, a network's item or lane that network's stream.
|
|
765
|
+
const streams = new Set(this.streams.map(stream => stream.protocol));
|
|
766
|
+
const rows = [];
|
|
767
|
+
for (const raw of value.items.slice(0, 25)) {
|
|
768
|
+
const row = topRow(raw);
|
|
769
|
+
if (row && !rows.some(existing => existing.key === row.key) && streams.has(row.kind === 'minds' ? 'minds' : JSON.parse(row.key)[0]))
|
|
770
|
+
rows.push(row);
|
|
771
|
+
}
|
|
772
|
+
const lanes = new Map();
|
|
773
|
+
const networks = value.networks && typeof value.networks === 'object' ? value.networks : {};
|
|
774
|
+
for (const protocol of TOP_NETWORKS) {
|
|
775
|
+
const entries = networks[protocol];
|
|
776
|
+
if (!streams.has(protocol) || !Array.isArray(entries))
|
|
777
|
+
continue;
|
|
778
|
+
const lane = [];
|
|
779
|
+
for (const raw of entries.slice(0, 12)) {
|
|
780
|
+
const item = laneItem(raw, protocol);
|
|
781
|
+
if (item && !lane.some(existing => existing.key === item.key))
|
|
782
|
+
lane.push({ ...item, top: { ...item.top, position: lane.length, block: lane.length < 10 ? 'top' : 'below' } });
|
|
783
|
+
}
|
|
784
|
+
if (lane.length)
|
|
785
|
+
lanes.set(protocol, lane);
|
|
786
|
+
}
|
|
787
|
+
const lead = this.radar ? storyCluster(value.lead) : null;
|
|
788
|
+
const impressions = (value.meta?.flag?.impressions) === 'on';
|
|
789
|
+
if (!rows.length && !lanes.size && !lead)
|
|
790
|
+
return null;
|
|
791
|
+
// The page's story is the top's lead: its member reads start now, alongside the reads below, not after them.
|
|
792
|
+
this.topLead = lead?.id ?? null;
|
|
793
|
+
if (lead && this.radarLoading)
|
|
794
|
+
void this.radarLoading.then(clusters => this.memberPosts(clusters.find(candidate => candidate.id === lead.id) ?? lead));
|
|
795
|
+
// The posts the rows need, read at once under the top's one deadline: a Minds post by id; a network post from its
|
|
796
|
+
// lane's card, else from its story's member posts. A row nothing read in time is left out (the first hot page may
|
|
797
|
+
// still carry it).
|
|
798
|
+
const inTime = (read, fallback) => Promise.race([Promise.resolve().then(read).catch(() => fallback), deadline.then(() => fallback)]);
|
|
799
|
+
const posts = new Map();
|
|
800
|
+
for (const lane of lanes.values())
|
|
801
|
+
for (const item of lane)
|
|
802
|
+
posts.set(item.key, { key: item.key, kind: 'social', post: item.post });
|
|
803
|
+
const readPost = this.top?.post ?? this.members?.minds;
|
|
804
|
+
const mindsReads = readPost ? rows.filter(row => row.kind === 'minds').map(row => inTime(() => readPost(row.id).then(({ data }) => data && data.id === row.id ? { key: row.key, kind: 'minds', post: data } : null), null)) : [];
|
|
805
|
+
const stories = [...new Set(rows.filter(row => row.kind === 'social' && !posts.has(row.key) && row.cluster).map(row => row.cluster))].slice(0, TOP_MAX_STORY_READS);
|
|
806
|
+
const wanted = new Set(rows.filter(row => row.kind === 'social').map(row => row.key));
|
|
807
|
+
const social = this.members?.social;
|
|
808
|
+
const storyReads = social ? stories.map(cluster => inTime(() => social({ cluster }).then(({ data }) => !data || data.cluster !== cluster || !Array.isArray(data.items) ? [] : data.items
|
|
809
|
+
.filter(post => post && idString(post.nativeId) && post.author?.protocol === post.protocol && wanted.has(key(post.protocol, post.nativeId)))
|
|
810
|
+
.map(post => ({ key: key(post.protocol, post.nativeId), kind: 'social', post }))), [])) : [];
|
|
811
|
+
const [minds, network] = await Promise.all([Promise.all(mindsReads), Promise.all(storyReads)]);
|
|
812
|
+
for (const item of [...minds, ...network.flat()])
|
|
813
|
+
if (item && !posts.has(item.key))
|
|
814
|
+
posts.set(item.key, item);
|
|
815
|
+
return { rows, posts, lanes, lead, impressions };
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Places the top on page 0: each lane stands in for its network's first read (a topics network's catalog previews; a
|
|
819
|
+
* posts network's read follows its lane), and the top's items, read by id or carried by this page, become the page's
|
|
820
|
+
* first items, marked seen so no hot page repeats them. Without impressions in the top's order, the member's seen
|
|
821
|
+
* items follow the unseen ones.
|
|
822
|
+
*/
|
|
823
|
+
placeTop(top, selected, reads) {
|
|
824
|
+
for (const [protocol, lane] of top.lanes) {
|
|
825
|
+
const at = selected.findIndex(stream => stream.protocol === protocol);
|
|
826
|
+
const read = at >= 0 ? reads[at] : null;
|
|
827
|
+
if (read) {
|
|
828
|
+
const kept = read.previews ? [] : read.items;
|
|
829
|
+
reads[at] = { ...read, items: [...lane, ...kept], layouts: [...lane.map(() => ordinary), ...kept.map((_item, index) => read.layouts?.[index] ?? windowLayout(read.layout, index))] };
|
|
830
|
+
}
|
|
831
|
+
else if (this.streams.some(stream => stream.protocol === protocol))
|
|
832
|
+
reads.push({ items: lane, layout: ordinary, partial: false });
|
|
833
|
+
}
|
|
834
|
+
const carried = new Map(reads.flatMap(read => read.items.map(item => [item.key, item])));
|
|
835
|
+
const seen = (row) => !!this.appSeen && row.seenRefs.some(ref => this.appSeen.has(ref));
|
|
836
|
+
const rows = top.impressions || !this.appSeen ? top.rows : [...top.rows.filter(row => !seen(row)), ...top.rows.filter(seen)];
|
|
837
|
+
const placed = [];
|
|
838
|
+
for (const row of rows) {
|
|
839
|
+
const found = top.posts.get(row.key) ?? carried.get(row.key);
|
|
840
|
+
if (!found || this.seen.has(row.key))
|
|
841
|
+
continue;
|
|
842
|
+
const position = placed.length;
|
|
843
|
+
placed.push({ ...found, top: { ref: row.ref, position, block: position < 10 ? 'top' : 'below' } });
|
|
844
|
+
this.seen.add(row.key);
|
|
845
|
+
this.topKeys.add(row.key);
|
|
846
|
+
}
|
|
847
|
+
return placed;
|
|
531
848
|
}
|
|
532
849
|
/** The story's members as feed items, read once per story and bounded to three seconds; a failed read is simply none. */
|
|
533
850
|
memberPosts(cluster) {
|
|
@@ -645,11 +962,13 @@ export class DiscoveryResource {
|
|
|
645
962
|
protocols;
|
|
646
963
|
curator;
|
|
647
964
|
http;
|
|
648
|
-
|
|
965
|
+
feeds;
|
|
966
|
+
constructor(posts, protocols, curator, http, feeds) {
|
|
649
967
|
this.posts = posts;
|
|
650
968
|
this.protocols = protocols;
|
|
651
969
|
this.curator = curator;
|
|
652
970
|
this.http = http;
|
|
971
|
+
this.feeds = feeds;
|
|
653
972
|
}
|
|
654
973
|
/**
|
|
655
974
|
* Suggestions for the signed-in viewer from the app's one recommender: active or warm members, groups
|
|
@@ -726,12 +1045,23 @@ export class DiscoveryResource {
|
|
|
726
1045
|
* Discover's feed session. `compass` (a Politics choice, as `posts.list` takes it) goes on every Minds read, whose
|
|
727
1046
|
* pages then carry `meta.compass_filter`; the open networks' reads and the radar are not filtered by it, so a view
|
|
728
1047
|
* that leaves posts out (Hide politics, Choose sides) is best read with `source: 'minds'`.
|
|
1048
|
+
*
|
|
1049
|
+
* `top` opens the first page with the top of the feed (`feeds.top`, with the same `compass`, unless `top.read` is
|
|
1050
|
+
* given); `seen` leaves what the member saw out of the hot pages; `seenClusters` and `bucket` rotate the story when
|
|
1051
|
+
* the top names none (DiscoveryFeedSessionOptions). With none of them the session is what it was before they existed.
|
|
729
1052
|
*/
|
|
730
1053
|
createFeed(options) {
|
|
731
1054
|
const compass = options.compass;
|
|
732
1055
|
const posts = compass ? { list: params => this.posts.list({ ...params, compass }) } : this.posts;
|
|
1056
|
+
const feeds = this.feeds;
|
|
1057
|
+
const http = this.http;
|
|
1058
|
+
const read = options.top?.read ?? (feeds ? params => feeds.top({ ...params, ...(compass ? { compass } : {}) }, NO_RETRY) : undefined);
|
|
1059
|
+
// A top item the first hot page does not carry is read by id once, never retried: the page holds its own deadline.
|
|
1060
|
+
const post = options.top?.post ?? (http ? (id) => http.get(`/posts/${encodeURIComponent(id)}`, undefined, NO_RETRY) : undefined);
|
|
1061
|
+
const top = options.top && read ? { ...options.top, read, ...(post ? { post } : {}) } : undefined;
|
|
733
1062
|
// The radar rides along by default; an older server answers 404 and the session simply shows no story module.
|
|
734
|
-
return new DiscoveryFeedSession(posts, this.protocols, options.networks, options.category, options.source, options.learning, options.recommend, params => this.protocols.getPublicRadar(params, NO_RETRY), { social: params => this.protocols.getPublicRadarPosts(params, NO_RETRY), minds: id => this.posts.get(id) })
|
|
1063
|
+
return new DiscoveryFeedSession(posts, this.protocols, options.networks, options.category, options.source, options.learning, options.recommend, params => this.protocols.getPublicRadar(params, NO_RETRY), { social: params => this.protocols.getPublicRadarPosts(params, NO_RETRY), minds: id => this.posts.get(id) }, { ...(top ? { top } : {}), ...(options.seen ? { seen: options.seen } : {}), ...(options.seenClusters ? { seenClusters: options.seenClusters } : {}),
|
|
1064
|
+
...(options.bucket !== undefined ? { bucket: options.bucket } : {}) });
|
|
735
1065
|
}
|
|
736
1066
|
}
|
|
737
1067
|
/** A read bounded to three seconds; a late or failed one resolves to `fallback`, never an error. */
|