birdclaw 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -1
- package/README.md +50 -5
- package/package.json +3 -2
- package/scripts/browser-perf.mjs +1 -0
- package/scripts/start-test-server.mjs +16 -3
- package/src/cli.ts +376 -13
- package/src/components/AccountSwitcher.tsx +186 -0
- package/src/components/AppNav.tsx +27 -7
- package/src/components/AvatarChip.tsx +9 -3
- package/src/components/DmWorkspace.tsx +18 -8
- package/src/components/LinkPreviewCard.tsx +40 -18
- package/src/components/MarkdownViewer.tsx +452 -0
- package/src/components/SyncNowButton.tsx +57 -25
- package/src/components/ThemeSlider.tsx +55 -50
- package/src/components/TimelineCard.tsx +225 -93
- package/src/components/TimelineRouteFrame.tsx +22 -8
- package/src/components/TweetMediaGrid.tsx +87 -38
- package/src/components/TweetRichText.tsx +15 -11
- package/src/components/account-selection.ts +64 -0
- package/src/components/useTimelineRouteData.ts +23 -7
- package/src/lib/account-sync-job.ts +654 -0
- package/src/lib/actions-transport.ts +216 -146
- package/src/lib/api-client.ts +128 -53
- package/src/lib/archive-finder.ts +78 -63
- package/src/lib/archive-import.ts +1364 -1300
- package/src/lib/authored-live.ts +261 -204
- package/src/lib/avatar-cache.ts +159 -44
- package/src/lib/backup.ts +1532 -951
- package/src/lib/bird-actions.ts +139 -57
- package/src/lib/bird-command.ts +101 -28
- package/src/lib/bird.ts +549 -194
- package/src/lib/blocklist.ts +40 -23
- package/src/lib/blocks-write.ts +129 -80
- package/src/lib/blocks.ts +165 -97
- package/src/lib/bookmark-sync-job.ts +250 -160
- package/src/lib/conversation-surface.ts +79 -48
- package/src/lib/db.ts +33 -3
- package/src/lib/dms-live.ts +720 -66
- package/src/lib/effect-runtime.ts +45 -0
- package/src/lib/follow-graph.ts +224 -180
- package/src/lib/http-effect.ts +222 -0
- package/src/lib/inbox.ts +74 -43
- package/src/lib/link-index.ts +88 -76
- package/src/lib/link-insights.ts +24 -0
- package/src/lib/link-preview-metadata.ts +472 -52
- package/src/lib/media-fetch.ts +286 -213
- package/src/lib/mention-threads-live.ts +352 -288
- package/src/lib/mentions-live.ts +390 -342
- package/src/lib/moderation-target.ts +102 -65
- package/src/lib/moderation-write.ts +77 -18
- package/src/lib/mutes-write.ts +129 -80
- package/src/lib/mutes.ts +8 -1
- package/src/lib/openai.ts +84 -53
- package/src/lib/period-digest.ts +953 -0
- package/src/lib/profile-affiliation-hydration.ts +93 -54
- package/src/lib/profile-hydration.ts +124 -72
- package/src/lib/profile-replies.ts +60 -43
- package/src/lib/profile-resolver.ts +402 -294
- package/src/lib/queries.ts +969 -199
- package/src/lib/research.ts +165 -120
- package/src/lib/sqlite.ts +1 -0
- package/src/lib/timeline-collections-live.ts +204 -167
- package/src/lib/timeline-live.ts +60 -39
- package/src/lib/tweet-lookup.ts +30 -19
- package/src/lib/types.ts +38 -1
- package/src/lib/ui.ts +30 -7
- package/src/lib/url-expansion.ts +226 -55
- package/src/lib/url-safety.ts +220 -0
- package/src/lib/web-sync.ts +216 -148
- package/src/lib/whois.ts +166 -147
- package/src/lib/x-web.ts +102 -71
- package/src/lib/xurl.ts +681 -411
- package/src/routeTree.gen.ts +42 -0
- package/src/routes/__root.tsx +25 -5
- package/src/routes/api/action.tsx +127 -78
- package/src/routes/api/avatar.tsx +39 -30
- package/src/routes/api/blocks.tsx +26 -23
- package/src/routes/api/conversation.tsx +25 -14
- package/src/routes/api/inbox.tsx +27 -21
- package/src/routes/api/link-insights.tsx +31 -25
- package/src/routes/api/link-preview.tsx +25 -21
- package/src/routes/api/period-digest.tsx +123 -0
- package/src/routes/api/profile-hydrate.tsx +31 -28
- package/src/routes/api/query.tsx +79 -55
- package/src/routes/api/status.tsx +18 -10
- package/src/routes/api/sync.tsx +75 -29
- package/src/routes/dms.tsx +95 -28
- package/src/routes/inbox.tsx +32 -19
- package/src/routes/today.tsx +441 -0
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { Database } from "./sqlite";
|
|
2
|
-
import {
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
import { listThreadViaBirdEffect } from "./bird";
|
|
3
4
|
import { getNativeDb } from "./db";
|
|
5
|
+
import { runEffectPromise } from "./effect-runtime";
|
|
4
6
|
import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
|
|
5
7
|
import type {
|
|
6
8
|
XurlMentionData,
|
|
@@ -11,7 +13,7 @@ import type {
|
|
|
11
13
|
} from "./types";
|
|
12
14
|
import { upsertTweetAccountEdge } from "./tweet-account-edges";
|
|
13
15
|
import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
|
|
14
|
-
import {
|
|
16
|
+
import { getTweetByIdEffect, searchRecentByConversationIdEffect } from "./xurl";
|
|
15
17
|
|
|
16
18
|
const DEFAULT_LIMIT = 30;
|
|
17
19
|
const DEFAULT_DELAY_MS = 1500;
|
|
@@ -21,6 +23,15 @@ const DEFAULT_FALLBACK_DEPTH = 12;
|
|
|
21
23
|
const MAX_XURL_SEARCH_RESULTS = 100;
|
|
22
24
|
|
|
23
25
|
export type MentionThreadsMode = "bird" | "xurl";
|
|
26
|
+
export interface SyncMentionThreadsOptions {
|
|
27
|
+
account?: string;
|
|
28
|
+
mode?: string;
|
|
29
|
+
limit?: number;
|
|
30
|
+
delayMs?: number;
|
|
31
|
+
timeoutMs?: number;
|
|
32
|
+
all?: boolean;
|
|
33
|
+
maxPages?: number;
|
|
34
|
+
}
|
|
24
35
|
|
|
25
36
|
interface LocalMention {
|
|
26
37
|
id: string;
|
|
@@ -28,6 +39,15 @@ interface LocalMention {
|
|
|
28
39
|
conversationId?: string;
|
|
29
40
|
rawTweet?: XurlMentionData;
|
|
30
41
|
}
|
|
42
|
+
interface ThreadFetchResult {
|
|
43
|
+
strategy: string;
|
|
44
|
+
payload: XurlMentionsResponse;
|
|
45
|
+
pages?: number;
|
|
46
|
+
fallbackDepth?: number;
|
|
47
|
+
generalReadTweets: number;
|
|
48
|
+
truncated?: boolean;
|
|
49
|
+
warnings: string[];
|
|
50
|
+
}
|
|
31
51
|
|
|
32
52
|
function assertPositiveInteger(value: number, name: string) {
|
|
33
53
|
if (!Number.isFinite(value) || value < 1) {
|
|
@@ -54,8 +74,15 @@ function parseMode(value: string | undefined): MentionThreadsMode {
|
|
|
54
74
|
return mode;
|
|
55
75
|
}
|
|
56
76
|
|
|
57
|
-
function
|
|
58
|
-
return new
|
|
77
|
+
function toError(error: unknown) {
|
|
78
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function trySync<T>(try_: () => T) {
|
|
82
|
+
return Effect.try({
|
|
83
|
+
try: try_,
|
|
84
|
+
catch: toError,
|
|
85
|
+
});
|
|
59
86
|
}
|
|
60
87
|
|
|
61
88
|
function getRemainingThreadTimeoutMs(
|
|
@@ -341,7 +368,7 @@ function mergeMentionThreadIntoLocalStore({
|
|
|
341
368
|
})();
|
|
342
369
|
}
|
|
343
370
|
|
|
344
|
-
|
|
371
|
+
function fetchConversationViaRecentSearchEffect({
|
|
345
372
|
conversationId,
|
|
346
373
|
all,
|
|
347
374
|
maxPages,
|
|
@@ -354,39 +381,46 @@ async function fetchConversationViaRecentSearch({
|
|
|
354
381
|
timeoutMs: number;
|
|
355
382
|
deadlineMs: number;
|
|
356
383
|
}) {
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
384
|
+
return Effect.gen(function* () {
|
|
385
|
+
const pages: XurlTweetsResponse[] = [];
|
|
386
|
+
let nextToken: string | undefined;
|
|
387
|
+
let pageCount = 0;
|
|
388
|
+
|
|
389
|
+
do {
|
|
390
|
+
const payload = yield* searchRecentByConversationIdEffect(
|
|
391
|
+
conversationId,
|
|
392
|
+
{
|
|
393
|
+
maxResults: MAX_XURL_SEARCH_RESULTS,
|
|
394
|
+
paginationToken: nextToken,
|
|
395
|
+
timeoutMs: yield* trySync(() =>
|
|
396
|
+
getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
|
|
397
|
+
),
|
|
398
|
+
},
|
|
399
|
+
);
|
|
400
|
+
pages.push(payload);
|
|
401
|
+
nextToken =
|
|
402
|
+
typeof payload.meta?.next_token === "string"
|
|
403
|
+
? payload.meta.next_token
|
|
404
|
+
: undefined;
|
|
405
|
+
pageCount += 1;
|
|
406
|
+
} while (
|
|
407
|
+
(all || maxPages !== undefined) &&
|
|
408
|
+
nextToken &&
|
|
409
|
+
(maxPages === undefined || pageCount < maxPages)
|
|
410
|
+
);
|
|
378
411
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
412
|
+
const payload = mergePayloads(pages);
|
|
413
|
+
const paginationRequested = all || maxPages !== undefined;
|
|
414
|
+
return {
|
|
415
|
+
payload,
|
|
416
|
+
pages: pageCount,
|
|
417
|
+
truncated: paginationRequested && Boolean(nextToken),
|
|
418
|
+
generalReadTweets: payload.data.length,
|
|
419
|
+
};
|
|
420
|
+
});
|
|
387
421
|
}
|
|
388
422
|
|
|
389
|
-
|
|
423
|
+
function fetchParentChainViaXurlEffect({
|
|
390
424
|
mention,
|
|
391
425
|
maxDepth,
|
|
392
426
|
timeoutMs,
|
|
@@ -397,76 +431,82 @@ async function fetchParentChainViaXurl({
|
|
|
397
431
|
timeoutMs: number;
|
|
398
432
|
deadlineMs: number;
|
|
399
433
|
}) {
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
let shouldUseRawAnchor = Boolean(rawAnchorPayload);
|
|
412
|
-
|
|
413
|
-
if (!nextParentId) {
|
|
414
|
-
const anchorPayload = await getTweetById(mention.id, {
|
|
415
|
-
timeoutMs: getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
|
|
416
|
-
});
|
|
417
|
-
pages.push(anchorPayload);
|
|
418
|
-
generalReadTweets += anchorPayload.data.length;
|
|
419
|
-
const anchorTweet = anchorPayload.data[0];
|
|
420
|
-
if (anchorTweet) {
|
|
421
|
-
shouldUseRawAnchor = false;
|
|
422
|
-
seenTweetIds.add(anchorTweet.id);
|
|
423
|
-
nextParentId = anchorTweet.in_reply_to_user_id
|
|
424
|
-
? getReplyToId(anchorTweet)
|
|
434
|
+
return Effect.gen(function* () {
|
|
435
|
+
const pages: XurlTweetsResponse[] = [];
|
|
436
|
+
const warnings: string[] = [];
|
|
437
|
+
const seenTweetIds = new Set([mention.id]);
|
|
438
|
+
let nextParentId = mention.replyToId;
|
|
439
|
+
let fallbackDepth = 0;
|
|
440
|
+
let generalReadTweets = 0;
|
|
441
|
+
|
|
442
|
+
const rawAnchorPayload =
|
|
443
|
+
mention.rawTweet && mention.rawTweet.id === mention.id
|
|
444
|
+
? ({ data: [mention.rawTweet] } satisfies XurlTweetsResponse)
|
|
425
445
|
: undefined;
|
|
426
|
-
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
if (shouldUseRawAnchor && rawAnchorPayload) {
|
|
430
|
-
pages.unshift(rawAnchorPayload);
|
|
431
|
-
}
|
|
446
|
+
let shouldUseRawAnchor = Boolean(rawAnchorPayload);
|
|
432
447
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
448
|
+
if (!nextParentId) {
|
|
449
|
+
const anchorPayload = yield* getTweetByIdEffect(mention.id, {
|
|
450
|
+
timeoutMs: yield* trySync(() =>
|
|
451
|
+
getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
|
|
452
|
+
),
|
|
453
|
+
});
|
|
454
|
+
pages.push(anchorPayload);
|
|
455
|
+
generalReadTweets += anchorPayload.data.length;
|
|
456
|
+
const anchorTweet = anchorPayload.data[0];
|
|
457
|
+
if (anchorTweet) {
|
|
458
|
+
shouldUseRawAnchor = false;
|
|
459
|
+
seenTweetIds.add(anchorTweet.id);
|
|
460
|
+
nextParentId = anchorTweet.in_reply_to_user_id
|
|
461
|
+
? getReplyToId(anchorTweet)
|
|
462
|
+
: undefined;
|
|
463
|
+
}
|
|
439
464
|
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
);
|
|
444
|
-
break;
|
|
465
|
+
|
|
466
|
+
if (shouldUseRawAnchor && rawAnchorPayload) {
|
|
467
|
+
pages.unshift(rawAnchorPayload);
|
|
445
468
|
}
|
|
446
469
|
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
470
|
+
while (nextParentId) {
|
|
471
|
+
if (fallbackDepth >= maxDepth) {
|
|
472
|
+
warnings.push(
|
|
473
|
+
`fallback parent-chain depth cap reached for ${mention.id} after ${maxDepth} hops`,
|
|
474
|
+
);
|
|
475
|
+
break;
|
|
476
|
+
}
|
|
477
|
+
if (seenTweetIds.has(nextParentId)) {
|
|
478
|
+
warnings.push(
|
|
479
|
+
`fallback parent-chain cycle detected for ${mention.id} at ${nextParentId}`,
|
|
480
|
+
);
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
fallbackDepth += 1;
|
|
485
|
+
const parentPayload = yield* getTweetByIdEffect(nextParentId, {
|
|
486
|
+
timeoutMs: yield* trySync(() =>
|
|
487
|
+
getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
|
|
488
|
+
),
|
|
489
|
+
});
|
|
490
|
+
pages.push(parentPayload);
|
|
491
|
+
generalReadTweets += parentPayload.data.length;
|
|
492
|
+
const parentTweet = parentPayload.data[0];
|
|
493
|
+
if (!parentTweet) {
|
|
494
|
+
break;
|
|
495
|
+
}
|
|
496
|
+
seenTweetIds.add(parentTweet.id);
|
|
497
|
+
nextParentId = parentTweet.in_reply_to_user_id
|
|
498
|
+
? getReplyToId(parentTweet)
|
|
499
|
+
: undefined;
|
|
456
500
|
}
|
|
457
|
-
seenTweetIds.add(parentTweet.id);
|
|
458
|
-
nextParentId = parentTweet.in_reply_to_user_id
|
|
459
|
-
? getReplyToId(parentTweet)
|
|
460
|
-
: undefined;
|
|
461
|
-
}
|
|
462
501
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
502
|
+
const payload = mergePayloads(pages);
|
|
503
|
+
return {
|
|
504
|
+
payload,
|
|
505
|
+
fallbackDepth,
|
|
506
|
+
warnings,
|
|
507
|
+
generalReadTweets,
|
|
508
|
+
};
|
|
509
|
+
});
|
|
470
510
|
}
|
|
471
511
|
|
|
472
512
|
function findMissingAncestorId(
|
|
@@ -505,7 +545,7 @@ function findMissingAncestorId(
|
|
|
505
545
|
return undefined;
|
|
506
546
|
}
|
|
507
547
|
|
|
508
|
-
|
|
548
|
+
function fetchThreadContextViaXurlEffect({
|
|
509
549
|
mention,
|
|
510
550
|
all,
|
|
511
551
|
maxPages,
|
|
@@ -518,98 +558,100 @@ async function fetchThreadContextViaXurl({
|
|
|
518
558
|
maxFallbackDepth: number;
|
|
519
559
|
timeoutMs: number;
|
|
520
560
|
}) {
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
if (mention.
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
561
|
+
return Effect.gen(function* () {
|
|
562
|
+
const deadlineMs = Date.now() + timeoutMs;
|
|
563
|
+
if (!mention.conversationId) {
|
|
564
|
+
if (mention.replyToId) {
|
|
565
|
+
const fallback = yield* fetchParentChainViaXurlEffect({
|
|
566
|
+
mention,
|
|
567
|
+
maxDepth: maxFallbackDepth,
|
|
568
|
+
timeoutMs,
|
|
569
|
+
deadlineMs,
|
|
570
|
+
});
|
|
571
|
+
return {
|
|
572
|
+
strategy: "parent_walk" as const,
|
|
573
|
+
pages: 0,
|
|
574
|
+
truncated: false,
|
|
575
|
+
payload: fallback.payload,
|
|
576
|
+
fallbackDepth: fallback.fallbackDepth,
|
|
577
|
+
generalReadTweets: fallback.generalReadTweets,
|
|
578
|
+
warnings: [
|
|
579
|
+
`missing conversation_id for ${mention.id}; used parent walk`,
|
|
580
|
+
...fallback.warnings,
|
|
581
|
+
],
|
|
582
|
+
};
|
|
583
|
+
}
|
|
530
584
|
return {
|
|
531
|
-
strategy: "
|
|
585
|
+
strategy: "skipped:no_conversation_id" as const,
|
|
586
|
+
payload: { data: [] } satisfies XurlMentionsResponse,
|
|
532
587
|
pages: 0,
|
|
588
|
+
fallbackDepth: 0,
|
|
589
|
+
generalReadTweets: 0,
|
|
590
|
+
warnings: [`skipped ${mention.id}: missing conversation_id`],
|
|
533
591
|
truncated: false,
|
|
534
|
-
payload: fallback.payload,
|
|
535
|
-
fallbackDepth: fallback.fallbackDepth,
|
|
536
|
-
generalReadTweets: fallback.generalReadTweets,
|
|
537
|
-
warnings: [
|
|
538
|
-
`missing conversation_id for ${mention.id}; used parent walk`,
|
|
539
|
-
...fallback.warnings,
|
|
540
|
-
],
|
|
541
592
|
};
|
|
542
593
|
}
|
|
543
|
-
return {
|
|
544
|
-
strategy: "skipped:no_conversation_id" as const,
|
|
545
|
-
payload: { data: [] } satisfies XurlMentionsResponse,
|
|
546
|
-
pages: 0,
|
|
547
|
-
fallbackDepth: 0,
|
|
548
|
-
generalReadTweets: 0,
|
|
549
|
-
warnings: [`skipped ${mention.id}: missing conversation_id`],
|
|
550
|
-
truncated: false,
|
|
551
|
-
};
|
|
552
|
-
}
|
|
553
594
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
595
|
+
const search = yield* fetchConversationViaRecentSearchEffect({
|
|
596
|
+
conversationId: mention.conversationId,
|
|
597
|
+
all,
|
|
598
|
+
maxPages,
|
|
599
|
+
timeoutMs,
|
|
600
|
+
deadlineMs,
|
|
601
|
+
});
|
|
602
|
+
if (search.payload.data.length > 0) {
|
|
603
|
+
const missingAncestorId = findMissingAncestorId(mention, search.payload);
|
|
604
|
+
if (missingAncestorId) {
|
|
605
|
+
const fallback = yield* fetchParentChainViaXurlEffect({
|
|
606
|
+
mention,
|
|
607
|
+
maxDepth: maxFallbackDepth,
|
|
608
|
+
timeoutMs,
|
|
609
|
+
deadlineMs,
|
|
610
|
+
});
|
|
611
|
+
return {
|
|
612
|
+
strategy: "conversation_search+parent_walk" as const,
|
|
613
|
+
pages: search.pages,
|
|
614
|
+
truncated: search.truncated,
|
|
615
|
+
payload: mergePayloads([search.payload, fallback.payload]),
|
|
616
|
+
fallbackDepth: fallback.fallbackDepth,
|
|
617
|
+
generalReadTweets:
|
|
618
|
+
search.generalReadTweets + fallback.generalReadTweets,
|
|
619
|
+
warnings: [
|
|
620
|
+
`recent search missed ancestor ${missingAncestorId} for conversation ${mention.conversationId}; used parent walk`,
|
|
621
|
+
...fallback.warnings,
|
|
622
|
+
],
|
|
623
|
+
};
|
|
624
|
+
}
|
|
570
625
|
return {
|
|
571
|
-
strategy: "conversation_search
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
fallbackDepth: fallback.fallbackDepth,
|
|
576
|
-
generalReadTweets:
|
|
577
|
-
search.generalReadTweets + fallback.generalReadTweets,
|
|
578
|
-
warnings: [
|
|
579
|
-
`recent search missed ancestor ${missingAncestorId} for conversation ${mention.conversationId}; used parent walk`,
|
|
580
|
-
...fallback.warnings,
|
|
581
|
-
],
|
|
626
|
+
strategy: "conversation_search" as const,
|
|
627
|
+
fallbackDepth: 0,
|
|
628
|
+
warnings: [] as string[],
|
|
629
|
+
...search,
|
|
582
630
|
};
|
|
583
631
|
}
|
|
632
|
+
|
|
633
|
+
const fallback = yield* fetchParentChainViaXurlEffect({
|
|
634
|
+
mention,
|
|
635
|
+
maxDepth: maxFallbackDepth,
|
|
636
|
+
timeoutMs,
|
|
637
|
+
deadlineMs,
|
|
638
|
+
});
|
|
584
639
|
return {
|
|
585
|
-
strategy: "
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
640
|
+
strategy: "parent_walk" as const,
|
|
641
|
+
pages: search.pages,
|
|
642
|
+
truncated: search.truncated,
|
|
643
|
+
payload: fallback.payload,
|
|
644
|
+
fallbackDepth: fallback.fallbackDepth,
|
|
645
|
+
generalReadTweets: search.generalReadTweets + fallback.generalReadTweets,
|
|
646
|
+
warnings: [
|
|
647
|
+
`recent search returned no tweets for conversation ${mention.conversationId}; used parent walk`,
|
|
648
|
+
...fallback.warnings,
|
|
649
|
+
],
|
|
589
650
|
};
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
const fallback = await fetchParentChainViaXurl({
|
|
593
|
-
mention,
|
|
594
|
-
maxDepth: maxFallbackDepth,
|
|
595
|
-
timeoutMs,
|
|
596
|
-
deadlineMs,
|
|
597
651
|
});
|
|
598
|
-
return {
|
|
599
|
-
strategy: "parent_walk" as const,
|
|
600
|
-
pages: search.pages,
|
|
601
|
-
truncated: search.truncated,
|
|
602
|
-
payload: fallback.payload,
|
|
603
|
-
fallbackDepth: fallback.fallbackDepth,
|
|
604
|
-
generalReadTweets: search.generalReadTweets + fallback.generalReadTweets,
|
|
605
|
-
warnings: [
|
|
606
|
-
`recent search returned no tweets for conversation ${mention.conversationId}; used parent walk`,
|
|
607
|
-
...fallback.warnings,
|
|
608
|
-
],
|
|
609
|
-
};
|
|
610
652
|
}
|
|
611
653
|
|
|
612
|
-
export
|
|
654
|
+
export function syncMentionThreadsEffect({
|
|
613
655
|
account,
|
|
614
656
|
mode = DEFAULT_MODE,
|
|
615
657
|
limit = DEFAULT_LIMIT,
|
|
@@ -617,84 +659,111 @@ export async function syncMentionThreads({
|
|
|
617
659
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
618
660
|
all = false,
|
|
619
661
|
maxPages,
|
|
620
|
-
}: {
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
try {
|
|
665
|
-
const fetchResult =
|
|
662
|
+
}: SyncMentionThreadsOptions) {
|
|
663
|
+
return Effect.gen(function* () {
|
|
664
|
+
const parsedMode = yield* trySync(() => parseMode(mode));
|
|
665
|
+
const parsedLimit = yield* trySync(() =>
|
|
666
|
+
assertPositiveInteger(limit, "--limit"),
|
|
667
|
+
);
|
|
668
|
+
const parsedDelayMs =
|
|
669
|
+
(yield* trySync(() => parseNonNegativeInteger(delayMs, "--delay-ms"))) ??
|
|
670
|
+
0;
|
|
671
|
+
const parsedTimeoutMs = yield* trySync(() =>
|
|
672
|
+
assertPositiveInteger(timeoutMs, "--timeout-ms"),
|
|
673
|
+
);
|
|
674
|
+
const parsedMaxPages = yield* trySync(() =>
|
|
675
|
+
parseNonNegativeInteger(maxPages, "--max-pages"),
|
|
676
|
+
);
|
|
677
|
+
const db = yield* trySync(() => getNativeDb());
|
|
678
|
+
const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
|
|
679
|
+
const mentions = yield* trySync(() =>
|
|
680
|
+
listRecentMentions(db, resolvedAccount.accountId, parsedLimit),
|
|
681
|
+
);
|
|
682
|
+
const mentionIds = mentions.map((mention) => mention.id);
|
|
683
|
+
const mentionIdSet = new Set(mentionIds);
|
|
684
|
+
const results: Array<{
|
|
685
|
+
tweetId: string;
|
|
686
|
+
conversationId?: string | null;
|
|
687
|
+
ok: boolean;
|
|
688
|
+
count: number;
|
|
689
|
+
strategy?: string;
|
|
690
|
+
pages?: number;
|
|
691
|
+
fallbackDepth?: number;
|
|
692
|
+
truncated?: boolean;
|
|
693
|
+
warnings?: string[];
|
|
694
|
+
error?: string;
|
|
695
|
+
}> = [];
|
|
696
|
+
let mergedTweets = 0;
|
|
697
|
+
let generalReadTweets = 0;
|
|
698
|
+
const uniqueTweetIds = new Set<string>();
|
|
699
|
+
const warnings: string[] = [];
|
|
700
|
+
|
|
701
|
+
for (const [index, mention] of mentions.entries()) {
|
|
702
|
+
if (index > 0 && parsedDelayMs > 0) {
|
|
703
|
+
yield* Effect.sleep(parsedDelayMs);
|
|
704
|
+
}
|
|
705
|
+
const fetchEffect: Effect.Effect<ThreadFetchResult, unknown, never> =
|
|
666
706
|
parsedMode === "bird"
|
|
667
|
-
? {
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
707
|
+
? listThreadViaBirdEffect({
|
|
708
|
+
tweetId: mention.id,
|
|
709
|
+
all,
|
|
710
|
+
maxPages: parsedMaxPages,
|
|
711
|
+
timeoutMs: parsedTimeoutMs,
|
|
712
|
+
}).pipe(
|
|
713
|
+
Effect.map((payload) => ({
|
|
714
|
+
strategy: "bird" as const,
|
|
715
|
+
payload,
|
|
716
|
+
pages: undefined,
|
|
717
|
+
fallbackDepth: undefined,
|
|
718
|
+
generalReadTweets: 0,
|
|
719
|
+
truncated: undefined,
|
|
720
|
+
warnings: [] as string[],
|
|
721
|
+
})),
|
|
722
|
+
)
|
|
723
|
+
: fetchThreadContextViaXurlEffect({
|
|
682
724
|
mention,
|
|
683
725
|
all,
|
|
684
726
|
maxPages: parsedMaxPages,
|
|
685
727
|
maxFallbackDepth: DEFAULT_FALLBACK_DEPTH,
|
|
686
728
|
timeoutMs: parsedTimeoutMs,
|
|
687
729
|
});
|
|
730
|
+
const fetched = yield* fetchEffect.pipe(
|
|
731
|
+
Effect.flatMap((fetchResult) =>
|
|
732
|
+
trySync(() =>
|
|
733
|
+
mergeMentionThreadIntoLocalStore({
|
|
734
|
+
db,
|
|
735
|
+
accountId: resolvedAccount.accountId,
|
|
736
|
+
accountHandle: resolvedAccount.handle,
|
|
737
|
+
mentionIds: mentionIdSet,
|
|
738
|
+
payload: fetchResult.payload,
|
|
739
|
+
source: parsedMode,
|
|
740
|
+
writeThreadContextEdges: parsedMode === "xurl",
|
|
741
|
+
}),
|
|
742
|
+
).pipe(Effect.as(fetchResult)),
|
|
743
|
+
),
|
|
744
|
+
Effect.map((fetchResult) => ({ ok: true as const, fetchResult })),
|
|
745
|
+
Effect.catchAll((error) =>
|
|
746
|
+
Effect.succeed({ ok: false as const, error }),
|
|
747
|
+
),
|
|
748
|
+
);
|
|
749
|
+
|
|
750
|
+
if (!fetched.ok) {
|
|
751
|
+
results.push({
|
|
752
|
+
tweetId: mention.id,
|
|
753
|
+
conversationId: mention.conversationId ?? null,
|
|
754
|
+
ok: false,
|
|
755
|
+
count: 0,
|
|
756
|
+
strategy: parsedMode,
|
|
757
|
+
error:
|
|
758
|
+
fetched.error instanceof Error
|
|
759
|
+
? fetched.error.message
|
|
760
|
+
: String(fetched.error),
|
|
761
|
+
});
|
|
762
|
+
continue;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
const { fetchResult } = fetched;
|
|
688
766
|
const { payload } = fetchResult;
|
|
689
|
-
mergeMentionThreadIntoLocalStore({
|
|
690
|
-
db,
|
|
691
|
-
accountId: resolvedAccount.accountId,
|
|
692
|
-
accountHandle: resolvedAccount.handle,
|
|
693
|
-
mentionIds: mentionIdSet,
|
|
694
|
-
payload,
|
|
695
|
-
source: parsedMode,
|
|
696
|
-
writeThreadContextEdges: parsedMode === "xurl",
|
|
697
|
-
});
|
|
698
767
|
for (const tweet of payload.data) {
|
|
699
768
|
uniqueTweetIds.add(tweet.id);
|
|
700
769
|
}
|
|
@@ -713,47 +782,42 @@ export async function syncMentionThreads({
|
|
|
713
782
|
warnings:
|
|
714
783
|
fetchResult.warnings.length > 0 ? fetchResult.warnings : undefined,
|
|
715
784
|
});
|
|
716
|
-
} catch (error) {
|
|
717
|
-
results.push({
|
|
718
|
-
tweetId: mention.id,
|
|
719
|
-
conversationId: mention.conversationId ?? null,
|
|
720
|
-
ok: false,
|
|
721
|
-
count: 0,
|
|
722
|
-
strategy: parsedMode,
|
|
723
|
-
error: error instanceof Error ? error.message : String(error),
|
|
724
|
-
});
|
|
725
785
|
}
|
|
726
|
-
}
|
|
727
786
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
787
|
+
const failures = results.filter((item) => !item.ok);
|
|
788
|
+
const skipped = results.filter((item) =>
|
|
789
|
+
item.strategy?.startsWith("skipped:"),
|
|
790
|
+
);
|
|
791
|
+
const partial = results.some((item) => item.truncated === true);
|
|
792
|
+
return {
|
|
793
|
+
ok: true,
|
|
794
|
+
source: parsedMode,
|
|
795
|
+
accountId: resolvedAccount.accountId,
|
|
796
|
+
mentions: mentionIds.length,
|
|
797
|
+
threads: results.length,
|
|
798
|
+
succeeded: results.length - failures.length - skipped.length,
|
|
799
|
+
skipped: skipped.length,
|
|
800
|
+
failed: failures.length,
|
|
801
|
+
mergedTweets,
|
|
802
|
+
uniqueTweets: uniqueTweetIds.size,
|
|
803
|
+
generalReadTweets: parsedMode === "xurl" ? generalReadTweets : 0,
|
|
804
|
+
partial,
|
|
805
|
+
options: {
|
|
806
|
+
mode: parsedMode,
|
|
807
|
+
limit: parsedLimit,
|
|
808
|
+
delayMs: parsedDelayMs,
|
|
809
|
+
timeoutMs: parsedTimeoutMs,
|
|
810
|
+
all,
|
|
811
|
+
maxPages: parsedMaxPages ?? null,
|
|
812
|
+
maxFallbackDepth: DEFAULT_FALLBACK_DEPTH,
|
|
813
|
+
},
|
|
814
|
+
results,
|
|
815
|
+
failures,
|
|
816
|
+
warnings,
|
|
817
|
+
};
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
export function syncMentionThreads(options: SyncMentionThreadsOptions) {
|
|
822
|
+
return runEffectPromise(syncMentionThreadsEffect(options));
|
|
759
823
|
}
|