birdclaw 0.8.3 → 0.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (284) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/README.md +10 -8
  3. package/bin/birdclaw.mjs +4 -66
  4. package/dist/cli/birdclaw.js +28164 -0
  5. package/dist/client/assets/AvatarChip-DLoqM5xD.js +1 -0
  6. package/dist/client/assets/ProfilePreview-DQX6_xc3.js +1 -0
  7. package/dist/client/assets/SavedTimelineView-BhSiVyR1.js +1 -0
  8. package/dist/client/assets/SmartTimestamp-BR0-9se2.js +1 -0
  9. package/dist/client/assets/TimelineRouteFrame-9NDLnXzH.js +1 -0
  10. package/dist/client/assets/blocks-CFp_njvG.js +1 -0
  11. package/dist/client/assets/bookmarks-LbYOsoKf.js +1 -0
  12. package/dist/client/assets/chunk-C0phanGV.js +1 -0
  13. package/dist/client/assets/circle-check-C_Vq3ElD.js +1 -0
  14. package/dist/client/assets/client-stream-contracts-sR4xMQxY.js +5 -0
  15. package/dist/client/assets/createLucideIcon-JV8isFFR.js +1 -0
  16. package/dist/client/assets/data-sources-euTr6T_L.js +1 -0
  17. package/dist/client/assets/discuss-BWKu_p7B.js +1 -0
  18. package/dist/client/assets/dms-SLXMtC4v.js +1 -0
  19. package/dist/client/assets/inbox-BybSI5Vf.js +1 -0
  20. package/dist/client/assets/index-DZVisyna.js +31 -0
  21. package/dist/client/assets/index-bpFrQ5yG.css +1 -0
  22. package/dist/client/assets/likes-dyh9p8y_.js +1 -0
  23. package/dist/client/assets/links-CIwqK_dX.js +1 -0
  24. package/dist/client/assets/mapbox-B4zqX7r0.js +2 -0
  25. package/dist/client/assets/mapbox-gl-DYNcLcll.js +3215 -0
  26. package/dist/client/assets/mentions-BoVVAIT7.js +1 -0
  27. package/dist/client/assets/network-map-zRysIY_b.js +2 -0
  28. package/dist/client/assets/profile-analyze-C_yDeX6-.js +1 -0
  29. package/dist/client/assets/profile-hydration-client-YEtJsNTv.js +1 -0
  30. package/dist/client/assets/profiles._handle-DdIVPqU-.js +1 -0
  31. package/dist/client/assets/query-client-fYu2rzRC.js +17 -0
  32. package/dist/client/assets/rate-limits-BWPmPOSh.js +1 -0
  33. package/dist/client/assets/react-dom-DHl6Ug7F.js +1 -0
  34. package/dist/client/assets/react-nBylq1U5.js +1 -0
  35. package/dist/client/assets/repeat-2-xCwHM9hD.js +1 -0
  36. package/dist/client/assets/routes-CiLuYpZr.js +1 -0
  37. package/dist/client/assets/schemas-I9r6F6ya.js +64 -0
  38. package/dist/client/assets/search-Dq5kYsxA.js +1 -0
  39. package/dist/client/assets/sparkles-CQVlvHsZ.js +1 -0
  40. package/dist/client/assets/styles-B8s3tt3r.css +2 -0
  41. package/dist/client/assets/today-BoO64YoY.js +1 -0
  42. package/dist/client/assets/triangle-alert-CBL7s2pf.js +1 -0
  43. package/dist/client/assets/useTimelineRouteData-BaO3Rna1.js +1 -0
  44. package/dist/client/assets/users-CHbKlqUr.js +1 -0
  45. package/dist/server/assets/AvatarChip-zUDAEmjL.js +105 -0
  46. package/dist/server/assets/FeedState-ntW0Q2GB.js +214 -0
  47. package/dist/server/assets/ProfileAnalysisStream-L7bs6IHH.js +198 -0
  48. package/dist/server/assets/ProfilePreview-Cg-AOHBE.js +751 -0
  49. package/dist/server/assets/SavedTimelineView-BtCq1dHk.js +64 -0
  50. package/dist/server/assets/SmartTimestamp-DfwE1NGf.js +45 -0
  51. package/dist/server/assets/SyncNowButton-BOn_IBXS.js +76 -0
  52. package/dist/server/assets/TimelineRouteFrame-CjMIh63G.js +85 -0
  53. package/dist/server/assets/TweetRichText-DDywESRF.js +74 -0
  54. package/dist/server/assets/_tanstack-start-manifest_v-aLWUdbAc.js +148 -0
  55. package/dist/server/assets/account-selection-CTdFGWRk.js +44 -0
  56. package/dist/server/assets/blocks-C1mgGTB3.js +299 -0
  57. package/dist/server/assets/blocks-DSgQghqP.js +17 -0
  58. package/dist/server/assets/bookmarks-D_YkmCK3.js +14 -0
  59. package/dist/server/assets/client-stream-contracts-Bq6kMmhh.js +769 -0
  60. package/dist/server/assets/data-sources-BnA1Q94p.js +180 -0
  61. package/dist/server/assets/discuss-C1QMBBSk.js +338 -0
  62. package/dist/server/assets/discuss-DAq6ehL9.js +15 -0
  63. package/dist/server/assets/dms-BuLj4bp8.js +368 -0
  64. package/dist/server/assets/dms-DAyQrVM2.js +240 -0
  65. package/dist/server/assets/empty-plugin-adapters-DzrEkdBF.js +5 -0
  66. package/dist/server/assets/inbox-CkGOaW1q.js +214 -0
  67. package/dist/server/assets/inbox-DpWd1EzC.js +146 -0
  68. package/dist/server/assets/likes-BMm2JWYI.js +14 -0
  69. package/dist/server/assets/links-Da4tjWtI.js +468 -0
  70. package/dist/server/assets/links-dqXb35YM.js +240 -0
  71. package/dist/server/assets/mentions-CEyEcOFw.js +26 -0
  72. package/dist/server/assets/network-map-D3CzRcC4.js +224 -0
  73. package/dist/server/assets/network-map-D5DoLOwR.js +582 -0
  74. package/dist/server/assets/profile-analyze-CGCj4VFd.js +90 -0
  75. package/dist/server/assets/profile-analyze-DH1TSBkT.js +9 -0
  76. package/dist/server/assets/profile-hydration-client-GJ4kwJS8.js +133 -0
  77. package/dist/server/assets/profiles._handle-B5V-huoo.js +12 -0
  78. package/dist/server/assets/profiles._handle-CJ6rgzt8.js +184 -0
  79. package/dist/server/assets/query-client-BdauHBgd.js +923 -0
  80. package/dist/server/assets/rate-limits-VAr2qzYD.js +262 -0
  81. package/dist/server/assets/route-search-BUlQ5lGM.js +117 -0
  82. package/dist/server/assets/router-C4cz9ypd.js +17026 -0
  83. package/dist/server/assets/routes-B9aikOdB.js +26 -0
  84. package/dist/server/assets/start-FaDOTmto.js +4 -0
  85. package/dist/server/assets/today-B_TA16QG.js +303 -0
  86. package/dist/server/assets/today-JkxZXdBt.js +17 -0
  87. package/dist/server/assets/ui-qbmcMSSX.js +137 -0
  88. package/{src/components/useDebouncedValue.ts → dist/server/assets/useDebouncedValue-DOnA6Xcy.js} +4 -4
  89. package/dist/server/assets/useTimelineRouteData-BLXvMGb2.js +1101 -0
  90. package/dist/server/server.js +5520 -0
  91. package/package.json +26 -34
  92. package/playwright.config.ts +0 -35
  93. package/scripts/browser-perf.mjs +0 -427
  94. package/scripts/build-docs-site.mjs +0 -967
  95. package/scripts/docs-site-assets.mjs +0 -311
  96. package/scripts/run-vitest.mjs +0 -21
  97. package/scripts/sanitize-node-options.mjs +0 -23
  98. package/scripts/start-test-server.mjs +0 -42
  99. package/src/cli/command-context.ts +0 -17
  100. package/src/cli/register-analysis.ts +0 -500
  101. package/src/cli/register-compose.ts +0 -40
  102. package/src/cli/register-graph.ts +0 -132
  103. package/src/cli/register-inbox.ts +0 -41
  104. package/src/cli/register-storage.ts +0 -106
  105. package/src/cli-moderation.ts +0 -210
  106. package/src/cli.ts +0 -1616
  107. package/src/components/AccountSwitcher.tsx +0 -178
  108. package/src/components/AppNav.tsx +0 -118
  109. package/src/components/AvatarChip.tsx +0 -58
  110. package/src/components/AvatarPreload.ts +0 -149
  111. package/src/components/BrandMark.tsx +0 -67
  112. package/src/components/ConversationThread.tsx +0 -131
  113. package/src/components/DmWorkspace.tsx +0 -286
  114. package/src/components/EmbeddedTweetCard.tsx +0 -55
  115. package/src/components/FeedState.tsx +0 -147
  116. package/src/components/FloatingPreview.tsx +0 -382
  117. package/src/components/InboxCard.tsx +0 -152
  118. package/src/components/LinkPreviewCard.tsx +0 -292
  119. package/src/components/MarkdownCitations.tsx +0 -680
  120. package/src/components/MarkdownViewer.tsx +0 -111
  121. package/src/components/ProfileAnalysisClient.ts +0 -191
  122. package/src/components/ProfileAnalysisStream.tsx +0 -259
  123. package/src/components/ProfilePreview.tsx +0 -131
  124. package/src/components/SavedTimelineView.tsx +0 -155
  125. package/src/components/SmartTimestamp.tsx +0 -72
  126. package/src/components/SyncNowButton.tsx +0 -140
  127. package/src/components/ThemeSlider.tsx +0 -80
  128. package/src/components/TimelineCard.tsx +0 -533
  129. package/src/components/TimelineRouteFrame.tsx +0 -186
  130. package/src/components/TweetArticleCard.tsx +0 -66
  131. package/src/components/TweetMediaGrid.tsx +0 -185
  132. package/src/components/TweetRichText.tsx +0 -159
  133. package/src/components/account-selection.ts +0 -64
  134. package/src/components/links-controller.ts +0 -162
  135. package/src/components/links-model.ts +0 -198
  136. package/src/components/network-map-controller.ts +0 -84
  137. package/src/components/network-map-model.ts +0 -255
  138. package/src/components/useTimelineRouteData.ts +0 -193
  139. package/src/lib/account-sync-job.ts +0 -666
  140. package/src/lib/actions-transport.ts +0 -300
  141. package/src/lib/analysis-runtime.ts +0 -238
  142. package/src/lib/api-client.ts +0 -220
  143. package/src/lib/api-contracts.ts +0 -328
  144. package/src/lib/archive-finder.ts +0 -185
  145. package/src/lib/archive-import-plan.ts +0 -102
  146. package/src/lib/archive-import.ts +0 -2695
  147. package/src/lib/authored-live.ts +0 -1027
  148. package/src/lib/avatar-cache.ts +0 -349
  149. package/src/lib/backup.ts +0 -2652
  150. package/src/lib/bird-actions.ts +0 -294
  151. package/src/lib/bird-command.ts +0 -130
  152. package/src/lib/bird.ts +0 -1140
  153. package/src/lib/blocklist.ts +0 -136
  154. package/src/lib/blocks-write.ts +0 -171
  155. package/src/lib/blocks.ts +0 -387
  156. package/src/lib/bookmark-sync-job.ts +0 -536
  157. package/src/lib/config.ts +0 -224
  158. package/src/lib/conversation-surface.ts +0 -205
  159. package/src/lib/data-sources.ts +0 -219
  160. package/src/lib/database-metrics.ts +0 -88
  161. package/src/lib/database-migrations.ts +0 -34
  162. package/src/lib/database-schema.ts +0 -312
  163. package/src/lib/database-writer.ts +0 -69
  164. package/src/lib/db.ts +0 -889
  165. package/src/lib/dm-read-model.ts +0 -533
  166. package/src/lib/dms-live.ts +0 -870
  167. package/src/lib/effect-runtime.ts +0 -45
  168. package/src/lib/follow-graph.ts +0 -1087
  169. package/src/lib/geocoding.ts +0 -296
  170. package/src/lib/http-effect.ts +0 -222
  171. package/src/lib/identity-search-index.ts +0 -369
  172. package/src/lib/import-repository.ts +0 -138
  173. package/src/lib/inbox.ts +0 -242
  174. package/src/lib/link-index.ts +0 -641
  175. package/src/lib/link-insights.ts +0 -860
  176. package/src/lib/link-preview-metadata.ts +0 -754
  177. package/src/lib/live-sync-engine.ts +0 -209
  178. package/src/lib/live-transport-gateway.ts +0 -128
  179. package/src/lib/location.ts +0 -137
  180. package/src/lib/media-fetch.ts +0 -860
  181. package/src/lib/media-includes.ts +0 -165
  182. package/src/lib/mention-threads-live.ts +0 -830
  183. package/src/lib/mentions-export.ts +0 -147
  184. package/src/lib/mentions-live.ts +0 -1114
  185. package/src/lib/moderation-target.ts +0 -227
  186. package/src/lib/moderation-write.ts +0 -131
  187. package/src/lib/mutes-write.ts +0 -171
  188. package/src/lib/mutes.ts +0 -84
  189. package/src/lib/ndjson-stream.ts +0 -106
  190. package/src/lib/network-map.ts +0 -382
  191. package/src/lib/openai-response-runtime.ts +0 -251
  192. package/src/lib/openai.ts +0 -117
  193. package/src/lib/paginated-sync.ts +0 -93
  194. package/src/lib/period-digest.ts +0 -1364
  195. package/src/lib/present.ts +0 -118
  196. package/src/lib/profile-affiliation-hydration.ts +0 -228
  197. package/src/lib/profile-affiliations.ts +0 -262
  198. package/src/lib/profile-analysis.ts +0 -1198
  199. package/src/lib/profile-bio-entities.ts +0 -318
  200. package/src/lib/profile-history.ts +0 -164
  201. package/src/lib/profile-hydration.ts +0 -180
  202. package/src/lib/profile-replies.ts +0 -77
  203. package/src/lib/profile-resolver.ts +0 -650
  204. package/src/lib/queries.ts +0 -6
  205. package/src/lib/query-actions.ts +0 -437
  206. package/src/lib/query-client.tsx +0 -47
  207. package/src/lib/query-read-model-shared.ts +0 -52
  208. package/src/lib/query-read-models.ts +0 -5
  209. package/src/lib/query-resource.ts +0 -41
  210. package/src/lib/query-status.ts +0 -164
  211. package/src/lib/research.ts +0 -652
  212. package/src/lib/runtime-services.ts +0 -20
  213. package/src/lib/search-discussion.ts +0 -812
  214. package/src/lib/seed.ts +0 -613
  215. package/src/lib/server-runtime-services.ts +0 -30
  216. package/src/lib/sqlite.ts +0 -183
  217. package/src/lib/streaming-ingestion.ts +0 -240
  218. package/src/lib/sync-cache.ts +0 -70
  219. package/src/lib/sync-plan.ts +0 -175
  220. package/src/lib/theme-transition.ts +0 -113
  221. package/src/lib/theme.tsx +0 -165
  222. package/src/lib/timeline-collections-live.ts +0 -407
  223. package/src/lib/timeline-live.ts +0 -302
  224. package/src/lib/timeline-read-model.ts +0 -1191
  225. package/src/lib/tweet-account-edges.ts +0 -48
  226. package/src/lib/tweet-lookup.ts +0 -46
  227. package/src/lib/tweet-render.ts +0 -430
  228. package/src/lib/tweet-repository.ts +0 -156
  229. package/src/lib/tweet-search-live.ts +0 -462
  230. package/src/lib/types.ts +0 -723
  231. package/src/lib/ui.ts +0 -529
  232. package/src/lib/url-expansion-store.ts +0 -110
  233. package/src/lib/url-expansion.ts +0 -343
  234. package/src/lib/url-safety.ts +0 -220
  235. package/src/lib/web-sync.ts +0 -533
  236. package/src/lib/whois.ts +0 -1080
  237. package/src/lib/x-profile.ts +0 -365
  238. package/src/lib/x-web.ts +0 -199
  239. package/src/lib/xurl-rate-limits.ts +0 -267
  240. package/src/lib/xurl.ts +0 -1677
  241. package/src/routeTree.gen.ts +0 -723
  242. package/src/router.tsx +0 -20
  243. package/src/routes/__root.tsx +0 -82
  244. package/src/routes/api/action.tsx +0 -147
  245. package/src/routes/api/avatar.tsx +0 -50
  246. package/src/routes/api/blocks.tsx +0 -35
  247. package/src/routes/api/conversation.tsx +0 -45
  248. package/src/routes/api/data-sources.tsx +0 -24
  249. package/src/routes/api/inbox.tsx +0 -43
  250. package/src/routes/api/link-insights.tsx +0 -85
  251. package/src/routes/api/link-preview.tsx +0 -47
  252. package/src/routes/api/network-map.tsx +0 -55
  253. package/src/routes/api/period-digest.tsx +0 -101
  254. package/src/routes/api/profile-analysis.tsx +0 -97
  255. package/src/routes/api/profile-hydrate.tsx +0 -54
  256. package/src/routes/api/query.tsx +0 -121
  257. package/src/routes/api/search-discussion.tsx +0 -113
  258. package/src/routes/api/status.tsx +0 -30
  259. package/src/routes/api/sync.tsx +0 -108
  260. package/src/routes/api/xurl-rate-limits.tsx +0 -24
  261. package/src/routes/blocks.tsx +0 -343
  262. package/src/routes/bookmarks.tsx +0 -18
  263. package/src/routes/data-sources.tsx +0 -249
  264. package/src/routes/discuss.tsx +0 -419
  265. package/src/routes/dms.tsx +0 -415
  266. package/src/routes/inbox.tsx +0 -232
  267. package/src/routes/index.tsx +0 -32
  268. package/src/routes/likes.tsx +0 -18
  269. package/src/routes/links.tsx +0 -631
  270. package/src/routes/mentions.tsx +0 -32
  271. package/src/routes/network-map.tsx +0 -732
  272. package/src/routes/profile-analyze.tsx +0 -112
  273. package/src/routes/profiles.$handle.tsx +0 -228
  274. package/src/routes/rate-limits.tsx +0 -305
  275. package/src/routes/today.tsx +0 -511
  276. package/src/styles.css +0 -257
  277. package/tsconfig.json +0 -29
  278. package/vite.config.ts +0 -29
  279. /package/{public → dist/client}/birdclaw-mark.png +0 -0
  280. /package/{public → dist/client}/favicon.ico +0 -0
  281. /package/{public → dist/client}/logo192.png +0 -0
  282. /package/{public → dist/client}/logo512.png +0 -0
  283. /package/{public → dist/client}/manifest.json +0 -0
  284. /package/{public → dist/client}/robots.txt +0 -0
@@ -1,830 +0,0 @@
1
- import type { Database } from "./sqlite";
2
- import { Effect } from "effect";
3
- import { databaseWriteEffect } from "./database-writer";
4
- import { getNativeDb } from "./db";
5
- import { runEffectPromise } from "./effect-runtime";
6
- import { liveTransportGateway } from "./live-transport-gateway";
7
- import { resolveLiveSyncAccount } from "./live-sync-engine";
8
- import { runSyncPlanEffect } from "./sync-plan";
9
- import type {
10
- XurlMentionData,
11
- XurlMentionsResponse,
12
- XurlMentionUser,
13
- XurlMediaItem,
14
- XurlTweetsResponse,
15
- } from "./types";
16
- import { ingestTweetPayload } from "./tweet-repository";
17
-
18
- const DEFAULT_LIMIT = 30;
19
- const DEFAULT_DELAY_MS = 1500;
20
- const DEFAULT_TIMEOUT_MS = 15_000;
21
- const DEFAULT_MODE = "bird";
22
- const DEFAULT_FALLBACK_DEPTH = 12;
23
- const MAX_XURL_SEARCH_RESULTS = 100;
24
-
25
- export type MentionThreadsMode = "bird" | "xurl";
26
- export interface MentionThreadsProgress {
27
- source: MentionThreadsMode;
28
- processed: number;
29
- total: number;
30
- fetched: number;
31
- done: boolean;
32
- }
33
- export interface SyncMentionThreadsOptions {
34
- account?: string;
35
- mode?: string;
36
- limit?: number;
37
- tweetIds?: string[];
38
- delayMs?: number;
39
- timeoutMs?: number;
40
- all?: boolean;
41
- maxPages?: number;
42
- onProgress?: (progress: MentionThreadsProgress) => void;
43
- }
44
-
45
- interface LocalMention {
46
- id: string;
47
- replyToId?: string;
48
- conversationId?: string;
49
- rawTweet?: XurlMentionData;
50
- }
51
- interface ThreadFetchResult {
52
- strategy: string;
53
- payload: XurlMentionsResponse;
54
- pages?: number;
55
- fallbackDepth?: number;
56
- generalReadTweets: number;
57
- truncated?: boolean;
58
- warnings: string[];
59
- }
60
-
61
- function assertPositiveInteger(value: number, name: string) {
62
- if (!Number.isFinite(value) || value < 1) {
63
- throw new Error(`${name} must be at least 1`);
64
- }
65
- return Math.floor(value);
66
- }
67
-
68
- function parseNonNegativeInteger(value: number | undefined, name: string) {
69
- if (value === undefined) {
70
- return undefined;
71
- }
72
- if (!Number.isFinite(value) || value < 0) {
73
- throw new Error(`${name} must be non-negative`);
74
- }
75
- return Math.floor(value);
76
- }
77
-
78
- function parseMode(value: string | undefined): MentionThreadsMode {
79
- const mode = value ?? DEFAULT_MODE;
80
- if (mode !== "bird" && mode !== "xurl") {
81
- throw new Error("--mode must be bird or xurl");
82
- }
83
- return mode;
84
- }
85
-
86
- function toError(error: unknown) {
87
- return error instanceof Error ? error : new Error(String(error));
88
- }
89
-
90
- function trySync<T>(try_: () => T) {
91
- return Effect.try({
92
- try: try_,
93
- catch: toError,
94
- });
95
- }
96
-
97
- function getRemainingThreadTimeoutMs(
98
- deadlineMs: number,
99
- originalTimeoutMs: number,
100
- ) {
101
- const remainingMs = deadlineMs - Date.now();
102
- if (remainingMs <= 0) {
103
- throw new Error(
104
- `xurl thread timed out after ${String(originalTimeoutMs)}ms`,
105
- );
106
- }
107
- return remainingMs;
108
- }
109
- function getReplyToId(tweet: XurlMentionData) {
110
- return tweet.referenced_tweets?.find((entry) => entry.type === "replied_to")
111
- ?.id;
112
- }
113
-
114
- function mergePayloads(pages: XurlTweetsResponse[]): XurlMentionsResponse {
115
- const tweets: XurlMentionData[] = [];
116
- const seenTweetIds = new Set<string>();
117
- const users: XurlMentionUser[] = [];
118
- const seenUserIds = new Set<string>();
119
- const media: XurlMediaItem[] = [];
120
- const seenMediaKeys = new Set<string>();
121
-
122
- for (const page of pages) {
123
- for (const tweet of page.data) {
124
- if (seenTweetIds.has(tweet.id)) {
125
- continue;
126
- }
127
- seenTweetIds.add(tweet.id);
128
- tweets.push(tweet);
129
- }
130
-
131
- for (const user of page.includes?.users ?? []) {
132
- if (seenUserIds.has(user.id)) {
133
- continue;
134
- }
135
- seenUserIds.add(user.id);
136
- users.push(user);
137
- }
138
-
139
- for (const item of page.includes?.media ?? []) {
140
- if (seenMediaKeys.has(item.media_key)) {
141
- continue;
142
- }
143
- seenMediaKeys.add(item.media_key);
144
- media.push(item);
145
- }
146
- }
147
-
148
- const lastMeta = pages.at(-1)?.meta;
149
- return {
150
- data: tweets,
151
- includes:
152
- users.length > 0 || media.length > 0
153
- ? {
154
- ...(users.length > 0 ? { users } : {}),
155
- ...(media.length > 0 ? { media } : {}),
156
- }
157
- : undefined,
158
- meta: {
159
- ...lastMeta,
160
- result_count: tweets.length,
161
- page_count: pages.length,
162
- next_token:
163
- typeof lastMeta?.next_token === "string" ? lastMeta.next_token : null,
164
- },
165
- };
166
- }
167
-
168
- function parseRawTweet(value: string | null | undefined) {
169
- if (!value || value === "{}" || value === "null") {
170
- return undefined;
171
- }
172
-
173
- try {
174
- const parsed = JSON.parse(value) as unknown;
175
- if (parsed && typeof parsed === "object") {
176
- return parsed as XurlMentionData;
177
- }
178
- } catch {
179
- return undefined;
180
- }
181
-
182
- return undefined;
183
- }
184
-
185
- function listRecentMentions(
186
- db: Database,
187
- accountId: string,
188
- limit: number,
189
- ): LocalMention[] {
190
- const rows = db
191
- .prepare(
192
- `
193
- with local_mentions as (
194
- select
195
- t.id,
196
- t.created_at as createdAt,
197
- t.reply_to_id as replyToId,
198
- edge.raw_json as rawJson
199
- from tweet_account_edges edge
200
- join tweets t on t.id = edge.tweet_id
201
- where edge.kind = 'mention' and edge.account_id = ?
202
- union all
203
- select
204
- t.id,
205
- t.created_at as createdAt,
206
- t.reply_to_id as replyToId,
207
- '{}' as rawJson
208
- from tweets t
209
- where t.kind = 'mention' and t.account_id = ?
210
- and not exists (
211
- select 1
212
- from tweet_account_edges edge
213
- where edge.account_id = t.account_id
214
- and edge.tweet_id = t.id
215
- and edge.kind = 'mention'
216
- )
217
- )
218
- select id, createdAt, replyToId, rawJson
219
- from local_mentions
220
- order by createdAt desc
221
- limit ?
222
- `,
223
- )
224
- .all(accountId, accountId, limit) as Array<{
225
- id: string;
226
- createdAt: string;
227
- replyToId: string | null;
228
- rawJson: string | null;
229
- }>;
230
-
231
- return rows.map((row) => {
232
- const rawTweet = parseRawTweet(row.rawJson);
233
- return {
234
- id: row.id,
235
- replyToId:
236
- row.replyToId ?? (rawTweet ? getReplyToId(rawTweet) : undefined),
237
- conversationId:
238
- typeof rawTweet?.conversation_id === "string"
239
- ? rawTweet.conversation_id
240
- : undefined,
241
- rawTweet,
242
- };
243
- });
244
- }
245
-
246
- function listMentionsByIds(
247
- db: Database,
248
- accountId: string,
249
- tweetIds: string[],
250
- limit: number,
251
- ): LocalMention[] {
252
- const ids = [...new Set(tweetIds.filter((id) => id.trim().length > 0))].slice(
253
- 0,
254
- limit,
255
- );
256
- if (ids.length === 0) return [];
257
- const placeholders = ids.map(() => "?").join(", ");
258
- const rows = db
259
- .prepare(
260
- `
261
- select
262
- t.id,
263
- t.created_at as createdAt,
264
- t.reply_to_id as replyToId,
265
- coalesce(edge.raw_json, '{}') as rawJson
266
- from tweets t
267
- left join tweet_account_edges edge
268
- on edge.tweet_id = t.id
269
- and edge.account_id = ?
270
- and edge.kind = 'mention'
271
- where t.id in (${placeholders})
272
- and (
273
- edge.tweet_id is not null
274
- or (t.kind = 'mention' and t.account_id = ?)
275
- )
276
- order by t.created_at desc
277
- limit ?
278
- `,
279
- )
280
- .all(accountId, ...ids, accountId, limit) as Array<{
281
- id: string;
282
- createdAt: string;
283
- replyToId: string | null;
284
- rawJson: string | null;
285
- }>;
286
-
287
- return rows.map((row) => {
288
- const rawTweet = parseRawTweet(row.rawJson);
289
- return {
290
- id: row.id,
291
- replyToId:
292
- row.replyToId ?? (rawTweet ? getReplyToId(rawTweet) : undefined),
293
- conversationId:
294
- typeof rawTweet?.conversation_id === "string"
295
- ? rawTweet.conversation_id
296
- : undefined,
297
- rawTweet,
298
- };
299
- });
300
- }
301
-
302
- function mergeMentionThreadIntoLocalStore({
303
- db,
304
- accountId,
305
- accountHandle,
306
- mentionIds,
307
- payload,
308
- source = "bird",
309
- writeThreadContextEdges = false,
310
- }: {
311
- db: Database;
312
- accountId: string;
313
- accountHandle: string;
314
- mentionIds: Set<string>;
315
- payload: XurlMentionsResponse;
316
- source?: "bird" | "xurl";
317
- writeThreadContextEdges?: boolean;
318
- }) {
319
- const usersById = new Map(
320
- (payload.includes?.users ?? []).map((user) => [user.id, user]),
321
- );
322
- const groups = new Map<"mention" | "home" | "thread", typeof payload.data>([
323
- ["mention", []],
324
- ["home", []],
325
- ["thread", []],
326
- ]);
327
- for (const tweet of payload.data) {
328
- const handle = usersById.get(tweet.author_id)?.username.toLowerCase();
329
- const kind = mentionIds.has(tweet.id)
330
- ? "mention"
331
- : handle === accountHandle
332
- ? "home"
333
- : "thread";
334
- groups.get(kind)?.push(tweet);
335
- }
336
- for (const [kind, data] of groups) {
337
- if (data.length === 0) continue;
338
- ingestTweetPayload(db, {
339
- accountId,
340
- payload: { ...payload, data },
341
- kind,
342
- edgeKind: writeThreadContextEdges ? "thread_context" : undefined,
343
- markRepliesAsReplied: true,
344
- replaceSecondaryKind: true,
345
- source,
346
- });
347
- }
348
- }
349
-
350
- function fetchConversationViaRecentSearchEffect({
351
- conversationId,
352
- all,
353
- maxPages,
354
- timeoutMs,
355
- deadlineMs,
356
- }: {
357
- conversationId: string;
358
- all: boolean;
359
- maxPages?: number;
360
- timeoutMs: number;
361
- deadlineMs: number;
362
- }) {
363
- return Effect.gen(function* () {
364
- const result = yield* runSyncPlanEffect({
365
- fetchPage: ({ cursor }) =>
366
- trySync(() => getRemainingThreadTimeoutMs(deadlineMs, timeoutMs)).pipe(
367
- Effect.flatMap((remainingTimeoutMs) =>
368
- liveTransportGateway.xurl.searchConversation(conversationId, {
369
- maxResults: MAX_XURL_SEARCH_RESULTS,
370
- paginationToken: cursor,
371
- timeoutMs: remainingTimeoutMs,
372
- }),
373
- ),
374
- ),
375
- getNextCursor: (page) =>
376
- typeof page.meta?.next_token === "string"
377
- ? page.meta.next_token
378
- : undefined,
379
- maxPages: all || maxPages !== undefined ? maxPages : 1,
380
- });
381
- const payload = mergePayloads(result.pages);
382
- const paginationRequested = all || maxPages !== undefined;
383
- return {
384
- payload,
385
- pages: result.pages.length,
386
- truncated: paginationRequested && Boolean(result.nextCursor),
387
- generalReadTweets: payload.data.length,
388
- };
389
- });
390
- }
391
-
392
- function fetchParentChainViaXurlEffect({
393
- mention,
394
- maxDepth,
395
- timeoutMs,
396
- deadlineMs,
397
- }: {
398
- mention: LocalMention;
399
- maxDepth: number;
400
- timeoutMs: number;
401
- deadlineMs: number;
402
- }) {
403
- return Effect.gen(function* () {
404
- const pages: XurlTweetsResponse[] = [];
405
- const warnings: string[] = [];
406
- const seenTweetIds = new Set([mention.id]);
407
- let nextParentId = mention.replyToId;
408
- let fallbackDepth = 0;
409
- let generalReadTweets = 0;
410
-
411
- const rawAnchorPayload =
412
- mention.rawTweet && mention.rawTweet.id === mention.id
413
- ? ({ data: [mention.rawTweet] } satisfies XurlTweetsResponse)
414
- : undefined;
415
- let shouldUseRawAnchor = Boolean(rawAnchorPayload);
416
-
417
- if (!nextParentId) {
418
- const anchorPayload = yield* liveTransportGateway.xurl.getTweetById(
419
- mention.id,
420
- {
421
- timeoutMs: yield* trySync(() =>
422
- getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
423
- ),
424
- },
425
- );
426
- pages.push(anchorPayload);
427
- generalReadTweets += anchorPayload.data.length;
428
- const anchorTweet = anchorPayload.data[0];
429
- if (anchorTweet) {
430
- shouldUseRawAnchor = false;
431
- seenTweetIds.add(anchorTweet.id);
432
- nextParentId = anchorTweet.in_reply_to_user_id
433
- ? getReplyToId(anchorTweet)
434
- : undefined;
435
- }
436
- }
437
-
438
- if (shouldUseRawAnchor && rawAnchorPayload) {
439
- pages.unshift(rawAnchorPayload);
440
- }
441
-
442
- while (nextParentId) {
443
- if (fallbackDepth >= maxDepth) {
444
- warnings.push(
445
- `fallback parent-chain depth cap reached for ${mention.id} after ${maxDepth} hops`,
446
- );
447
- break;
448
- }
449
- if (seenTweetIds.has(nextParentId)) {
450
- warnings.push(
451
- `fallback parent-chain cycle detected for ${mention.id} at ${nextParentId}`,
452
- );
453
- break;
454
- }
455
-
456
- fallbackDepth += 1;
457
- const parentPayload = yield* liveTransportGateway.xurl.getTweetById(
458
- nextParentId,
459
- {
460
- timeoutMs: yield* trySync(() =>
461
- getRemainingThreadTimeoutMs(deadlineMs, timeoutMs),
462
- ),
463
- },
464
- );
465
- pages.push(parentPayload);
466
- generalReadTweets += parentPayload.data.length;
467
- const parentTweet = parentPayload.data[0];
468
- if (!parentTweet) {
469
- break;
470
- }
471
- seenTweetIds.add(parentTweet.id);
472
- nextParentId = parentTweet.in_reply_to_user_id
473
- ? getReplyToId(parentTweet)
474
- : undefined;
475
- }
476
-
477
- const payload = mergePayloads(pages);
478
- return {
479
- payload,
480
- fallbackDepth,
481
- warnings,
482
- generalReadTweets,
483
- };
484
- });
485
- }
486
-
487
- function findMissingAncestorId(
488
- mention: LocalMention,
489
- payload: XurlMentionsResponse,
490
- ) {
491
- const tweetsById = new Map(payload.data.map((tweet) => [tweet.id, tweet]));
492
- const seenTweetIds = new Set<string>([mention.id]);
493
- const anchorTweet = tweetsById.get(mention.id) ?? mention.rawTweet;
494
- let nextParentId = anchorTweet
495
- ? getReplyToId(anchorTweet)
496
- : mention.replyToId;
497
-
498
- while (nextParentId) {
499
- if (seenTweetIds.has(nextParentId)) {
500
- return undefined;
501
- }
502
- const parentTweet = tweetsById.get(nextParentId);
503
- if (!parentTweet) {
504
- return nextParentId;
505
- }
506
- seenTweetIds.add(parentTweet.id);
507
- nextParentId = parentTweet.in_reply_to_user_id
508
- ? getReplyToId(parentTweet)
509
- : undefined;
510
- }
511
-
512
- if (
513
- mention.conversationId &&
514
- mention.conversationId !== mention.id &&
515
- !tweetsById.has(mention.conversationId)
516
- ) {
517
- return mention.conversationId;
518
- }
519
-
520
- return undefined;
521
- }
522
-
523
- function fetchThreadContextViaXurlEffect({
524
- mention,
525
- all,
526
- maxPages,
527
- maxFallbackDepth,
528
- timeoutMs,
529
- }: {
530
- mention: LocalMention;
531
- all: boolean;
532
- maxPages?: number;
533
- maxFallbackDepth: number;
534
- timeoutMs: number;
535
- }) {
536
- return Effect.gen(function* () {
537
- const deadlineMs = Date.now() + timeoutMs;
538
- if (!mention.conversationId) {
539
- if (mention.replyToId) {
540
- const fallback = yield* fetchParentChainViaXurlEffect({
541
- mention,
542
- maxDepth: maxFallbackDepth,
543
- timeoutMs,
544
- deadlineMs,
545
- });
546
- return {
547
- strategy: "parent_walk" as const,
548
- pages: 0,
549
- truncated: false,
550
- payload: fallback.payload,
551
- fallbackDepth: fallback.fallbackDepth,
552
- generalReadTweets: fallback.generalReadTweets,
553
- warnings: [
554
- `missing conversation_id for ${mention.id}; used parent walk`,
555
- ...fallback.warnings,
556
- ],
557
- };
558
- }
559
- return {
560
- strategy: "skipped:no_conversation_id" as const,
561
- payload: { data: [] } satisfies XurlMentionsResponse,
562
- pages: 0,
563
- fallbackDepth: 0,
564
- generalReadTweets: 0,
565
- warnings: [`skipped ${mention.id}: missing conversation_id`],
566
- truncated: false,
567
- };
568
- }
569
-
570
- const search = yield* fetchConversationViaRecentSearchEffect({
571
- conversationId: mention.conversationId,
572
- all,
573
- maxPages,
574
- timeoutMs,
575
- deadlineMs,
576
- });
577
- if (search.payload.data.length > 0) {
578
- const missingAncestorId = findMissingAncestorId(mention, search.payload);
579
- if (missingAncestorId) {
580
- const fallback = yield* fetchParentChainViaXurlEffect({
581
- mention,
582
- maxDepth: maxFallbackDepth,
583
- timeoutMs,
584
- deadlineMs,
585
- });
586
- return {
587
- strategy: "conversation_search+parent_walk" as const,
588
- pages: search.pages,
589
- truncated: search.truncated,
590
- payload: mergePayloads([search.payload, fallback.payload]),
591
- fallbackDepth: fallback.fallbackDepth,
592
- generalReadTweets:
593
- search.generalReadTweets + fallback.generalReadTweets,
594
- warnings: [
595
- `recent search missed ancestor ${missingAncestorId} for conversation ${mention.conversationId}; used parent walk`,
596
- ...fallback.warnings,
597
- ],
598
- };
599
- }
600
- return {
601
- strategy: "conversation_search" as const,
602
- fallbackDepth: 0,
603
- warnings: [] as string[],
604
- ...search,
605
- };
606
- }
607
-
608
- const fallback = yield* fetchParentChainViaXurlEffect({
609
- mention,
610
- maxDepth: maxFallbackDepth,
611
- timeoutMs,
612
- deadlineMs,
613
- });
614
- return {
615
- strategy: "parent_walk" as const,
616
- pages: search.pages,
617
- truncated: search.truncated,
618
- payload: fallback.payload,
619
- fallbackDepth: fallback.fallbackDepth,
620
- generalReadTweets: search.generalReadTweets + fallback.generalReadTweets,
621
- warnings: [
622
- `recent search returned no tweets for conversation ${mention.conversationId}; used parent walk`,
623
- ...fallback.warnings,
624
- ],
625
- };
626
- });
627
- }
628
-
629
- export function syncMentionThreadsEffect({
630
- account,
631
- mode = DEFAULT_MODE,
632
- limit = DEFAULT_LIMIT,
633
- tweetIds,
634
- delayMs = DEFAULT_DELAY_MS,
635
- timeoutMs = DEFAULT_TIMEOUT_MS,
636
- all = false,
637
- maxPages,
638
- onProgress,
639
- }: SyncMentionThreadsOptions) {
640
- return Effect.gen(function* () {
641
- const parsedMode = yield* trySync(() => parseMode(mode));
642
- const parsedLimit = yield* trySync(() =>
643
- assertPositiveInteger(limit, "--limit"),
644
- );
645
- const parsedDelayMs =
646
- (yield* trySync(() => parseNonNegativeInteger(delayMs, "--delay-ms"))) ??
647
- 0;
648
- const parsedTimeoutMs = yield* trySync(() =>
649
- assertPositiveInteger(timeoutMs, "--timeout-ms"),
650
- );
651
- const parsedMaxPages = yield* trySync(() =>
652
- parseNonNegativeInteger(maxPages, "--max-pages"),
653
- );
654
- const db = yield* trySync(() => getNativeDb());
655
- const resolvedAccount = yield* trySync(() =>
656
- resolveLiveSyncAccount(db, account),
657
- );
658
- const mentions = yield* trySync(() =>
659
- tweetIds
660
- ? listMentionsByIds(
661
- db,
662
- resolvedAccount.accountId,
663
- tweetIds,
664
- parsedLimit,
665
- )
666
- : listRecentMentions(db, resolvedAccount.accountId, parsedLimit),
667
- );
668
- const mentionIds = mentions.map((mention) => mention.id);
669
- const mentionIdSet = new Set(mentionIds);
670
- const results: Array<{
671
- tweetId: string;
672
- conversationId?: string | null;
673
- ok: boolean;
674
- count: number;
675
- strategy?: string;
676
- pages?: number;
677
- fallbackDepth?: number;
678
- truncated?: boolean;
679
- warnings?: string[];
680
- error?: string;
681
- }> = [];
682
- let mergedTweets = 0;
683
- let generalReadTweets = 0;
684
- const uniqueTweetIds = new Set<string>();
685
- const warnings: string[] = [];
686
-
687
- for (const [index, mention] of mentions.entries()) {
688
- if (index > 0 && parsedDelayMs > 0) {
689
- yield* Effect.sleep(parsedDelayMs);
690
- }
691
- const fetchEffect: Effect.Effect<ThreadFetchResult, unknown, never> =
692
- parsedMode === "bird"
693
- ? liveTransportGateway.bird
694
- .listThread({
695
- tweetId: mention.id,
696
- all,
697
- maxPages: parsedMaxPages,
698
- timeoutMs: parsedTimeoutMs,
699
- })
700
- .pipe(
701
- Effect.map((payload) => ({
702
- strategy: "bird" as const,
703
- payload,
704
- pages: undefined,
705
- fallbackDepth: undefined,
706
- generalReadTweets: 0,
707
- truncated: undefined,
708
- warnings: [] as string[],
709
- })),
710
- )
711
- : fetchThreadContextViaXurlEffect({
712
- mention,
713
- all,
714
- maxPages: parsedMaxPages,
715
- maxFallbackDepth: DEFAULT_FALLBACK_DEPTH,
716
- timeoutMs: parsedTimeoutMs,
717
- });
718
- const fetched = yield* fetchEffect.pipe(
719
- Effect.flatMap((fetchResult) =>
720
- databaseWriteEffect((writeDb) =>
721
- mergeMentionThreadIntoLocalStore({
722
- db: writeDb,
723
- accountId: resolvedAccount.accountId,
724
- accountHandle: resolvedAccount.username.toLowerCase(),
725
- mentionIds: mentionIdSet,
726
- payload: fetchResult.payload,
727
- source: parsedMode,
728
- writeThreadContextEdges: parsedMode === "xurl",
729
- }),
730
- ).pipe(Effect.as(fetchResult)),
731
- ),
732
- Effect.map((fetchResult) => ({ ok: true as const, fetchResult })),
733
- Effect.catchAll((error) =>
734
- Effect.succeed({ ok: false as const, error }),
735
- ),
736
- );
737
-
738
- if (!fetched.ok) {
739
- results.push({
740
- tweetId: mention.id,
741
- conversationId: mention.conversationId ?? null,
742
- ok: false,
743
- count: 0,
744
- strategy: parsedMode,
745
- error:
746
- fetched.error instanceof Error
747
- ? fetched.error.message
748
- : String(fetched.error),
749
- });
750
- yield* Effect.sync(() =>
751
- onProgress?.({
752
- source: parsedMode,
753
- processed: index + 1,
754
- total: mentions.length,
755
- fetched: uniqueTweetIds.size,
756
- done: index + 1 === mentions.length,
757
- }),
758
- );
759
- continue;
760
- }
761
-
762
- const { fetchResult } = fetched;
763
- const { payload } = fetchResult;
764
- for (const tweet of payload.data) {
765
- uniqueTweetIds.add(tweet.id);
766
- }
767
- mergedTweets += payload.data.length;
768
- generalReadTweets += fetchResult.generalReadTweets;
769
- warnings.push(...fetchResult.warnings);
770
- results.push({
771
- tweetId: mention.id,
772
- conversationId: mention.conversationId ?? null,
773
- ok: true,
774
- count: payload.data.length,
775
- strategy: fetchResult.strategy,
776
- pages: fetchResult.pages,
777
- fallbackDepth: fetchResult.fallbackDepth,
778
- truncated: fetchResult.truncated,
779
- warnings:
780
- fetchResult.warnings.length > 0 ? fetchResult.warnings : undefined,
781
- });
782
- yield* Effect.sync(() =>
783
- onProgress?.({
784
- source: parsedMode,
785
- processed: index + 1,
786
- total: mentions.length,
787
- fetched: uniqueTweetIds.size,
788
- done: index + 1 === mentions.length,
789
- }),
790
- );
791
- }
792
-
793
- const failures = results.filter((item) => !item.ok);
794
- const skipped = results.filter((item) =>
795
- item.strategy?.startsWith("skipped:"),
796
- );
797
- const partial = results.some((item) => item.truncated === true);
798
- return {
799
- ok: true,
800
- source: parsedMode,
801
- accountId: resolvedAccount.accountId,
802
- mentions: mentionIds.length,
803
- threads: results.length,
804
- succeeded: results.length - failures.length - skipped.length,
805
- skipped: skipped.length,
806
- failed: failures.length,
807
- mergedTweets,
808
- uniqueTweets: uniqueTweetIds.size,
809
- generalReadTweets: parsedMode === "xurl" ? generalReadTweets : 0,
810
- partial,
811
- options: {
812
- mode: parsedMode,
813
- limit: parsedLimit,
814
- delayMs: parsedDelayMs,
815
- timeoutMs: parsedTimeoutMs,
816
- all,
817
- maxPages: parsedMaxPages ?? null,
818
- maxFallbackDepth: DEFAULT_FALLBACK_DEPTH,
819
- tweetIds: tweetIds ?? null,
820
- },
821
- results,
822
- failures,
823
- warnings,
824
- };
825
- });
826
- }
827
-
828
- export function syncMentionThreads(options: SyncMentionThreadsOptions) {
829
- return runEffectPromise(syncMentionThreadsEffect(options));
830
- }