birdclaw 0.8.2 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.md +10 -8
  3. package/bin/birdclaw.mjs +4 -66
  4. package/dist/cli/birdclaw.js +28165 -0
  5. package/dist/client/assets/AvatarChip-DLoqM5xD.js +1 -0
  6. package/dist/client/assets/ProfilePreview-BejAuYsJ.js +1 -0
  7. package/dist/client/assets/SavedTimelineView-BIS7HKpp.js +1 -0
  8. package/dist/client/assets/SmartTimestamp-BR0-9se2.js +1 -0
  9. package/dist/client/assets/TimelineRouteFrame-CyGPj0eM.js +1 -0
  10. package/dist/client/assets/blocks-7UDKQp8H.js +1 -0
  11. package/dist/client/assets/bookmarks-CbBkMvXk.js +1 -0
  12. package/dist/client/assets/chunk-C0phanGV.js +1 -0
  13. package/dist/client/assets/circle-check-C_Vq3ElD.js +1 -0
  14. package/dist/client/assets/client-stream-contracts-ys3XhWG4.js +5 -0
  15. package/dist/client/assets/createLucideIcon-JV8isFFR.js +1 -0
  16. package/dist/client/assets/data-sources-Cm2bpNvg.js +1 -0
  17. package/dist/client/assets/discuss-DTjaOugw.js +1 -0
  18. package/dist/client/assets/dms-CGd6Eogq.js +1 -0
  19. package/dist/client/assets/inbox-BHlGuOy2.js +1 -0
  20. package/dist/client/assets/index-CEIG7S6s.js +31 -0
  21. package/dist/client/assets/index-bpFrQ5yG.css +1 -0
  22. package/dist/client/assets/likes-cLrcAQem.js +1 -0
  23. package/dist/client/assets/links-CM9QO1LC.js +1 -0
  24. package/dist/client/assets/mapbox-BxTlqXt9.js +2 -0
  25. package/dist/client/assets/mapbox-gl-dgf_hX3a.js +3215 -0
  26. package/dist/client/assets/mentions-CxKCgzIf.js +1 -0
  27. package/dist/client/assets/network-map-B5a3whqj.js +2 -0
  28. package/dist/client/assets/profile-analyze-CRtNEwlb.js +1 -0
  29. package/dist/client/assets/profile-hydration-client-YEtJsNTv.js +1 -0
  30. package/dist/client/assets/profiles._handle-C82_HoKX.js +1 -0
  31. package/dist/client/assets/query-client-fYu2rzRC.js +17 -0
  32. package/dist/client/assets/rate-limits-ScaEwdVK.js +1 -0
  33. package/dist/client/assets/react-dom-DHl6Ug7F.js +1 -0
  34. package/dist/client/assets/react-nBylq1U5.js +1 -0
  35. package/dist/client/assets/repeat-2-xCwHM9hD.js +1 -0
  36. package/dist/client/assets/routes-q7ZaW4En.js +1 -0
  37. package/dist/client/assets/schemas-I9r6F6ya.js +64 -0
  38. package/dist/client/assets/search-Dq5kYsxA.js +1 -0
  39. package/dist/client/assets/sparkles-CQVlvHsZ.js +1 -0
  40. package/dist/client/assets/styles-B8s3tt3r.css +2 -0
  41. package/dist/client/assets/today-CeG1O7Py.js +1 -0
  42. package/dist/client/assets/triangle-alert-CBL7s2pf.js +1 -0
  43. package/dist/client/assets/useTimelineRouteData-1HsqgvUi.js +1 -0
  44. package/dist/client/assets/users-CHbKlqUr.js +1 -0
  45. package/dist/server/assets/AvatarChip-zUDAEmjL.js +105 -0
  46. package/dist/server/assets/FeedState-ntW0Q2GB.js +214 -0
  47. package/dist/server/assets/ProfileAnalysisStream-BFvhVdhn.js +198 -0
  48. package/dist/server/assets/ProfilePreview-C6gAor_j.js +752 -0
  49. package/dist/server/assets/SavedTimelineView-BG1jnBp1.js +64 -0
  50. package/dist/server/assets/SmartTimestamp-DfwE1NGf.js +45 -0
  51. package/dist/server/assets/SyncNowButton-BOn_IBXS.js +76 -0
  52. package/dist/server/assets/TimelineRouteFrame-CU7ilM7U.js +85 -0
  53. package/dist/server/assets/TweetRichText-DhCVv1-6.js +74 -0
  54. package/dist/server/assets/_tanstack-start-manifest_v-C-ZRnG_n.js +148 -0
  55. package/dist/server/assets/account-selection-CTdFGWRk.js +44 -0
  56. package/dist/server/assets/blocks-C1mgGTB3.js +299 -0
  57. package/dist/server/assets/blocks-DSgQghqP.js +17 -0
  58. package/dist/server/assets/bookmarks-BJN8tykY.js +14 -0
  59. package/dist/server/assets/client-stream-contracts-C2xbWVjm.js +769 -0
  60. package/dist/server/assets/data-sources-BnA1Q94p.js +180 -0
  61. package/dist/server/assets/discuss-CJ1dau1j.js +15 -0
  62. package/dist/server/assets/discuss-QTo08NCN.js +338 -0
  63. package/dist/server/assets/dms-BuLj4bp8.js +368 -0
  64. package/dist/server/assets/dms-DAyQrVM2.js +240 -0
  65. package/dist/server/assets/empty-plugin-adapters-DzrEkdBF.js +5 -0
  66. package/dist/server/assets/inbox-CkGOaW1q.js +214 -0
  67. package/dist/server/assets/inbox-DpWd1EzC.js +146 -0
  68. package/dist/server/assets/likes-DBOAtsJU.js +14 -0
  69. package/dist/server/assets/links-DAhSjynu.js +240 -0
  70. package/dist/server/assets/links-DmH5lxHO.js +468 -0
  71. package/dist/server/assets/mentions-BH2MzKtB.js +26 -0
  72. package/dist/server/assets/network-map-D3CzRcC4.js +224 -0
  73. package/dist/server/assets/network-map-D5DoLOwR.js +582 -0
  74. package/dist/server/assets/profile-analyze-BP65PED0.js +90 -0
  75. package/dist/server/assets/profile-analyze-C1NoM0tV.js +9 -0
  76. package/dist/server/assets/profile-hydration-client-GJ4kwJS8.js +133 -0
  77. package/dist/server/assets/profiles._handle-DjwlN_ac.js +184 -0
  78. package/dist/server/assets/profiles._handle-DsLNj2Ca.js +12 -0
  79. package/dist/server/assets/query-client-BdauHBgd.js +923 -0
  80. package/dist/server/assets/rate-limits-VAr2qzYD.js +262 -0
  81. package/dist/server/assets/route-search-BUlQ5lGM.js +117 -0
  82. package/dist/server/assets/router-DqkwSPiM.js +17026 -0
  83. package/dist/server/assets/routes-A9c9ZhwN.js +26 -0
  84. package/dist/server/assets/start-FaDOTmto.js +4 -0
  85. package/dist/server/assets/today-B5hVM57Z.js +17 -0
  86. package/dist/server/assets/today-fvGiZ7ud.js +303 -0
  87. package/dist/server/assets/ui-qbmcMSSX.js +137 -0
  88. package/{src/components/useDebouncedValue.ts → dist/server/assets/useDebouncedValue-DOnA6Xcy.js} +4 -4
  89. package/dist/server/assets/useTimelineRouteData-27cxynxa.js +1101 -0
  90. package/dist/server/server.js +5520 -0
  91. package/package.json +27 -34
  92. package/playwright.config.ts +0 -35
  93. package/scripts/browser-perf.mjs +0 -427
  94. package/scripts/build-docs-site.mjs +0 -967
  95. package/scripts/docs-site-assets.mjs +0 -311
  96. package/scripts/run-vitest.mjs +0 -21
  97. package/scripts/sanitize-node-options.mjs +0 -23
  98. package/scripts/start-test-server.mjs +0 -42
  99. package/src/cli-moderation.ts +0 -210
  100. package/src/cli.ts +0 -2336
  101. package/src/components/AccountSwitcher.tsx +0 -186
  102. package/src/components/AppNav.tsx +0 -118
  103. package/src/components/AvatarChip.tsx +0 -58
  104. package/src/components/BrandMark.tsx +0 -67
  105. package/src/components/ConversationThread.tsx +0 -131
  106. package/src/components/DmWorkspace.tsx +0 -286
  107. package/src/components/EmbeddedTweetCard.tsx +0 -55
  108. package/src/components/FeedState.tsx +0 -147
  109. package/src/components/FloatingPreview.tsx +0 -382
  110. package/src/components/InboxCard.tsx +0 -152
  111. package/src/components/LinkPreviewCard.tsx +0 -292
  112. package/src/components/MarkdownViewer.tsx +0 -777
  113. package/src/components/ProfileAnalysisStream.tsx +0 -428
  114. package/src/components/ProfilePreview.tsx +0 -129
  115. package/src/components/SavedTimelineView.tsx +0 -155
  116. package/src/components/SmartTimestamp.tsx +0 -72
  117. package/src/components/SyncNowButton.tsx +0 -140
  118. package/src/components/ThemeSlider.tsx +0 -80
  119. package/src/components/TimelineCard.tsx +0 -533
  120. package/src/components/TimelineRouteFrame.tsx +0 -186
  121. package/src/components/TweetArticleCard.tsx +0 -66
  122. package/src/components/TweetMediaGrid.tsx +0 -185
  123. package/src/components/TweetRichText.tsx +0 -159
  124. package/src/components/account-selection.ts +0 -64
  125. package/src/components/useTimelineRouteData.ts +0 -323
  126. package/src/lib/account-sync-job.ts +0 -666
  127. package/src/lib/actions-transport.ts +0 -300
  128. package/src/lib/api-client.ts +0 -419
  129. package/src/lib/archive-finder.ts +0 -185
  130. package/src/lib/archive-import.ts +0 -2764
  131. package/src/lib/authored-live.ts +0 -1072
  132. package/src/lib/avatar-cache.ts +0 -349
  133. package/src/lib/backup.ts +0 -2741
  134. package/src/lib/bird-actions.ts +0 -294
  135. package/src/lib/bird-command.ts +0 -130
  136. package/src/lib/bird.ts +0 -1140
  137. package/src/lib/blocklist.ts +0 -136
  138. package/src/lib/blocks-write.ts +0 -167
  139. package/src/lib/blocks.ts +0 -389
  140. package/src/lib/bookmark-sync-job.ts +0 -536
  141. package/src/lib/client-cache.ts +0 -109
  142. package/src/lib/config.ts +0 -224
  143. package/src/lib/conversation-surface.ts +0 -205
  144. package/src/lib/data-sources.ts +0 -219
  145. package/src/lib/db.ts +0 -1135
  146. package/src/lib/dms-live.ts +0 -933
  147. package/src/lib/effect-runtime.ts +0 -45
  148. package/src/lib/follow-graph.ts +0 -1097
  149. package/src/lib/geocoding.ts +0 -296
  150. package/src/lib/http-effect.ts +0 -222
  151. package/src/lib/identity-search-index.ts +0 -369
  152. package/src/lib/inbox.ts +0 -241
  153. package/src/lib/link-index.ts +0 -641
  154. package/src/lib/link-insights.ts +0 -860
  155. package/src/lib/link-preview-metadata.ts +0 -754
  156. package/src/lib/location.ts +0 -137
  157. package/src/lib/media-fetch.ts +0 -860
  158. package/src/lib/media-includes.ts +0 -165
  159. package/src/lib/mention-threads-live.ts +0 -917
  160. package/src/lib/mentions-export.ts +0 -147
  161. package/src/lib/mentions-live.ts +0 -1238
  162. package/src/lib/moderation-target.ts +0 -216
  163. package/src/lib/moderation-write.ts +0 -131
  164. package/src/lib/mutes-write.ts +0 -167
  165. package/src/lib/mutes.ts +0 -84
  166. package/src/lib/ndjson-stream.ts +0 -106
  167. package/src/lib/network-map.ts +0 -382
  168. package/src/lib/openai.ts +0 -117
  169. package/src/lib/period-digest.ts +0 -1552
  170. package/src/lib/present.ts +0 -118
  171. package/src/lib/profile-affiliation-hydration.ts +0 -228
  172. package/src/lib/profile-affiliations.ts +0 -262
  173. package/src/lib/profile-analysis.ts +0 -1272
  174. package/src/lib/profile-bio-entities.ts +0 -318
  175. package/src/lib/profile-history.ts +0 -164
  176. package/src/lib/profile-hydration.ts +0 -180
  177. package/src/lib/profile-replies.ts +0 -77
  178. package/src/lib/profile-resolver.ts +0 -650
  179. package/src/lib/queries.ts +0 -2381
  180. package/src/lib/research.ts +0 -652
  181. package/src/lib/search-discussion.ts +0 -1016
  182. package/src/lib/seed.ts +0 -613
  183. package/src/lib/sqlite.ts +0 -147
  184. package/src/lib/sync-cache.ts +0 -65
  185. package/src/lib/theme-transition.ts +0 -113
  186. package/src/lib/theme.tsx +0 -165
  187. package/src/lib/timeline-collections-live.ts +0 -581
  188. package/src/lib/timeline-live.ts +0 -452
  189. package/src/lib/tweet-account-edges.ts +0 -48
  190. package/src/lib/tweet-lookup.ts +0 -46
  191. package/src/lib/tweet-render.ts +0 -430
  192. package/src/lib/tweet-search-live.ts +0 -566
  193. package/src/lib/types.ts +0 -723
  194. package/src/lib/ui.ts +0 -529
  195. package/src/lib/url-expansion-store.ts +0 -110
  196. package/src/lib/url-expansion.ts +0 -343
  197. package/src/lib/url-safety.ts +0 -220
  198. package/src/lib/web-sync.ts +0 -516
  199. package/src/lib/whois.ts +0 -1079
  200. package/src/lib/x-profile.ts +0 -365
  201. package/src/lib/x-web.ts +0 -199
  202. package/src/lib/xurl-rate-limits.ts +0 -267
  203. package/src/lib/xurl.ts +0 -1677
  204. package/src/routeTree.gen.ts +0 -723
  205. package/src/router.tsx +0 -20
  206. package/src/routes/__root.tsx +0 -79
  207. package/src/routes/api/action.tsx +0 -147
  208. package/src/routes/api/avatar.tsx +0 -50
  209. package/src/routes/api/blocks.tsx +0 -35
  210. package/src/routes/api/conversation.tsx +0 -45
  211. package/src/routes/api/data-sources.tsx +0 -24
  212. package/src/routes/api/inbox.tsx +0 -43
  213. package/src/routes/api/link-insights.tsx +0 -85
  214. package/src/routes/api/link-preview.tsx +0 -47
  215. package/src/routes/api/network-map.tsx +0 -55
  216. package/src/routes/api/period-digest.tsx +0 -101
  217. package/src/routes/api/profile-analysis.tsx +0 -97
  218. package/src/routes/api/profile-hydrate.tsx +0 -54
  219. package/src/routes/api/query.tsx +0 -115
  220. package/src/routes/api/search-discussion.tsx +0 -113
  221. package/src/routes/api/status.tsx +0 -28
  222. package/src/routes/api/sync.tsx +0 -105
  223. package/src/routes/api/xurl-rate-limits.tsx +0 -24
  224. package/src/routes/blocks.tsx +0 -377
  225. package/src/routes/bookmarks.tsx +0 -18
  226. package/src/routes/data-sources.tsx +0 -257
  227. package/src/routes/discuss.tsx +0 -419
  228. package/src/routes/dms.tsx +0 -432
  229. package/src/routes/inbox.tsx +0 -226
  230. package/src/routes/index.tsx +0 -32
  231. package/src/routes/likes.tsx +0 -18
  232. package/src/routes/links.tsx +0 -994
  233. package/src/routes/mentions.tsx +0 -32
  234. package/src/routes/network-map.tsx +0 -1035
  235. package/src/routes/profile-analyze.tsx +0 -112
  236. package/src/routes/profiles.$handle.tsx +0 -228
  237. package/src/routes/rate-limits.tsx +0 -309
  238. package/src/routes/today.tsx +0 -511
  239. package/src/styles.css +0 -257
  240. package/tsconfig.json +0 -29
  241. package/vite.config.ts +0 -29
  242. /package/{public → dist/client}/birdclaw-mark.png +0 -0
  243. /package/{public → dist/client}/favicon.ico +0 -0
  244. /package/{public → dist/client}/logo192.png +0 -0
  245. /package/{public → dist/client}/logo512.png +0 -0
  246. /package/{public → dist/client}/manifest.json +0 -0
  247. /package/{public → dist/client}/robots.txt +0 -0
@@ -1,1552 +0,0 @@
1
- import { createHash } from "node:crypto";
2
- import { Effect } from "effect";
3
- import { z } from "zod";
4
- import { maybeAutoSyncBackupEffect } from "./backup";
5
- import { runEffectPromise, tryPromise } from "./effect-runtime";
6
- import { getLinkInsights } from "./link-insights";
7
- import { syncMentionThreadsEffect } from "./mention-threads-live";
8
- import { syncMentionsEffect } from "./mentions-live";
9
- import {
10
- getTweetsByIds,
11
- listDmConversations,
12
- listTimelineItems,
13
- } from "./queries";
14
- import { readSyncCache, writeSyncCache } from "./sync-cache";
15
- import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
16
- import type { EmbeddedTweet, ProfileRecord, TweetEntities } from "./types";
17
-
18
- export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
19
- export type PeriodDigestSourceKind =
20
- | "home"
21
- | "mentions"
22
- | "authored"
23
- | "likes"
24
- | "bookmarks"
25
- | "dms";
26
-
27
- export interface PeriodDigestOptions {
28
- period?: string;
29
- since?: string;
30
- until?: string;
31
- account?: string;
32
- includeDms?: boolean;
33
- refresh?: boolean;
34
- model?: string;
35
- language?: string;
36
- reasoningEffort?: "minimal" | "low" | "medium" | "high";
37
- serviceTier?: "default" | "flex" | "priority";
38
- signal?: AbortSignal;
39
- maxTweets?: number;
40
- maxLinks?: number;
41
- liveSync?: boolean;
42
- liveSyncMode?: HomeTimelineMode;
43
- liveTimelineLimit?: number;
44
- liveTimelineMaxPages?: number;
45
- liveMentionsLimit?: number;
46
- liveMentionsMaxPages?: number;
47
- liveThreadLimit?: number;
48
- }
49
-
50
- export interface PeriodDigestWindow {
51
- label: string;
52
- since: string;
53
- until: string;
54
- }
55
-
56
- export interface PeriodDigestRunResult {
57
- context: PeriodDigestContext;
58
- digest: PeriodDigest;
59
- markdown: string;
60
- model: string;
61
- reasoningEffort: string;
62
- serviceTier: string;
63
- cached: boolean;
64
- updatedAt: string;
65
- }
66
-
67
- export interface PeriodDigestStreamHandlers {
68
- onDelta?: (delta: string) => void;
69
- onEvent?: (event: PeriodDigestStreamEvent) => void;
70
- }
71
-
72
- export type PeriodDigestStreamEvent =
73
- | { type: "status"; label: string; detail?: string }
74
- | { type: "start"; context: PeriodDigestContext; cached: boolean }
75
- | { type: "delta"; delta: string }
76
- | { type: "done"; result: PeriodDigestRunResult }
77
- | { type: "error"; error: string };
78
-
79
- const PeriodDigestSchema = z.object({
80
- title: z.string().min(1),
81
- summary: z.string().min(1),
82
- keyTopics: z.array(
83
- z.object({
84
- title: z.string().min(1),
85
- summary: z.string().min(1),
86
- tweetIds: z.array(z.string()).default([]),
87
- handles: z.array(z.string()).default([]),
88
- }),
89
- ),
90
- notableLinks: z.array(
91
- z.object({
92
- title: z.string().min(1),
93
- url: z.string().min(1),
94
- why: z.string().min(1),
95
- sourceTweetIds: z.array(z.string()).default([]),
96
- }),
97
- ),
98
- people: z.array(
99
- z.object({
100
- handle: z.string().min(1),
101
- name: z.string().optional(),
102
- why: z.string().min(1),
103
- }),
104
- ),
105
- actionItems: z.array(
106
- z.object({
107
- kind: z.enum(["reply", "follow_up", "read", "sync"]),
108
- label: z.string().min(1),
109
- tweetId: z.string().optional(),
110
- dmConversationId: z.string().optional(),
111
- }),
112
- ),
113
- sourceTweetIds: z.array(z.string()).default([]),
114
- });
115
-
116
- const MAX_DIGEST_LANGUAGE_LENGTH = 64;
117
-
118
- export function normalizeDigestLanguage(
119
- value: string | undefined,
120
- ): string | undefined {
121
- const trimmed = value?.trim();
122
- if (!trimmed) return undefined;
123
- if (
124
- trimmed.length > MAX_DIGEST_LANGUAGE_LENGTH ||
125
- !/^[a-z0-9]+(?:-[a-z0-9]+)*$/i.test(trimmed)
126
- ) {
127
- throw new Error(
128
- "Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
129
- );
130
- }
131
- try {
132
- const [canonical] = Intl.getCanonicalLocales(trimmed);
133
- if (!canonical) throw new Error("missing canonical locale");
134
- return canonical;
135
- } catch {
136
- throw new Error(
137
- "Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
138
- );
139
- }
140
- }
141
-
142
- export type PeriodDigest = z.infer<typeof PeriodDigestSchema>;
143
-
144
- interface CompactTweet {
145
- id: string;
146
- url: string;
147
- source: PeriodDigestSourceKind;
148
- author: string;
149
- name: string;
150
- authorProfile: ProfileRecord;
151
- createdAt: string;
152
- text: string;
153
- entities?: TweetEntities;
154
- likeCount: number;
155
- liked: boolean;
156
- bookmarked: boolean;
157
- needsReply: boolean;
158
- replyToId?: string | null;
159
- replyToTweet?: {
160
- id: string;
161
- url: string;
162
- author: string;
163
- name: string;
164
- createdAt: string;
165
- text: string;
166
- } | null;
167
- }
168
-
169
- interface CompactDm {
170
- id: string;
171
- participant: string;
172
- name: string;
173
- lastMessageAt: string;
174
- text: string;
175
- needsReply: boolean;
176
- influenceScore: number;
177
- }
178
-
179
- interface CompactLink {
180
- title: string;
181
- url: string;
182
- displayUrl: string;
183
- description?: string | null;
184
- shareCount: number;
185
- commentCount: number;
186
- lastSeenAt: string;
187
- mentions: Array<{
188
- id: string;
189
- sourceKind: string;
190
- sourceId: string;
191
- createdAt: string;
192
- author?: string;
193
- text: string;
194
- tweetId?: string | null;
195
- }>;
196
- }
197
-
198
- export interface PeriodDigestContext {
199
- window: PeriodDigestWindow;
200
- account?: string;
201
- includeDms: boolean;
202
- counts: Record<PeriodDigestSourceKind | "links", number>;
203
- tweets: CompactTweet[];
204
- dms: CompactDm[];
205
- links: CompactLink[];
206
- hash: string;
207
- }
208
-
209
- interface OpenAIStreamState {
210
- eventBuffer: string;
211
- rawText: string;
212
- pendingVisible: string;
213
- jsonMode: boolean;
214
- responseId?: string;
215
- usage?: unknown;
216
- error?: string;
217
- }
218
-
219
- const DEFAULT_MODEL = "gpt-5.5";
220
- const DEFAULT_REASONING_EFFORT = "medium";
221
- const DEFAULT_SERVICE_TIER = "priority";
222
- const DEFAULT_MAX_TWEETS = 2_500;
223
- const DEFAULT_MAX_LINKS = 12;
224
- const DEFAULT_LIVE_TIMELINE_MAX_PAGES = undefined;
225
- const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
226
- const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
227
- const DEFAULT_LIVE_THREAD_LIMIT = 12;
228
- const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
229
- const DEFAULT_DIGEST_FRESHNESS_MS = 5 * 60_000;
230
- const MAX_PROMPT_DATA_CHARS = 1_200_000;
231
- const DELIMITER_PATTERN = /\n---\s*\n/;
232
- const VISIBLE_DELIMITER_HOLD = 8;
233
-
234
- function toError(error: unknown) {
235
- return error instanceof Error ? error : new Error(String(error));
236
- }
237
-
238
- function tryDigestSync<T>(try_: () => T): Effect.Effect<T, Error> {
239
- return Effect.try({
240
- try: try_,
241
- catch: toError,
242
- });
243
- }
244
-
245
- function tryDigestPromise<T>(
246
- try_: () => PromiseLike<T>,
247
- ): Effect.Effect<T, Error> {
248
- return tryPromise(try_).pipe(Effect.mapError(toError));
249
- }
250
-
251
- function localDateStart(date: Date) {
252
- return new Date(date.getFullYear(), date.getMonth(), date.getDate());
253
- }
254
-
255
- function addDays(date: Date, days: number) {
256
- const next = new Date(date);
257
- next.setDate(next.getDate() + days);
258
- return next;
259
- }
260
-
261
- function parseDate(value: string | undefined) {
262
- if (!value?.trim()) return null;
263
- const parsed = new Date(value);
264
- return Number.isNaN(parsed.getTime()) ? null : parsed;
265
- }
266
-
267
- function floorIsoToHour(value: string) {
268
- const date = new Date(value);
269
- date.setUTCMinutes(0, 0, 0);
270
- return date.toISOString();
271
- }
272
-
273
- function normalizePeriod(value: string | undefined): PeriodDigestPreset {
274
- const normalized = value?.trim().toLowerCase();
275
- if (normalized === "yesterday") return "yesterday";
276
- if (normalized === "24h" || normalized === "day") return "24h";
277
- if (normalized === "week" || normalized === "7d") return "week";
278
- return "today";
279
- }
280
-
281
- export function resolvePeriodDigestWindow(
282
- options: Pick<PeriodDigestOptions, "period" | "since" | "until"> & {
283
- now?: Date;
284
- } = {},
285
- ): PeriodDigestWindow {
286
- const now = options.now ?? new Date();
287
- const explicitSince = parseDate(options.since);
288
- const explicitUntil = parseDate(options.until);
289
- if (explicitSince || explicitUntil) {
290
- const since = explicitSince ?? addDays(explicitUntil ?? now, -1);
291
- const until = explicitUntil ?? now;
292
- return {
293
- label: `${since.toLocaleString()} - ${until.toLocaleString()}`,
294
- since: since.toISOString(),
295
- until: until.toISOString(),
296
- };
297
- }
298
-
299
- const period = normalizePeriod(options.period);
300
- if (period === "24h") {
301
- const since = new Date(now.getTime() - 24 * 60 * 60 * 1000);
302
- return {
303
- label: "Last 24 hours",
304
- since: since.toISOString(),
305
- until: now.toISOString(),
306
- };
307
- }
308
- if (period === "week") {
309
- const since = addDays(now, -7);
310
- return {
311
- label: "Last 7 days",
312
- since: since.toISOString(),
313
- until: now.toISOString(),
314
- };
315
- }
316
- if (period === "yesterday") {
317
- const today = localDateStart(now);
318
- const yesterday = addDays(today, -1);
319
- return {
320
- label: "Yesterday",
321
- since: yesterday.toISOString(),
322
- until: today.toISOString(),
323
- };
324
- }
325
-
326
- const start = localDateStart(now);
327
- return {
328
- label: "Today",
329
- since: start.toISOString(),
330
- until: now.toISOString(),
331
- };
332
- }
333
-
334
- function tweetUrl(handle: string, id: string) {
335
- return `https://x.com/${handle}/status/${id}`;
336
- }
337
-
338
- function compactTweet(
339
- source: PeriodDigestSourceKind,
340
- item: ReturnType<typeof listTimelineItems>[number],
341
- ): CompactTweet {
342
- const replyToTweet = item.replyToTweet
343
- ? {
344
- id: item.replyToTweet.id,
345
- url: tweetUrl(item.replyToTweet.author.handle, item.replyToTweet.id),
346
- author: item.replyToTweet.author.handle,
347
- name: item.replyToTweet.author.displayName,
348
- createdAt: item.replyToTweet.createdAt,
349
- text: item.replyToTweet.text,
350
- }
351
- : null;
352
- return {
353
- id: item.id,
354
- url: tweetUrl(item.author.handle, item.id),
355
- source,
356
- author: item.author.handle,
357
- name: item.author.displayName,
358
- authorProfile: item.author,
359
- createdAt: item.createdAt,
360
- text: item.text,
361
- entities: item.entities,
362
- likeCount: item.likeCount,
363
- liked: item.liked,
364
- bookmarked: item.bookmarked,
365
- needsReply: !item.isReplied,
366
- replyToId: item.replyToId ?? null,
367
- replyToTweet,
368
- };
369
- }
370
-
371
- function compactEmbeddedTweet(item: EmbeddedTweet): CompactTweet {
372
- return {
373
- id: item.id,
374
- url: tweetUrl(item.author.handle, item.id),
375
- source: "home",
376
- author: item.author.handle,
377
- name: item.author.displayName,
378
- authorProfile: item.author,
379
- createdAt: item.createdAt,
380
- text: item.text,
381
- entities: item.entities,
382
- likeCount: item.likeCount ?? 0,
383
- liked: Boolean(item.liked),
384
- bookmarked: Boolean(item.bookmarked),
385
- needsReply: !item.isReplied,
386
- replyToId: item.replyToId ?? null,
387
- replyToTweet: null,
388
- };
389
- }
390
-
391
- function dedupeTweets(tweets: CompactTweet[]) {
392
- const seen = new Set<string>();
393
- const items: CompactTweet[] = [];
394
- for (const tweet of tweets) {
395
- if (seen.has(tweet.id)) continue;
396
- seen.add(tweet.id);
397
- items.push(tweet);
398
- }
399
- return items.sort((left, right) =>
400
- right.createdAt.localeCompare(left.createdAt),
401
- );
402
- }
403
-
404
- function collectTweetsForSource(
405
- source: Exclude<PeriodDigestSourceKind, "dms">,
406
- options: {
407
- account?: string;
408
- window: PeriodDigestWindow;
409
- limit: number;
410
- },
411
- ) {
412
- if (source === "likes" || source === "bookmarks") {
413
- return listTimelineItems({
414
- resource: "home",
415
- account: options.account,
416
- since: options.window.since,
417
- until: options.window.until,
418
- likedOnly: source === "likes",
419
- bookmarkedOnly: source === "bookmarks",
420
- limit: Math.ceil(options.limit / 3),
421
- }).map((item) => compactTweet(source, item));
422
- }
423
- return listTimelineItems({
424
- resource: source,
425
- account: options.account,
426
- since: options.window.since,
427
- until: options.window.until,
428
- limit: source === "home" ? options.limit : Math.ceil(options.limit / 2),
429
- }).map((item) => compactTweet(source, item));
430
- }
431
-
432
- function collectDms(options: {
433
- account?: string;
434
- includeDms: boolean;
435
- window: PeriodDigestWindow;
436
- limit: number;
437
- }) {
438
- if (!options.includeDms) return [];
439
- return listDmConversations({
440
- account: options.account,
441
- since: options.window.since,
442
- until: options.window.until,
443
- sort: "recent",
444
- limit: options.limit,
445
- }).map(
446
- (item): CompactDm => ({
447
- id: item.id,
448
- participant: item.participant.handle,
449
- name: item.participant.displayName,
450
- lastMessageAt: item.lastMessageAt,
451
- text: item.lastMessagePreview,
452
- needsReply: item.needsReply,
453
- influenceScore: item.influenceScore,
454
- }),
455
- );
456
- }
457
-
458
- function compactLinks(options: {
459
- account?: string;
460
- window: PeriodDigestWindow;
461
- limit: number;
462
- }) {
463
- return getLinkInsights({
464
- account: options.account,
465
- range: "today",
466
- sort: "rank",
467
- source: "tweet",
468
- since: options.window.since,
469
- until: options.window.until,
470
- limit: options.limit,
471
- commentsLimit: 5,
472
- }).items.map(
473
- (item): CompactLink => ({
474
- title: item.title ?? item.displayUrl,
475
- url: item.url,
476
- displayUrl: item.displayUrl,
477
- description: item.description,
478
- shareCount: item.shareCount,
479
- commentCount: item.commentCount,
480
- lastSeenAt: item.lastSeenAt,
481
- mentions: item.mentions.slice(0, 5).map((mention) => ({
482
- id: mention.id,
483
- sourceKind: mention.sourceKind,
484
- sourceId: mention.sourceId,
485
- createdAt: mention.createdAt,
486
- author: mention.sharedBy?.handle,
487
- text:
488
- mention.commentText || mention.sharedContentText || mention.rawText,
489
- tweetId: mention.timelineTweetId ?? mention.contentTweetId,
490
- })),
491
- }),
492
- );
493
- }
494
-
495
- function contextHash(context: Omit<PeriodDigestContext, "hash">) {
496
- return createHash("sha1")
497
- .update(
498
- JSON.stringify({
499
- window: {
500
- label: context.window.label,
501
- bucket: context.window.until.slice(0, 10),
502
- },
503
- account: context.account,
504
- includeDms: context.includeDms,
505
- tweets: context.tweets.map((tweet) => [
506
- tweet.id,
507
- tweet.url,
508
- tweet.source,
509
- tweet.author,
510
- tweet.name,
511
- tweet.authorProfile.bio,
512
- tweet.authorProfile.followersCount,
513
- tweet.createdAt,
514
- tweet.text,
515
- tweet.likeCount,
516
- tweet.liked,
517
- tweet.bookmarked,
518
- tweet.needsReply,
519
- tweet.replyToId,
520
- tweet.replyToTweet?.id,
521
- tweet.replyToTweet?.text,
522
- ]),
523
- dms: context.dms.map((dm) => [
524
- dm.id,
525
- dm.lastMessageAt,
526
- dm.text,
527
- dm.needsReply,
528
- ]),
529
- links: context.links.map((link) => [
530
- link.url,
531
- link.shareCount,
532
- link.commentCount,
533
- link.lastSeenAt,
534
- ]),
535
- }),
536
- )
537
- .digest("hex");
538
- }
539
-
540
- export function collectPeriodDigestContext(
541
- options: PeriodDigestOptions = {},
542
- ): PeriodDigestContext {
543
- const window = resolvePeriodDigestWindow(options);
544
- const maxTweets = Math.max(
545
- 20,
546
- Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
547
- );
548
- const maxLinks = Math.max(
549
- 3,
550
- Math.trunc(options.maxLinks ?? DEFAULT_MAX_LINKS),
551
- );
552
- const home = collectTweetsForSource("home", {
553
- account: options.account,
554
- window,
555
- limit: maxTweets,
556
- });
557
- const mentions = collectTweetsForSource("mentions", {
558
- account: options.account,
559
- window,
560
- limit: maxTweets,
561
- });
562
- const authored = collectTweetsForSource("authored", {
563
- account: options.account,
564
- window,
565
- limit: maxTweets,
566
- });
567
- const likes = collectTweetsForSource("likes", {
568
- account: options.account,
569
- window,
570
- limit: maxTweets,
571
- });
572
- const bookmarks = collectTweetsForSource("bookmarks", {
573
- account: options.account,
574
- window,
575
- limit: maxTweets,
576
- });
577
- const dms = collectDms({
578
- account: options.account,
579
- includeDms: Boolean(options.includeDms),
580
- window,
581
- limit: Math.ceil(maxTweets / 3),
582
- });
583
- const links = compactLinks({
584
- account: options.account,
585
- window,
586
- limit: maxLinks,
587
- });
588
- const tweets = dedupeTweets([
589
- ...home,
590
- ...mentions,
591
- ...authored,
592
- ...likes,
593
- ...bookmarks,
594
- ]).slice(0, maxTweets);
595
- const withoutHash = {
596
- window,
597
- ...(options.account ? { account: options.account } : {}),
598
- includeDms: Boolean(options.includeDms),
599
- counts: {
600
- home: home.length,
601
- mentions: mentions.length,
602
- authored: authored.length,
603
- likes: likes.length,
604
- bookmarks: bookmarks.length,
605
- dms: dms.length,
606
- links: links.length,
607
- },
608
- tweets,
609
- dms,
610
- links,
611
- } satisfies Omit<PeriodDigestContext, "hash">;
612
- return {
613
- ...withoutHash,
614
- hash: contextHash(withoutHash),
615
- };
616
- }
617
-
618
- function languageFromOptions(options: PeriodDigestOptions) {
619
- return normalizeDigestLanguage(
620
- options.language ?? process.env.BIRDCLAW_DIGEST_LANGUAGE,
621
- );
622
- }
623
-
624
- function modelFromOptions(options: PeriodDigestOptions) {
625
- return options.model ?? process.env.BIRDCLAW_AI_MODEL ?? DEFAULT_MODEL;
626
- }
627
-
628
- function reasoningEffortFromOptions(options: PeriodDigestOptions) {
629
- return (
630
- options.reasoningEffort ??
631
- (process.env.BIRDCLAW_OPENAI_REASONING_EFFORT as
632
- | PeriodDigestOptions["reasoningEffort"]
633
- | undefined) ??
634
- DEFAULT_REASONING_EFFORT
635
- );
636
- }
637
-
638
- function serviceTierFromOptions(options: PeriodDigestOptions) {
639
- return (
640
- options.serviceTier ??
641
- (process.env.BIRDCLAW_OPENAI_SERVICE_TIER as
642
- | PeriodDigestOptions["serviceTier"]
643
- | undefined) ??
644
- DEFAULT_SERVICE_TIER
645
- );
646
- }
647
-
648
- function boundedPositiveInteger(
649
- value: number | undefined,
650
- fallback: number,
651
- max: number,
652
- ) {
653
- if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
654
- return fallback;
655
- }
656
- return Math.min(max, Math.floor(value));
657
- }
658
-
659
- function emitDigestStatus(
660
- handlers: PeriodDigestStreamHandlers,
661
- label: string,
662
- detail?: string,
663
- ) {
664
- handlers.onEvent?.({
665
- type: "status",
666
- label,
667
- ...(detail ? { detail } : {}),
668
- });
669
- }
670
-
671
- function formatFetchedStatus({
672
- fetched,
673
- total,
674
- noun,
675
- }: {
676
- fetched: number;
677
- total: number;
678
- noun: string;
679
- }) {
680
- const count = `${String(Math.min(fetched, total))}/${String(total)}`;
681
- return `Fetched ${count} ${noun}`;
682
- }
683
-
684
- function formatPageDetail({
685
- source,
686
- page,
687
- maxPages,
688
- done,
689
- }: {
690
- source: string;
691
- page?: number;
692
- maxPages?: number;
693
- done: boolean;
694
- }) {
695
- const pageText =
696
- page === undefined
697
- ? undefined
698
- : `page ${String(page)}${maxPages === undefined ? "" : `/${String(maxPages)}`}`;
699
- return [source, pageText, done ? "done" : undefined]
700
- .filter(Boolean)
701
- .join(" · ");
702
- }
703
-
704
- function refreshPeriodDigestInputsEffect(
705
- options: PeriodDigestOptions,
706
- phase: {
707
- timeline?: boolean;
708
- mentions?: boolean;
709
- threads?: boolean;
710
- threadTweetIds?: string[];
711
- } = {},
712
- handlers: PeriodDigestStreamHandlers = {},
713
- ): Effect.Effect<void, unknown> {
714
- if (!options.liveSync) {
715
- return Effect.void;
716
- }
717
- const includeTimeline = phase.timeline ?? true;
718
- const includeMentions = phase.mentions ?? true;
719
- const includeThreads = phase.threads ?? true;
720
- const window = resolvePeriodDigestWindow(options);
721
- const liveStartTime = floorIsoToHour(window.since);
722
- const mode = options.liveSyncMode ?? "xurl";
723
- const contextTweetBudget = Math.max(
724
- 20,
725
- Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
726
- );
727
- const timelineLimit =
728
- options.liveTimelineLimit === undefined
729
- ? undefined
730
- : boundedPositiveInteger(options.liveTimelineLimit, 300, 100_000);
731
- const mentionsLimit = boundedPositiveInteger(
732
- options.liveMentionsLimit,
733
- DEFAULT_LIVE_MENTIONS_LIMIT,
734
- 100,
735
- );
736
- const threadLimit = boundedPositiveInteger(
737
- options.liveThreadLimit,
738
- DEFAULT_LIVE_THREAD_LIMIT,
739
- 100,
740
- );
741
- const timelineMaxPages =
742
- options.liveTimelineMaxPages === undefined
743
- ? DEFAULT_LIVE_TIMELINE_MAX_PAGES
744
- : boundedPositiveInteger(options.liveTimelineMaxPages, 3, 1_000);
745
- const mentionsMaxPages =
746
- options.liveMentionsMaxPages === undefined
747
- ? DEFAULT_LIVE_MENTIONS_MAX_PAGES
748
- : boundedPositiveInteger(options.liveMentionsMaxPages, 3, 1_000);
749
-
750
- return Effect.gen(function* () {
751
- if (includeTimeline) {
752
- yield* Effect.sync(() =>
753
- emitDigestStatus(
754
- handlers,
755
- "Fetching home timeline from X",
756
- "Walking the selected time window with xurl.",
757
- ),
758
- );
759
- const result = yield* syncHomeTimelineEffect({
760
- account: options.account,
761
- mode,
762
- limit: timelineLimit,
763
- maxPages: timelineMaxPages,
764
- startTime: liveStartTime,
765
- following: true,
766
- refresh: Boolean(options.refresh),
767
- cacheTtlMs: 2 * 60_000,
768
- timeoutMs: 30_000,
769
- onProgress: (progress) =>
770
- emitDigestStatus(
771
- handlers,
772
- formatFetchedStatus({
773
- fetched: progress.fetched,
774
- total: progress.total ?? contextTweetBudget,
775
- noun: "home tweets",
776
- }),
777
- formatPageDetail({
778
- source: progress.source,
779
- page: progress.page,
780
- maxPages: progress.maxPages,
781
- done: progress.done,
782
- }),
783
- ),
784
- }).pipe(
785
- Effect.match({
786
- onFailure: () => null,
787
- onSuccess: (value) => value,
788
- }),
789
- );
790
- yield* Effect.sync(() =>
791
- emitDigestStatus(
792
- handlers,
793
- result
794
- ? `Fetched ${String(result.count)} home tweets from ${result.source}`
795
- : "Home timeline fetch failed; using local data",
796
- ),
797
- );
798
- }
799
- if (includeMentions) {
800
- yield* Effect.sync(() =>
801
- emitDigestStatus(
802
- handlers,
803
- "Fetching mentions from X",
804
- "Reading replies and mentions for the selected window.",
805
- ),
806
- );
807
- const result = yield* syncMentionsEffect({
808
- account: options.account,
809
- mode: "xurl",
810
- limit: mentionsLimit,
811
- maxPages: mentionsMaxPages,
812
- startTime: liveStartTime,
813
- refresh: Boolean(options.refresh),
814
- cacheTtlMs: 2 * 60_000,
815
- onProgress: (progress) =>
816
- emitDigestStatus(
817
- handlers,
818
- formatFetchedStatus({
819
- fetched: progress.fetched,
820
- total: progress.total ?? contextTweetBudget,
821
- noun: "mentions",
822
- }),
823
- formatPageDetail({
824
- source: progress.source,
825
- page: progress.page,
826
- maxPages: progress.maxPages,
827
- done: progress.done,
828
- }),
829
- ),
830
- }).pipe(
831
- Effect.match({
832
- onFailure: () => null,
833
- onSuccess: (value) => value,
834
- }),
835
- );
836
- yield* Effect.sync(() =>
837
- emitDigestStatus(
838
- handlers,
839
- result
840
- ? `Fetched ${String(result.count)} mentions from ${result.source}`
841
- : "Mention fetch failed; using local data",
842
- ),
843
- );
844
- }
845
- if (includeThreads) {
846
- yield* Effect.sync(() =>
847
- emitDigestStatus(
848
- handlers,
849
- "Fetching mention conversations",
850
- "Pulling parent tweets so the AI sees what replies refer to.",
851
- ),
852
- );
853
- const result = yield* syncMentionThreadsEffect({
854
- account: options.account,
855
- mode: "xurl",
856
- limit: threadLimit,
857
- tweetIds: phase.threadTweetIds,
858
- delayMs: 100,
859
- timeoutMs: DEFAULT_LIVE_THREAD_TIMEOUT_MS,
860
- maxPages: 2,
861
- onProgress: (progress) =>
862
- emitDigestStatus(
863
- handlers,
864
- `Fetched conversations for ${String(progress.processed)}/${String(progress.total)} mentions`,
865
- `${String(progress.fetched)} tweets · ${progress.source}${
866
- progress.done ? " · done" : ""
867
- }`,
868
- ),
869
- }).pipe(
870
- Effect.match({
871
- onFailure: () => null,
872
- onSuccess: (value) => value,
873
- }),
874
- );
875
- yield* Effect.sync(() =>
876
- emitDigestStatus(
877
- handlers,
878
- result
879
- ? `Fetched ${String(result.uniqueTweets)} conversation tweets`
880
- : "Conversation fetch failed; using available context",
881
- ),
882
- );
883
- }
884
- yield* Effect.sync(() =>
885
- emitDigestStatus(handlers, "Preparing local AI context"),
886
- );
887
- yield* maybeAutoSyncBackupEffect().pipe(Effect.catchAll(() => Effect.void));
888
- }).pipe(Effect.asVoid);
889
- }
890
-
891
- function digestCacheKey(
892
- context: PeriodDigestContext,
893
- options: PeriodDigestOptions,
894
- ) {
895
- const parts = [
896
- "period-digest:v2",
897
- modelFromOptions(options),
898
- reasoningEffortFromOptions(options),
899
- serviceTierFromOptions(options),
900
- context.hash,
901
- ];
902
- const lang = languageFromOptions(options);
903
- if (lang) parts.push(`lang:${lang}`);
904
- return parts.join(":");
905
- }
906
-
907
- function latestDigestCacheKey(options: PeriodDigestOptions) {
908
- const period = normalizePeriod(options.period);
909
- const window = resolvePeriodDigestWindow(options);
910
- const identity = {
911
- period,
912
- day:
913
- period === "today" || period === "yesterday"
914
- ? window.since
915
- : localDateStart(new Date()).toISOString(),
916
- since: options.since?.trim() || null,
917
- until: options.until?.trim() || null,
918
- account: options.account?.trim() || null,
919
- includeDms: Boolean(options.includeDms),
920
- maxTweets: Math.max(
921
- 20,
922
- Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
923
- ),
924
- maxLinks: Math.max(3, Math.trunc(options.maxLinks ?? DEFAULT_MAX_LINKS)),
925
- model: modelFromOptions(options),
926
- language: languageFromOptions(options) ?? null,
927
- reasoningEffort: reasoningEffortFromOptions(options),
928
- serviceTier: serviceTierFromOptions(options),
929
- };
930
- return `period-digest-latest:v1:${createHash("sha1")
931
- .update(JSON.stringify(identity))
932
- .digest("hex")}`;
933
- }
934
-
935
- function collectDigestTweetIds(digest: PeriodDigest) {
936
- const tweetIds = new Set(digest.sourceTweetIds);
937
- for (const topic of digest.keyTopics) {
938
- for (const tweetId of topic.tweetIds) tweetIds.add(tweetId);
939
- }
940
- for (const link of digest.notableLinks) {
941
- for (const tweetId of link.sourceTweetIds) tweetIds.add(tweetId);
942
- }
943
- for (const action of digest.actionItems) {
944
- if (action.tweetId) tweetIds.add(action.tweetId);
945
- }
946
- return [...tweetIds];
947
- }
948
-
949
- function enrichContextWithCitedTweets(
950
- context: PeriodDigestContext,
951
- digest: PeriodDigest,
952
- ) {
953
- const existingIds = new Set(context.tweets.map((tweet) => tweet.id));
954
- const missingIds = collectDigestTweetIds(digest).filter(
955
- (tweetId) => !existingIds.has(tweetId.replace(/^tweet_/, "")),
956
- );
957
- if (missingIds.length === 0) return context;
958
- const citedTweets = getTweetsByIds(missingIds, context.account).map(
959
- compactEmbeddedTweet,
960
- );
961
- return citedTweets.length > 0
962
- ? { ...context, tweets: [...context.tweets, ...citedTweets] }
963
- : context;
964
- }
965
-
966
- interface CachedPeriodDigestValue {
967
- context?: PeriodDigestContext;
968
- digest: PeriodDigest;
969
- markdown: string;
970
- model: string;
971
- reasoningEffort: string;
972
- serviceTier: string;
973
- updatedAt?: string;
974
- }
975
-
976
- function cachedDigestResult(
977
- cached: { value: CachedPeriodDigestValue; updatedAt: string },
978
- context: PeriodDigestContext,
979
- ): PeriodDigestRunResult {
980
- const digest = PeriodDigestSchema.parse(cached.value.digest);
981
- return {
982
- context: enrichContextWithCitedTweets(context, digest),
983
- digest,
984
- markdown: cached.value.markdown,
985
- model: cached.value.model,
986
- reasoningEffort: cached.value.reasoningEffort,
987
- serviceTier: cached.value.serviceTier,
988
- cached: true,
989
- updatedAt: cached.value.updatedAt ?? cached.updatedAt,
990
- };
991
- }
992
-
993
- function isFreshDigestCache(updatedAt: string) {
994
- const timestamp = Date.parse(updatedAt);
995
- return (
996
- Number.isFinite(timestamp) &&
997
- Date.now() - timestamp <= DEFAULT_DIGEST_FRESHNESS_MS
998
- );
999
- }
1000
-
1001
- function emitCachedDigest(
1002
- result: PeriodDigestRunResult,
1003
- handlers: PeriodDigestStreamHandlers,
1004
- ) {
1005
- handlers.onEvent?.({ type: "start", context: result.context, cached: true });
1006
- handlers.onDelta?.(result.markdown);
1007
- handlers.onEvent?.({ type: "delta", delta: result.markdown });
1008
- handlers.onEvent?.({ type: "done", result });
1009
- }
1010
-
1011
- function buildPrompt(
1012
- context: PeriodDigestContext,
1013
- options?: { language?: string },
1014
- ) {
1015
- const language = normalizeDigestLanguage(options?.language);
1016
- const promptTweets = context.tweets.map((tweet) => ({
1017
- id: tweet.id,
1018
- url: tweet.url,
1019
- source: tweet.source,
1020
- author: tweet.author,
1021
- name: tweet.name,
1022
- bio: tweet.authorProfile.bio,
1023
- followersCount: tweet.authorProfile.followersCount,
1024
- createdAt: tweet.createdAt,
1025
- text: tweet.text,
1026
- likeCount: tweet.likeCount,
1027
- liked: tweet.liked,
1028
- bookmarked: tweet.bookmarked,
1029
- needsReply: tweet.needsReply,
1030
- replyToId: tweet.replyToId,
1031
- replyToTweet: tweet.replyToTweet,
1032
- }));
1033
- const fitDataset = () => {
1034
- let tweetCount = promptTweets.length;
1035
- let dmCount = context.dms.length;
1036
- let linkCount = context.links.length;
1037
- const datasetFor = (tweets: number, dms: number, links: number) => ({
1038
- tweets: promptTweets.slice(0, tweets),
1039
- dms: context.dms.slice(0, dms),
1040
- links: context.links.slice(0, links),
1041
- });
1042
- const lengthFor = (tweets: number, dms: number, links: number) =>
1043
- JSON.stringify(datasetFor(tweets, dms, links)).length;
1044
- const fitCount = (max: number, fits: (count: number) => boolean) => {
1045
- let low = 0;
1046
- let high = max;
1047
- let best = 0;
1048
- while (low <= high) {
1049
- const mid = Math.floor((low + high) / 2);
1050
- if (fits(mid)) {
1051
- best = mid;
1052
- low = mid + 1;
1053
- } else {
1054
- high = mid - 1;
1055
- }
1056
- }
1057
- return best;
1058
- };
1059
- if (lengthFor(tweetCount, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS) {
1060
- return {
1061
- dataset: datasetFor(tweetCount, dmCount, linkCount),
1062
- tweetCount,
1063
- };
1064
- }
1065
- dmCount = fitCount(
1066
- dmCount,
1067
- (count) =>
1068
- lengthFor(tweetCount, count, linkCount) <= MAX_PROMPT_DATA_CHARS,
1069
- );
1070
- if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
1071
- linkCount = fitCount(
1072
- linkCount,
1073
- (count) =>
1074
- lengthFor(tweetCount, dmCount, count) <= MAX_PROMPT_DATA_CHARS,
1075
- );
1076
- }
1077
- if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
1078
- tweetCount = fitCount(
1079
- tweetCount,
1080
- (count) =>
1081
- lengthFor(count, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS,
1082
- );
1083
- }
1084
- return { dataset: datasetFor(tweetCount, dmCount, linkCount), tweetCount };
1085
- };
1086
- const { dataset, tweetCount } = fitDataset();
1087
-
1088
- return `Window: ${context.window.label}
1089
- Since: ${context.window.since}
1090
- Until: ${context.window.until}
1091
- Sources: ${JSON.stringify(context.counts)}
1092
- Prompt tweets: ${String(tweetCount)} of ${String(context.tweets.length)} selected context tweets
1093
-
1094
- Write a high-signal "what happened" report from this local Twitter/X dataset.
1095
-
1096
- Requirements:
1097
- - Stream one readable Markdown report first. The UI will show this text directly; do not rely on separate cards or structured summaries.
1098
- - Target 700-1100 words when there is enough data.
1099
- - Start with a 2-3 sentence lead that immediately says what people are talking about.
1100
- - Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies. Translate these section titles when a report language is requested.
1101
- - When a tweet has replyToTweet, use that parent context to understand what the author was replying to and whether Peter already joined the conversation.
1102
- - Use bullets under each section. Each bullet should be specific and explain why it matters.
1103
- - For tweets: cite every claim with inline tweet ids at the end of the relevant sentence or bullet, e.g. (tweet_123, tweet_456). These citations become hoverable source links.
1104
- - For links: emit normal Markdown links with no space between the label and URL, e.g. [title](https://example.com), then cite the sharing tweet ids in the same bullet.
1105
- - Prefer synthesis over chronology. Group repeated chatter into one bullet.
1106
- - Mention handles when useful, but do not make the report a list of handles.
1107
- - Do not include a generic "Action items" section.
1108
- - If there is no data, say that plainly in one short paragraph.
1109
- - DMs are private context and only present when explicitly included.
1110
- - After the Markdown, output a blank line, then a line containing only three hyphens, then one compact JSON object.
1111
- - Keep actionItems empty unless you wrote a "Worth replying to" section.
1112
- - Put every tweet id cited in the Markdown into sourceTweetIds.
1113
- - JSON shape: { "title": string, "summary": string, "keyTopics": [{ "title": string, "summary": string, "tweetIds": string[], "handles": string[] }], "notableLinks": [{ "title": string, "url": string, "why": string, "sourceTweetIds": string[] }], "people": [{ "handle": string, "name"?: string, "why": string }], "actionItems": [{ "kind": "reply"|"follow_up"|"read"|"sync", "label": string, "tweetId"?: string, "dmConversationId"?: string }], "sourceTweetIds": string[] }
1114
- ${language ? `- Write all human-readable prose, including section titles and JSON prose fields, in ${language}.\n- Preserve handles, URLs, tweet ids, and JSON property names exactly.` : ""}
1115
-
1116
- Dataset:
1117
- ${JSON.stringify(dataset)}`;
1118
- }
1119
-
1120
- function fallbackDigest(
1121
- context: PeriodDigestContext,
1122
- markdown: string,
1123
- language?: string,
1124
- ): PeriodDigest {
1125
- const normalized = markdown.replaceAll(/\s+/g, " ").trim();
1126
- const heading = markdown
1127
- .split("\n")
1128
- .map((line) => line.match(/^#{1,6}\s+(.+)$/)?.[1]?.trim())
1129
- .find(Boolean);
1130
- const neutralFallback = language ? `[${language}]` : undefined;
1131
- return {
1132
- title:
1133
- heading?.slice(0, 160) ??
1134
- neutralFallback ??
1135
- `${context.window.label} digest`,
1136
- summary:
1137
- normalized.slice(0, 280) ||
1138
- neutralFallback ||
1139
- "No model summary was returned.",
1140
- keyTopics: [],
1141
- notableLinks: [],
1142
- people: [],
1143
- actionItems: [],
1144
- sourceTweetIds: context.tweets.slice(0, 20).map((tweet) => tweet.id),
1145
- };
1146
- }
1147
-
1148
- function parseDigestFromHybridText(
1149
- context: PeriodDigestContext,
1150
- rawText: string,
1151
- language?: string,
1152
- ): { digest: PeriodDigest; markdown: string } {
1153
- const [markdownPart, jsonPart] = rawText.split(DELIMITER_PATTERN);
1154
- const markdown = (markdownPart ?? rawText).trim();
1155
- const candidate = jsonPart?.slice(
1156
- jsonPart.indexOf("{"),
1157
- jsonPart.lastIndexOf("}") + 1,
1158
- );
1159
- if (candidate?.startsWith("{")) {
1160
- try {
1161
- return {
1162
- markdown,
1163
- digest: PeriodDigestSchema.parse(JSON.parse(candidate)),
1164
- };
1165
- } catch {
1166
- return {
1167
- markdown,
1168
- digest: fallbackDigest(context, markdown, language),
1169
- };
1170
- }
1171
- }
1172
- return { markdown, digest: fallbackDigest(context, markdown, language) };
1173
- }
1174
-
1175
- function emitVisibleDelta(
1176
- state: OpenAIStreamState,
1177
- delta: string,
1178
- handlers: PeriodDigestStreamHandlers,
1179
- ) {
1180
- state.rawText += delta;
1181
- if (state.jsonMode) return;
1182
-
1183
- const combined = state.pendingVisible + delta;
1184
- const delimiterIndex = combined.search(DELIMITER_PATTERN);
1185
- if (delimiterIndex >= 0) {
1186
- const visible = combined.slice(0, delimiterIndex);
1187
- if (visible) {
1188
- handlers.onDelta?.(visible);
1189
- handlers.onEvent?.({ type: "delta", delta: visible });
1190
- }
1191
- state.pendingVisible = "";
1192
- state.jsonMode = true;
1193
- return;
1194
- }
1195
-
1196
- if (combined.length <= VISIBLE_DELIMITER_HOLD) {
1197
- state.pendingVisible = combined;
1198
- return;
1199
- }
1200
-
1201
- const visible = combined.slice(0, -VISIBLE_DELIMITER_HOLD);
1202
- state.pendingVisible = combined.slice(-VISIBLE_DELIMITER_HOLD);
1203
- if (visible) {
1204
- handlers.onDelta?.(visible);
1205
- handlers.onEvent?.({ type: "delta", delta: visible });
1206
- }
1207
- }
1208
-
1209
- function flushPendingVisible(
1210
- state: OpenAIStreamState,
1211
- handlers: PeriodDigestStreamHandlers,
1212
- ) {
1213
- if (state.jsonMode || !state.pendingVisible) return;
1214
- const delta = state.pendingVisible;
1215
- state.pendingVisible = "";
1216
- handlers.onDelta?.(delta);
1217
- handlers.onEvent?.({ type: "delta", delta });
1218
- }
1219
-
1220
- function handleOpenAIEvent(
1221
- state: OpenAIStreamState,
1222
- event: Record<string, unknown>,
1223
- handlers: PeriodDigestStreamHandlers,
1224
- ) {
1225
- const type = typeof event.type === "string" ? event.type : "";
1226
- if (
1227
- type === "response.output_text.delta" &&
1228
- typeof event.delta === "string"
1229
- ) {
1230
- emitVisibleDelta(state, event.delta, handlers);
1231
- return;
1232
- }
1233
- if (type === "response.completed") {
1234
- const response = event.response;
1235
- if (response && typeof response === "object") {
1236
- const record = response as Record<string, unknown>;
1237
- state.responseId = typeof record.id === "string" ? record.id : undefined;
1238
- state.usage = record.usage;
1239
- }
1240
- return;
1241
- }
1242
- if (type === "response.error" || type === "error") {
1243
- const error = event.error;
1244
- state.error =
1245
- error && typeof error === "object" && "message" in error
1246
- ? String((error as { message?: unknown }).message)
1247
- : "OpenAI stream failed";
1248
- return;
1249
- }
1250
- if (type === "response.failed" || type === "response.incomplete") {
1251
- const response = event.response;
1252
- const record =
1253
- response && typeof response === "object"
1254
- ? (response as Record<string, unknown>)
1255
- : {};
1256
- const error = record.error;
1257
- const incomplete = record.incomplete_details;
1258
- state.error =
1259
- error && typeof error === "object" && "message" in error
1260
- ? String((error as { message?: unknown }).message)
1261
- : incomplete && typeof incomplete === "object" && "reason" in incomplete
1262
- ? `OpenAI response incomplete: ${String((incomplete as { reason?: unknown }).reason)}`
1263
- : "OpenAI stream failed";
1264
- }
1265
- }
1266
-
1267
- function processSseChunk(
1268
- state: OpenAIStreamState,
1269
- chunk: string,
1270
- handlers: PeriodDigestStreamHandlers,
1271
- ) {
1272
- state.eventBuffer += chunk;
1273
- let boundary = state.eventBuffer.indexOf("\n\n");
1274
- while (boundary >= 0) {
1275
- const block = state.eventBuffer.slice(0, boundary);
1276
- state.eventBuffer = state.eventBuffer.slice(boundary + 2);
1277
- const data = block
1278
- .split("\n")
1279
- .filter((line) => line.startsWith("data:"))
1280
- .map((line) => line.slice(5).trimStart())
1281
- .join("\n");
1282
- if (data && data !== "[DONE]") {
1283
- try {
1284
- handleOpenAIEvent(
1285
- state,
1286
- JSON.parse(data) as Record<string, unknown>,
1287
- handlers,
1288
- );
1289
- } catch {
1290
- // Ignore malformed event frames; the final JSON parse will decide result quality.
1291
- }
1292
- }
1293
- boundary = state.eventBuffer.indexOf("\n\n");
1294
- }
1295
- }
1296
-
1297
- function createOpenAIRequestBody(
1298
- context: PeriodDigestContext,
1299
- options: PeriodDigestOptions,
1300
- ) {
1301
- return {
1302
- model: modelFromOptions(options),
1303
- reasoning: { effort: reasoningEffortFromOptions(options) },
1304
- service_tier: serviceTierFromOptions(options),
1305
- store: false,
1306
- stream: true,
1307
- max_output_tokens: 7000,
1308
- input: [
1309
- {
1310
- role: "system",
1311
- content:
1312
- "You are a precise local Twitter archive analyst. Stream Markdown first, then emit the requested JSON object after the delimiter. Do not invent events not present in the dataset.",
1313
- },
1314
- {
1315
- role: "user",
1316
- content: buildPrompt(context, {
1317
- language: languageFromOptions(options),
1318
- }),
1319
- },
1320
- ],
1321
- };
1322
- }
1323
-
1324
- function readOpenAIStreamEffect(
1325
- response: Response,
1326
- context: PeriodDigestContext,
1327
- options: PeriodDigestOptions,
1328
- handlers: PeriodDigestStreamHandlers,
1329
- ): Effect.Effect<PeriodDigestRunResult, Error> {
1330
- const reader = response.body?.getReader();
1331
- if (!reader) {
1332
- return Effect.fail(new Error("OpenAI response did not include a stream"));
1333
- }
1334
-
1335
- const decoder = new TextDecoder();
1336
- const state: OpenAIStreamState = {
1337
- eventBuffer: "",
1338
- rawText: "",
1339
- pendingVisible: "",
1340
- jsonMode: false,
1341
- };
1342
-
1343
- return Effect.gen(function* () {
1344
- for (;;) {
1345
- const { done, value } = yield* tryDigestPromise(() => reader.read());
1346
- if (!done) {
1347
- processSseChunk(
1348
- state,
1349
- decoder.decode(value, { stream: true }),
1350
- handlers,
1351
- );
1352
- continue;
1353
- }
1354
-
1355
- flushPendingVisible(state, handlers);
1356
- if (state.error) {
1357
- return yield* Effect.fail(new Error(state.error));
1358
- }
1359
-
1360
- const parsed = yield* tryDigestSync(() =>
1361
- parseDigestFromHybridText(
1362
- context,
1363
- state.rawText,
1364
- languageFromOptions(options),
1365
- ),
1366
- );
1367
- const enrichedContext = yield* tryDigestSync(() =>
1368
- enrichContextWithCitedTweets(context, parsed.digest),
1369
- );
1370
- const cacheKey = digestCacheKey(context, options);
1371
- const updatedAt = yield* tryDigestSync(() =>
1372
- writeSyncCache(cacheKey, {
1373
- digest: parsed.digest,
1374
- markdown: parsed.markdown,
1375
- model: modelFromOptions(options),
1376
- reasoningEffort: reasoningEffortFromOptions(options),
1377
- serviceTier: serviceTierFromOptions(options),
1378
- usage: state.usage,
1379
- responseId: state.responseId,
1380
- }),
1381
- );
1382
- const result: PeriodDigestRunResult = {
1383
- context: enrichedContext,
1384
- digest: parsed.digest,
1385
- markdown: parsed.markdown,
1386
- model: modelFromOptions(options),
1387
- reasoningEffort: reasoningEffortFromOptions(options),
1388
- serviceTier: serviceTierFromOptions(options),
1389
- cached: false,
1390
- updatedAt,
1391
- };
1392
- yield* tryDigestSync(() =>
1393
- writeSyncCache(latestDigestCacheKey(options), {
1394
- context: result.context,
1395
- digest: result.digest,
1396
- markdown: result.markdown,
1397
- model: result.model,
1398
- reasoningEffort: result.reasoningEffort,
1399
- serviceTier: result.serviceTier,
1400
- updatedAt: result.updatedAt,
1401
- }),
1402
- );
1403
- handlers.onEvent?.({ type: "done", result });
1404
- return result;
1405
- }
1406
- }).pipe(
1407
- Effect.ensuring(
1408
- Effect.sync(() => {
1409
- reader.releaseLock();
1410
- }),
1411
- ),
1412
- );
1413
- }
1414
-
1415
- export function streamPeriodDigestEffect(
1416
- options: PeriodDigestOptions = {},
1417
- handlers: PeriodDigestStreamHandlers = {},
1418
- ): Effect.Effect<PeriodDigestRunResult, Error> {
1419
- return Effect.gen(function* () {
1420
- const resolvedOptions = {
1421
- ...options,
1422
- language: yield* tryDigestSync(() => languageFromOptions(options)),
1423
- };
1424
- const latestCached = resolvedOptions.refresh
1425
- ? null
1426
- : !resolvedOptions.liveSync
1427
- ? yield* tryDigestSync(() =>
1428
- readSyncCache<CachedPeriodDigestValue>(
1429
- latestDigestCacheKey(resolvedOptions),
1430
- ),
1431
- )
1432
- : null;
1433
- const latestContext = latestCached?.value.context;
1434
- if (
1435
- latestCached &&
1436
- latestContext &&
1437
- isFreshDigestCache(latestCached.value.updatedAt ?? latestCached.updatedAt)
1438
- ) {
1439
- const result = yield* tryDigestSync(() =>
1440
- cachedDigestResult(latestCached, latestContext),
1441
- );
1442
- emitCachedDigest(result, handlers);
1443
- return result;
1444
- }
1445
-
1446
- yield* refreshPeriodDigestInputsEffect(
1447
- resolvedOptions,
1448
- { threads: false },
1449
- handlers,
1450
- ).pipe(Effect.catchAll(() => Effect.void));
1451
- let context = yield* tryDigestSync(() =>
1452
- collectPeriodDigestContext(resolvedOptions),
1453
- );
1454
- let cacheKey = digestCacheKey(context, resolvedOptions);
1455
- const cached = resolvedOptions.refresh
1456
- ? null
1457
- : yield* tryDigestSync(() =>
1458
- readSyncCache<CachedPeriodDigestValue>(cacheKey),
1459
- );
1460
-
1461
- if (cached) {
1462
- const result = yield* tryDigestSync(() =>
1463
- cachedDigestResult(cached, context),
1464
- );
1465
- yield* tryDigestSync(() =>
1466
- writeSyncCache(latestDigestCacheKey(resolvedOptions), {
1467
- context: result.context,
1468
- digest: result.digest,
1469
- markdown: result.markdown,
1470
- model: result.model,
1471
- reasoningEffort: result.reasoningEffort,
1472
- serviceTier: result.serviceTier,
1473
- updatedAt: result.updatedAt,
1474
- }),
1475
- );
1476
- emitCachedDigest(result, handlers);
1477
- return result;
1478
- }
1479
-
1480
- yield* refreshPeriodDigestInputsEffect(
1481
- resolvedOptions,
1482
- {
1483
- timeline: false,
1484
- mentions: false,
1485
- threads: true,
1486
- threadTweetIds: context.tweets
1487
- .filter((tweet) => tweet.source === "mentions")
1488
- .map((tweet) => tweet.id),
1489
- },
1490
- handlers,
1491
- ).pipe(Effect.catchAll(() => Effect.void));
1492
- context = yield* tryDigestSync(() =>
1493
- collectPeriodDigestContext(resolvedOptions),
1494
- );
1495
- cacheKey = digestCacheKey(context, resolvedOptions);
1496
-
1497
- const apiKey = process.env.OPENAI_API_KEY;
1498
- if (!apiKey) {
1499
- return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
1500
- }
1501
-
1502
- handlers.onEvent?.({ type: "start", context, cached: false });
1503
- emitDigestStatus(handlers, "Streaming AI summary");
1504
- const response = yield* tryDigestPromise(() =>
1505
- fetch("https://api.openai.com/v1/responses", {
1506
- method: "POST",
1507
- signal: resolvedOptions.signal,
1508
- headers: {
1509
- authorization: `Bearer ${apiKey}`,
1510
- "content-type": "application/json",
1511
- },
1512
- body: JSON.stringify(createOpenAIRequestBody(context, resolvedOptions)),
1513
- }),
1514
- );
1515
- if (!response.ok) {
1516
- const text = yield* tryDigestPromise(() => response.text());
1517
- return yield* Effect.fail(
1518
- new Error(
1519
- `OpenAI request failed: ${String(response.status)} ${text.slice(
1520
- 0,
1521
- 400,
1522
- )}`,
1523
- ),
1524
- );
1525
- }
1526
- return yield* readOpenAIStreamEffect(
1527
- response,
1528
- context,
1529
- resolvedOptions,
1530
- handlers,
1531
- );
1532
- });
1533
- }
1534
-
1535
- export function streamPeriodDigest(
1536
- options: PeriodDigestOptions = {},
1537
- handlers: PeriodDigestStreamHandlers = {},
1538
- ): Promise<PeriodDigestRunResult> {
1539
- return runEffectPromise(streamPeriodDigestEffect(options, handlers));
1540
- }
1541
-
1542
- export const __test__ = {
1543
- PeriodDigestSchema,
1544
- buildPrompt,
1545
- digestCacheKey,
1546
- languageFromOptions,
1547
- normalizeDigestLanguage,
1548
- readOpenAIStreamEffect,
1549
- parseDigestFromHybridText,
1550
- processSseChunk,
1551
- resolvePeriodDigestWindow,
1552
- };