birdclaw 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +32 -2
  3. package/package.json +29 -30
  4. package/scripts/build-docs-site.mjs +39 -12
  5. package/src/cli.ts +457 -26
  6. package/src/components/AppNav.tsx +10 -0
  7. package/src/components/MarkdownViewer.tsx +438 -72
  8. package/src/components/ProfileAnalysisStream.tsx +428 -0
  9. package/src/components/ProfilePreview.tsx +120 -9
  10. package/src/components/SavedTimelineView.tsx +30 -8
  11. package/src/components/SyncNowButton.tsx +5 -2
  12. package/src/components/TimelineCard.tsx +20 -4
  13. package/src/components/TimelineRouteFrame.tsx +16 -0
  14. package/src/components/TweetRichText.tsx +36 -12
  15. package/src/components/useTimelineRouteData.ts +74 -6
  16. package/src/lib/account-sync-job.ts +15 -3
  17. package/src/lib/archive-finder.ts +1 -1
  18. package/src/lib/archive-import.ts +245 -7
  19. package/src/lib/authored-live.ts +1 -0
  20. package/src/lib/avatar-cache.ts +50 -0
  21. package/src/lib/backup.ts +4 -3
  22. package/src/lib/bird.ts +33 -0
  23. package/src/lib/config.ts +35 -2
  24. package/src/lib/data-sources.ts +219 -0
  25. package/src/lib/db.ts +62 -1
  26. package/src/lib/geocoding.ts +296 -0
  27. package/src/lib/link-insights.ts +2 -0
  28. package/src/lib/location.ts +137 -0
  29. package/src/lib/mention-threads-live.ts +94 -1
  30. package/src/lib/mentions-live.ts +187 -40
  31. package/src/lib/network-map.ts +382 -0
  32. package/src/lib/period-digest.ts +468 -29
  33. package/src/lib/profile-analysis.ts +1272 -0
  34. package/src/lib/profile-bio-entities.ts +1 -1
  35. package/src/lib/queries.ts +14 -4
  36. package/src/lib/search-discussion.ts +1016 -0
  37. package/src/lib/timeline-live.ts +272 -19
  38. package/src/lib/tweet-account-edges.ts +2 -0
  39. package/src/lib/tweet-render.ts +141 -1
  40. package/src/lib/tweet-search-live.ts +565 -0
  41. package/src/lib/types.ts +37 -2
  42. package/src/lib/ui.ts +1 -1
  43. package/src/lib/web-sync.ts +7 -2
  44. package/src/lib/xurl-rate-limits.ts +267 -0
  45. package/src/lib/xurl.ts +551 -41
  46. package/src/routeTree.gen.ts +231 -0
  47. package/src/routes/__root.tsx +5 -6
  48. package/src/routes/api/data-sources.tsx +24 -0
  49. package/src/routes/api/network-map.tsx +55 -0
  50. package/src/routes/api/period-digest.tsx +29 -3
  51. package/src/routes/api/profile-analysis.tsx +152 -0
  52. package/src/routes/api/query.tsx +1 -0
  53. package/src/routes/api/search-discussion.tsx +169 -0
  54. package/src/routes/api/xurl-rate-limits.tsx +24 -0
  55. package/src/routes/data-sources.tsx +257 -0
  56. package/src/routes/discuss.tsx +419 -0
  57. package/src/routes/network-map.tsx +1035 -0
  58. package/src/routes/profile-analyze.tsx +112 -0
  59. package/src/routes/profiles.$handle.tsx +228 -0
  60. package/src/routes/rate-limits.tsx +309 -0
  61. package/src/routes/today.tsx +22 -8
  62. package/src/styles.css +22 -0
@@ -1,11 +1,15 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { Effect } from "effect";
3
3
  import { z } from "zod";
4
+ import { maybeAutoSyncBackupEffect } from "./backup";
4
5
  import { runEffectPromise, tryPromise } from "./effect-runtime";
5
6
  import { getLinkInsights } from "./link-insights";
7
+ import { syncMentionThreadsEffect } from "./mention-threads-live";
8
+ import { syncMentionsEffect } from "./mentions-live";
6
9
  import { listDmConversations, listTimelineItems } from "./queries";
7
10
  import { readSyncCache, writeSyncCache } from "./sync-cache";
8
- import type { ProfileRecord } from "./types";
11
+ import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
12
+ import type { ProfileRecord, TweetEntities } from "./types";
9
13
 
10
14
  export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
11
15
  export type PeriodDigestSourceKind =
@@ -24,11 +28,19 @@ export interface PeriodDigestOptions {
24
28
  includeDms?: boolean;
25
29
  refresh?: boolean;
26
30
  model?: string;
31
+ language?: string;
27
32
  reasoningEffort?: "minimal" | "low" | "medium" | "high";
28
33
  serviceTier?: "default" | "flex" | "priority";
29
34
  signal?: AbortSignal;
30
35
  maxTweets?: number;
31
36
  maxLinks?: number;
37
+ liveSync?: boolean;
38
+ liveSyncMode?: HomeTimelineMode;
39
+ liveTimelineLimit?: number;
40
+ liveTimelineMaxPages?: number;
41
+ liveMentionsLimit?: number;
42
+ liveMentionsMaxPages?: number;
43
+ liveThreadLimit?: number;
32
44
  }
33
45
 
34
46
  export interface PeriodDigestWindow {
@@ -54,6 +66,7 @@ export interface PeriodDigestStreamHandlers {
54
66
  }
55
67
 
56
68
  export type PeriodDigestStreamEvent =
69
+ | { type: "status"; label: string; detail?: string }
57
70
  | { type: "start"; context: PeriodDigestContext; cached: boolean }
58
71
  | { type: "delta"; delta: string }
59
72
  | { type: "done"; result: PeriodDigestRunResult }
@@ -96,6 +109,32 @@ const PeriodDigestSchema = z.object({
96
109
  sourceTweetIds: z.array(z.string()).default([]),
97
110
  });
98
111
 
112
+ const MAX_DIGEST_LANGUAGE_LENGTH = 64;
113
+
114
+ export function normalizeDigestLanguage(
115
+ value: string | undefined,
116
+ ): string | undefined {
117
+ const trimmed = value?.trim();
118
+ if (!trimmed) return undefined;
119
+ if (
120
+ trimmed.length > MAX_DIGEST_LANGUAGE_LENGTH ||
121
+ !/^[a-z0-9]+(?:-[a-z0-9]+)*$/i.test(trimmed)
122
+ ) {
123
+ throw new Error(
124
+ "Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
125
+ );
126
+ }
127
+ try {
128
+ const [canonical] = Intl.getCanonicalLocales(trimmed);
129
+ if (!canonical) throw new Error("missing canonical locale");
130
+ return canonical;
131
+ } catch {
132
+ throw new Error(
133
+ "Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
134
+ );
135
+ }
136
+ }
137
+
99
138
  export type PeriodDigest = z.infer<typeof PeriodDigestSchema>;
100
139
 
101
140
  interface CompactTweet {
@@ -107,10 +146,20 @@ interface CompactTweet {
107
146
  authorProfile: ProfileRecord;
108
147
  createdAt: string;
109
148
  text: string;
149
+ entities?: TweetEntities;
110
150
  likeCount: number;
111
151
  liked: boolean;
112
152
  bookmarked: boolean;
113
153
  needsReply: boolean;
154
+ replyToId?: string | null;
155
+ replyToTweet?: {
156
+ id: string;
157
+ url: string;
158
+ author: string;
159
+ name: string;
160
+ createdAt: string;
161
+ text: string;
162
+ } | null;
114
163
  }
115
164
 
116
165
  interface CompactDm {
@@ -166,8 +215,14 @@ interface OpenAIStreamState {
166
215
  const DEFAULT_MODEL = "gpt-5.5";
167
216
  const DEFAULT_REASONING_EFFORT = "medium";
168
217
  const DEFAULT_SERVICE_TIER = "priority";
169
- const DEFAULT_MAX_TWEETS = 300;
218
+ const DEFAULT_MAX_TWEETS = 2_500;
170
219
  const DEFAULT_MAX_LINKS = 12;
220
+ const DEFAULT_LIVE_TIMELINE_MAX_PAGES = undefined;
221
+ const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
222
+ const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
223
+ const DEFAULT_LIVE_THREAD_LIMIT = 12;
224
+ const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
225
+ const MAX_PROMPT_DATA_CHARS = 1_200_000;
171
226
  const DELIMITER_PATTERN = /\n---\s*\n/;
172
227
  const VISIBLE_DELIMITER_HOLD = 8;
173
228
 
@@ -204,6 +259,12 @@ function parseDate(value: string | undefined) {
204
259
  return Number.isNaN(parsed.getTime()) ? null : parsed;
205
260
  }
206
261
 
262
+ function floorIsoToHour(value: string) {
263
+ const date = new Date(value);
264
+ date.setUTCMinutes(0, 0, 0);
265
+ return date.toISOString();
266
+ }
267
+
207
268
  function normalizePeriod(value: string | undefined): PeriodDigestPreset {
208
269
  const normalized = value?.trim().toLowerCase();
209
270
  if (normalized === "yesterday") return "yesterday";
@@ -273,6 +334,16 @@ function compactTweet(
273
334
  source: PeriodDigestSourceKind,
274
335
  item: ReturnType<typeof listTimelineItems>[number],
275
336
  ): CompactTweet {
337
+ const replyToTweet = item.replyToTweet
338
+ ? {
339
+ id: item.replyToTweet.id,
340
+ url: tweetUrl(item.replyToTweet.author.handle, item.replyToTweet.id),
341
+ author: item.replyToTweet.author.handle,
342
+ name: item.replyToTweet.author.displayName,
343
+ createdAt: item.replyToTweet.createdAt,
344
+ text: item.replyToTweet.text,
345
+ }
346
+ : null;
276
347
  return {
277
348
  id: item.id,
278
349
  url: tweetUrl(item.author.handle, item.id),
@@ -282,10 +353,13 @@ function compactTweet(
282
353
  authorProfile: item.author,
283
354
  createdAt: item.createdAt,
284
355
  text: item.text,
356
+ entities: item.entities,
285
357
  likeCount: item.likeCount,
286
358
  liked: item.liked,
287
359
  bookmarked: item.bookmarked,
288
360
  needsReply: !item.isReplied,
361
+ replyToId: item.replyToId ?? null,
362
+ replyToTweet,
289
363
  };
290
364
  }
291
365
 
@@ -417,6 +491,9 @@ function contextHash(context: Omit<PeriodDigestContext, "hash">) {
417
491
  tweet.liked,
418
492
  tweet.bookmarked,
419
493
  tweet.needsReply,
494
+ tweet.replyToId,
495
+ tweet.replyToTweet?.id,
496
+ tweet.replyToTweet?.text,
420
497
  ]),
421
498
  dms: context.dms.map((dm) => [
422
499
  dm.id,
@@ -513,6 +590,12 @@ export function collectPeriodDigestContext(
513
590
  };
514
591
  }
515
592
 
593
+ function languageFromOptions(options: PeriodDigestOptions) {
594
+ return normalizeDigestLanguage(
595
+ options.language ?? process.env.BIRDCLAW_DIGEST_LANGUAGE,
596
+ );
597
+ }
598
+
516
599
  function modelFromOptions(options: PeriodDigestOptions) {
517
600
  return options.model ?? process.env.BIRDCLAW_AI_MODEL ?? DEFAULT_MODEL;
518
601
  }
@@ -537,20 +620,270 @@ function serviceTierFromOptions(options: PeriodDigestOptions) {
537
620
  );
538
621
  }
539
622
 
623
+ function boundedPositiveInteger(
624
+ value: number | undefined,
625
+ fallback: number,
626
+ max: number,
627
+ ) {
628
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
629
+ return fallback;
630
+ }
631
+ return Math.min(max, Math.floor(value));
632
+ }
633
+
634
+ function emitDigestStatus(
635
+ handlers: PeriodDigestStreamHandlers,
636
+ label: string,
637
+ detail?: string,
638
+ ) {
639
+ handlers.onEvent?.({
640
+ type: "status",
641
+ label,
642
+ ...(detail ? { detail } : {}),
643
+ });
644
+ }
645
+
646
+ function formatFetchedStatus({
647
+ fetched,
648
+ total,
649
+ noun,
650
+ }: {
651
+ fetched: number;
652
+ total: number;
653
+ noun: string;
654
+ }) {
655
+ const count = `${String(Math.min(fetched, total))}/${String(total)}`;
656
+ return `Fetched ${count} ${noun}`;
657
+ }
658
+
659
+ function formatPageDetail({
660
+ source,
661
+ page,
662
+ maxPages,
663
+ done,
664
+ }: {
665
+ source: string;
666
+ page?: number;
667
+ maxPages?: number;
668
+ done: boolean;
669
+ }) {
670
+ const pageText =
671
+ page === undefined
672
+ ? undefined
673
+ : `page ${String(page)}${maxPages === undefined ? "" : `/${String(maxPages)}`}`;
674
+ return [source, pageText, done ? "done" : undefined]
675
+ .filter(Boolean)
676
+ .join(" · ");
677
+ }
678
+
679
+ function refreshPeriodDigestInputsEffect(
680
+ options: PeriodDigestOptions,
681
+ phase: {
682
+ timeline?: boolean;
683
+ mentions?: boolean;
684
+ threads?: boolean;
685
+ threadTweetIds?: string[];
686
+ } = {},
687
+ handlers: PeriodDigestStreamHandlers = {},
688
+ ): Effect.Effect<void, unknown> {
689
+ if (!options.liveSync) {
690
+ return Effect.void;
691
+ }
692
+ const includeTimeline = phase.timeline ?? true;
693
+ const includeMentions = phase.mentions ?? true;
694
+ const includeThreads = phase.threads ?? true;
695
+ const window = resolvePeriodDigestWindow(options);
696
+ const liveStartTime = floorIsoToHour(window.since);
697
+ const mode = options.liveSyncMode ?? "xurl";
698
+ const contextTweetBudget = Math.max(
699
+ 20,
700
+ Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
701
+ );
702
+ const timelineLimit =
703
+ options.liveTimelineLimit === undefined
704
+ ? undefined
705
+ : boundedPositiveInteger(options.liveTimelineLimit, 300, 100_000);
706
+ const mentionsLimit = boundedPositiveInteger(
707
+ options.liveMentionsLimit,
708
+ DEFAULT_LIVE_MENTIONS_LIMIT,
709
+ 100,
710
+ );
711
+ const threadLimit = boundedPositiveInteger(
712
+ options.liveThreadLimit,
713
+ DEFAULT_LIVE_THREAD_LIMIT,
714
+ 100,
715
+ );
716
+ const timelineMaxPages =
717
+ options.liveTimelineMaxPages === undefined
718
+ ? DEFAULT_LIVE_TIMELINE_MAX_PAGES
719
+ : boundedPositiveInteger(options.liveTimelineMaxPages, 3, 1_000);
720
+ const mentionsMaxPages =
721
+ options.liveMentionsMaxPages === undefined
722
+ ? DEFAULT_LIVE_MENTIONS_MAX_PAGES
723
+ : boundedPositiveInteger(options.liveMentionsMaxPages, 3, 1_000);
724
+
725
+ return Effect.gen(function* () {
726
+ if (includeTimeline) {
727
+ yield* Effect.sync(() =>
728
+ emitDigestStatus(
729
+ handlers,
730
+ "Fetching home timeline from X",
731
+ "Walking the selected time window with xurl.",
732
+ ),
733
+ );
734
+ const result = yield* syncHomeTimelineEffect({
735
+ account: options.account,
736
+ mode,
737
+ limit: timelineLimit,
738
+ maxPages: timelineMaxPages,
739
+ startTime: liveStartTime,
740
+ following: true,
741
+ refresh: Boolean(options.refresh),
742
+ cacheTtlMs: 2 * 60_000,
743
+ timeoutMs: 30_000,
744
+ onProgress: (progress) =>
745
+ emitDigestStatus(
746
+ handlers,
747
+ formatFetchedStatus({
748
+ fetched: progress.fetched,
749
+ total: progress.total ?? contextTweetBudget,
750
+ noun: "home tweets",
751
+ }),
752
+ formatPageDetail({
753
+ source: progress.source,
754
+ page: progress.page,
755
+ maxPages: progress.maxPages,
756
+ done: progress.done,
757
+ }),
758
+ ),
759
+ }).pipe(
760
+ Effect.match({
761
+ onFailure: () => null,
762
+ onSuccess: (value) => value,
763
+ }),
764
+ );
765
+ yield* Effect.sync(() =>
766
+ emitDigestStatus(
767
+ handlers,
768
+ result
769
+ ? `Fetched ${String(result.count)} home tweets from ${result.source}`
770
+ : "Home timeline fetch failed; using local data",
771
+ ),
772
+ );
773
+ }
774
+ if (includeMentions) {
775
+ yield* Effect.sync(() =>
776
+ emitDigestStatus(
777
+ handlers,
778
+ "Fetching mentions from X",
779
+ "Reading replies and mentions for the selected window.",
780
+ ),
781
+ );
782
+ const result = yield* syncMentionsEffect({
783
+ account: options.account,
784
+ mode: "xurl",
785
+ limit: mentionsLimit,
786
+ maxPages: mentionsMaxPages,
787
+ startTime: liveStartTime,
788
+ refresh: Boolean(options.refresh),
789
+ cacheTtlMs: 2 * 60_000,
790
+ onProgress: (progress) =>
791
+ emitDigestStatus(
792
+ handlers,
793
+ formatFetchedStatus({
794
+ fetched: progress.fetched,
795
+ total: progress.total ?? contextTweetBudget,
796
+ noun: "mentions",
797
+ }),
798
+ formatPageDetail({
799
+ source: progress.source,
800
+ page: progress.page,
801
+ maxPages: progress.maxPages,
802
+ done: progress.done,
803
+ }),
804
+ ),
805
+ }).pipe(
806
+ Effect.match({
807
+ onFailure: () => null,
808
+ onSuccess: (value) => value,
809
+ }),
810
+ );
811
+ yield* Effect.sync(() =>
812
+ emitDigestStatus(
813
+ handlers,
814
+ result
815
+ ? `Fetched ${String(result.count)} mentions from ${result.source}`
816
+ : "Mention fetch failed; using local data",
817
+ ),
818
+ );
819
+ }
820
+ if (includeThreads) {
821
+ yield* Effect.sync(() =>
822
+ emitDigestStatus(
823
+ handlers,
824
+ "Fetching mention conversations",
825
+ "Pulling parent tweets so the AI sees what replies refer to.",
826
+ ),
827
+ );
828
+ const result = yield* syncMentionThreadsEffect({
829
+ account: options.account,
830
+ mode: "xurl",
831
+ limit: threadLimit,
832
+ tweetIds: phase.threadTweetIds,
833
+ delayMs: 100,
834
+ timeoutMs: DEFAULT_LIVE_THREAD_TIMEOUT_MS,
835
+ maxPages: 2,
836
+ onProgress: (progress) =>
837
+ emitDigestStatus(
838
+ handlers,
839
+ `Fetched conversations for ${String(progress.processed)}/${String(progress.total)} mentions`,
840
+ `${String(progress.fetched)} tweets · ${progress.source}${
841
+ progress.done ? " · done" : ""
842
+ }`,
843
+ ),
844
+ }).pipe(
845
+ Effect.match({
846
+ onFailure: () => null,
847
+ onSuccess: (value) => value,
848
+ }),
849
+ );
850
+ yield* Effect.sync(() =>
851
+ emitDigestStatus(
852
+ handlers,
853
+ result
854
+ ? `Fetched ${String(result.uniqueTweets)} conversation tweets`
855
+ : "Conversation fetch failed; using available context",
856
+ ),
857
+ );
858
+ }
859
+ yield* Effect.sync(() =>
860
+ emitDigestStatus(handlers, "Preparing local AI context"),
861
+ );
862
+ yield* maybeAutoSyncBackupEffect().pipe(Effect.catchAll(() => Effect.void));
863
+ }).pipe(Effect.asVoid);
864
+ }
865
+
540
866
  function digestCacheKey(
541
867
  context: PeriodDigestContext,
542
868
  options: PeriodDigestOptions,
543
869
  ) {
544
- return [
870
+ const parts = [
545
871
  "period-digest:v2",
546
872
  modelFromOptions(options),
547
873
  reasoningEffortFromOptions(options),
548
874
  serviceTierFromOptions(options),
549
875
  context.hash,
550
- ].join(":");
876
+ ];
877
+ const lang = languageFromOptions(options);
878
+ if (lang) parts.push(`lang:${lang}`);
879
+ return parts.join(":");
551
880
  }
552
881
 
553
- function buildPrompt(context: PeriodDigestContext) {
882
+ function buildPrompt(
883
+ context: PeriodDigestContext,
884
+ options?: { language?: string },
885
+ ) {
886
+ const language = normalizeDigestLanguage(options?.language);
554
887
  const promptTweets = context.tweets.map((tweet) => ({
555
888
  id: tweet.id,
556
889
  url: tweet.url,
@@ -565,12 +898,69 @@ function buildPrompt(context: PeriodDigestContext) {
565
898
  liked: tweet.liked,
566
899
  bookmarked: tweet.bookmarked,
567
900
  needsReply: tweet.needsReply,
901
+ replyToId: tweet.replyToId,
902
+ replyToTweet: tweet.replyToTweet,
568
903
  }));
904
+ const fitDataset = () => {
905
+ let tweetCount = promptTweets.length;
906
+ let dmCount = context.dms.length;
907
+ let linkCount = context.links.length;
908
+ const datasetFor = (tweets: number, dms: number, links: number) => ({
909
+ tweets: promptTweets.slice(0, tweets),
910
+ dms: context.dms.slice(0, dms),
911
+ links: context.links.slice(0, links),
912
+ });
913
+ const lengthFor = (tweets: number, dms: number, links: number) =>
914
+ JSON.stringify(datasetFor(tweets, dms, links)).length;
915
+ const fitCount = (max: number, fits: (count: number) => boolean) => {
916
+ let low = 0;
917
+ let high = max;
918
+ let best = 0;
919
+ while (low <= high) {
920
+ const mid = Math.floor((low + high) / 2);
921
+ if (fits(mid)) {
922
+ best = mid;
923
+ low = mid + 1;
924
+ } else {
925
+ high = mid - 1;
926
+ }
927
+ }
928
+ return best;
929
+ };
930
+ if (lengthFor(tweetCount, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS) {
931
+ return {
932
+ dataset: datasetFor(tweetCount, dmCount, linkCount),
933
+ tweetCount,
934
+ };
935
+ }
936
+ dmCount = fitCount(
937
+ dmCount,
938
+ (count) =>
939
+ lengthFor(tweetCount, count, linkCount) <= MAX_PROMPT_DATA_CHARS,
940
+ );
941
+ if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
942
+ linkCount = fitCount(
943
+ linkCount,
944
+ (count) =>
945
+ lengthFor(tweetCount, dmCount, count) <= MAX_PROMPT_DATA_CHARS,
946
+ );
947
+ }
948
+ if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
949
+ tweetCount = fitCount(
950
+ tweetCount,
951
+ (count) =>
952
+ lengthFor(count, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS,
953
+ );
954
+ }
955
+ return { dataset: datasetFor(tweetCount, dmCount, linkCount), tweetCount };
956
+ };
957
+ const { dataset, tweetCount } = fitDataset();
569
958
 
570
959
  return `Window: ${context.window.label}
571
960
  Since: ${context.window.since}
572
961
  Until: ${context.window.until}
573
962
  Sources: ${JSON.stringify(context.counts)}
963
+ Prompt tweets: ${String(tweetCount)} of ${String(context.tweets.length)} selected context tweets
574
964
 
575
965
  Write a high-signal "what happened" report from this local Twitter/X dataset.
576
966
 
@@ -578,7 +968,8 @@ Requirements:
578
968
  - Stream one readable Markdown report first. The UI will show this text directly; do not rely on separate cards or structured summaries.
579
969
  - Target 700-1100 words when there is enough data.
580
970
  - Start with a 2-3 sentence lead that immediately says what people are talking about.
581
- - Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies.
971
+ - Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies. Translate these section titles when a report language is requested.
972
+ - When a tweet has replyToTweet, use that parent context to understand what the author was replying to and whether Peter already joined the conversation.
582
973
  - Use bullets under each section. Each bullet should be specific and explain why it matters.
583
974
  - For tweets: cite every claim with inline tweet ids at the end of the relevant sentence or bullet, e.g. (tweet_123, tweet_456). These citations become hoverable source links.
584
975
  - For links: emit normal Markdown links with no space between the label and URL, e.g. [title](https://example.com), then cite the sharing tweet ids in the same bullet.
@@ -591,28 +982,31 @@ Requirements:
591
982
  - Keep actionItems empty unless you wrote a "Worth replying to" section.
592
983
  - Put every tweet id cited in the Markdown into sourceTweetIds.
593
984
  - JSON shape: { "title": string, "summary": string, "keyTopics": [{ "title": string, "summary": string, "tweetIds": string[], "handles": string[] }], "notableLinks": [{ "title": string, "url": string, "why": string, "sourceTweetIds": string[] }], "people": [{ "handle": string, "name"?: string, "why": string }], "actionItems": [{ "kind": "reply"|"follow_up"|"read"|"sync", "label": string, "tweetId"?: string, "dmConversationId"?: string }], "sourceTweetIds": string[] }
985
+ ${language ? `- Write all human-readable prose, including section titles and JSON prose fields, in ${language}.\n- Preserve handles, URLs, tweet ids, and JSON property names exactly.` : ""}
594
986
 
595
987
  Dataset:
596
- ${JSON.stringify(
597
- {
598
- tweets: promptTweets,
599
- dms: context.dms,
600
- links: context.links,
601
- },
602
- null,
603
- 2,
604
- )}`;
988
+ ${JSON.stringify(dataset)}`;
605
989
  }
606
990
 
607
991
  function fallbackDigest(
608
992
  context: PeriodDigestContext,
609
993
  markdown: string,
994
+ language?: string,
610
995
  ): PeriodDigest {
611
- const title = `${context.window.label} digest`;
996
+ const normalized = markdown.replaceAll(/\s+/g, " ").trim();
997
+ const heading = markdown
998
+ .split("\n")
999
+ .map((line) => line.match(/^#{1,6}\s+(.+)$/)?.[1]?.trim())
1000
+ .find(Boolean);
1001
+ const neutralFallback = language ? `[${language}]` : undefined;
612
1002
  return {
613
- title,
1003
+ title:
1004
+ heading?.slice(0, 160) ??
1005
+ neutralFallback ??
1006
+ `${context.window.label} digest`,
614
1007
  summary:
615
- markdown.replaceAll(/\s+/g, " ").trim().slice(0, 280) ||
1008
+ normalized.slice(0, 280) ||
1009
+ neutralFallback ||
616
1010
  "No model summary was returned.",
617
1011
  keyTopics: [],
618
1012
  notableLinks: [],
@@ -625,6 +1019,7 @@ function fallbackDigest(
625
1019
  function parseDigestFromHybridText(
626
1020
  context: PeriodDigestContext,
627
1021
  rawText: string,
1022
+ language?: string,
628
1023
  ): { digest: PeriodDigest; markdown: string } {
629
1024
  const [markdownPart, jsonPart] = rawText.split(DELIMITER_PATTERN);
630
1025
  const markdown = (markdownPart ?? rawText).trim();
@@ -639,10 +1034,13 @@ function parseDigestFromHybridText(
639
1034
  digest: PeriodDigestSchema.parse(JSON.parse(candidate)),
640
1035
  };
641
1036
  } catch {
642
- return { markdown, digest: fallbackDigest(context, markdown) };
1037
+ return {
1038
+ markdown,
1039
+ digest: fallbackDigest(context, markdown, language),
1040
+ };
643
1041
  }
644
1042
  }
645
- return { markdown, digest: fallbackDigest(context, markdown) };
1043
+ return { markdown, digest: fallbackDigest(context, markdown, language) };
646
1044
  }
647
1045
 
648
1046
  function emitVisibleDelta(
@@ -786,7 +1184,9 @@ function createOpenAIRequestBody(
786
1184
  },
787
1185
  {
788
1186
  role: "user",
789
- content: buildPrompt(context),
1187
+ content: buildPrompt(context, {
1188
+ language: languageFromOptions(options),
1189
+ }),
790
1190
  },
791
1191
  ],
792
1192
  };
@@ -829,7 +1229,11 @@ function readOpenAIStreamEffect(
829
1229
  }
830
1230
 
831
1231
  const parsed = yield* tryDigestSync(() =>
832
- parseDigestFromHybridText(context, state.rawText),
1232
+ parseDigestFromHybridText(
1233
+ context,
1234
+ state.rawText,
1235
+ languageFromOptions(options),
1236
+ ),
833
1237
  );
834
1238
  const cacheKey = digestCacheKey(context, options);
835
1239
  const updatedAt = yield* tryDigestSync(() =>
@@ -870,11 +1274,20 @@ export function streamPeriodDigestEffect(
870
1274
  handlers: PeriodDigestStreamHandlers = {},
871
1275
  ): Effect.Effect<PeriodDigestRunResult, Error> {
872
1276
  return Effect.gen(function* () {
873
- const context = yield* tryDigestSync(() =>
874
- collectPeriodDigestContext(options),
1277
+ const resolvedOptions = {
1278
+ ...options,
1279
+ language: yield* tryDigestSync(() => languageFromOptions(options)),
1280
+ };
1281
+ yield* refreshPeriodDigestInputsEffect(
1282
+ resolvedOptions,
1283
+ { threads: false },
1284
+ handlers,
1285
+ ).pipe(Effect.catchAll(() => Effect.void));
1286
+ let context = yield* tryDigestSync(() =>
1287
+ collectPeriodDigestContext(resolvedOptions),
875
1288
  );
876
- const cacheKey = digestCacheKey(context, options);
877
- const cached = options.refresh
1289
+ let cacheKey = digestCacheKey(context, resolvedOptions);
1290
+ const cached = resolvedOptions.refresh
878
1291
  ? null
879
1292
  : yield* tryDigestSync(() =>
880
1293
  readSyncCache<{
@@ -904,21 +1317,39 @@ export function streamPeriodDigestEffect(
904
1317
  return result;
905
1318
  }
906
1319
 
1320
+ yield* refreshPeriodDigestInputsEffect(
1321
+ resolvedOptions,
1322
+ {
1323
+ timeline: false,
1324
+ mentions: false,
1325
+ threads: true,
1326
+ threadTweetIds: context.tweets
1327
+ .filter((tweet) => tweet.source === "mentions")
1328
+ .map((tweet) => tweet.id),
1329
+ },
1330
+ handlers,
1331
+ ).pipe(Effect.catchAll(() => Effect.void));
1332
+ context = yield* tryDigestSync(() =>
1333
+ collectPeriodDigestContext(resolvedOptions),
1334
+ );
1335
+ cacheKey = digestCacheKey(context, resolvedOptions);
1336
+
907
1337
  const apiKey = process.env.OPENAI_API_KEY;
908
1338
  if (!apiKey) {
909
1339
  return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
910
1340
  }
911
1341
 
912
1342
  handlers.onEvent?.({ type: "start", context, cached: false });
1343
+ emitDigestStatus(handlers, "Streaming AI summary");
913
1344
  const response = yield* tryDigestPromise(() =>
914
1345
  fetch("https://api.openai.com/v1/responses", {
915
1346
  method: "POST",
916
- signal: options.signal,
1347
+ signal: resolvedOptions.signal,
917
1348
  headers: {
918
1349
  authorization: `Bearer ${apiKey}`,
919
1350
  "content-type": "application/json",
920
1351
  },
921
- body: JSON.stringify(createOpenAIRequestBody(context, options)),
1352
+ body: JSON.stringify(createOpenAIRequestBody(context, resolvedOptions)),
922
1353
  }),
923
1354
  );
924
1355
  if (!response.ok) {
@@ -932,7 +1363,12 @@ export function streamPeriodDigestEffect(
932
1363
  ),
933
1364
  );
934
1365
  }
935
- return yield* readOpenAIStreamEffect(response, context, options, handlers);
1366
+ return yield* readOpenAIStreamEffect(
1367
+ response,
1368
+ context,
1369
+ resolvedOptions,
1370
+ handlers,
1371
+ );
936
1372
  });
937
1373
  }
938
1374
 
@@ -946,6 +1382,9 @@ export function streamPeriodDigest(
946
1382
  export const __test__ = {
947
1383
  PeriodDigestSchema,
948
1384
  buildPrompt,
1385
+ digestCacheKey,
1386
+ languageFromOptions,
1387
+ normalizeDigestLanguage,
949
1388
  readOpenAIStreamEffect,
950
1389
  parseDigestFromHybridText,
951
1390
  processSseChunk,