birdclaw 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +32 -2
- package/package.json +29 -30
- package/scripts/build-docs-site.mjs +39 -12
- package/src/cli.ts +457 -26
- package/src/components/AppNav.tsx +10 -0
- package/src/components/MarkdownViewer.tsx +438 -72
- package/src/components/ProfileAnalysisStream.tsx +428 -0
- package/src/components/ProfilePreview.tsx +120 -9
- package/src/components/SavedTimelineView.tsx +30 -8
- package/src/components/SyncNowButton.tsx +5 -2
- package/src/components/TimelineCard.tsx +20 -4
- package/src/components/TimelineRouteFrame.tsx +16 -0
- package/src/components/TweetRichText.tsx +36 -12
- package/src/components/useTimelineRouteData.ts +74 -6
- package/src/lib/account-sync-job.ts +15 -3
- package/src/lib/archive-finder.ts +1 -1
- package/src/lib/archive-import.ts +245 -7
- package/src/lib/authored-live.ts +1 -0
- package/src/lib/avatar-cache.ts +50 -0
- package/src/lib/backup.ts +4 -3
- package/src/lib/bird.ts +33 -0
- package/src/lib/config.ts +35 -2
- package/src/lib/data-sources.ts +219 -0
- package/src/lib/db.ts +62 -1
- package/src/lib/geocoding.ts +296 -0
- package/src/lib/link-insights.ts +2 -0
- package/src/lib/location.ts +137 -0
- package/src/lib/mention-threads-live.ts +94 -1
- package/src/lib/mentions-live.ts +187 -40
- package/src/lib/network-map.ts +382 -0
- package/src/lib/period-digest.ts +468 -29
- package/src/lib/profile-analysis.ts +1272 -0
- package/src/lib/profile-bio-entities.ts +1 -1
- package/src/lib/queries.ts +14 -4
- package/src/lib/search-discussion.ts +1016 -0
- package/src/lib/timeline-live.ts +272 -19
- package/src/lib/tweet-account-edges.ts +2 -0
- package/src/lib/tweet-render.ts +141 -1
- package/src/lib/tweet-search-live.ts +565 -0
- package/src/lib/types.ts +37 -2
- package/src/lib/ui.ts +1 -1
- package/src/lib/web-sync.ts +7 -2
- package/src/lib/xurl-rate-limits.ts +267 -0
- package/src/lib/xurl.ts +551 -41
- package/src/routeTree.gen.ts +231 -0
- package/src/routes/__root.tsx +5 -6
- package/src/routes/api/data-sources.tsx +24 -0
- package/src/routes/api/network-map.tsx +55 -0
- package/src/routes/api/period-digest.tsx +29 -3
- package/src/routes/api/profile-analysis.tsx +152 -0
- package/src/routes/api/query.tsx +1 -0
- package/src/routes/api/search-discussion.tsx +169 -0
- package/src/routes/api/xurl-rate-limits.tsx +24 -0
- package/src/routes/data-sources.tsx +257 -0
- package/src/routes/discuss.tsx +419 -0
- package/src/routes/network-map.tsx +1035 -0
- package/src/routes/profile-analyze.tsx +112 -0
- package/src/routes/profiles.$handle.tsx +228 -0
- package/src/routes/rate-limits.tsx +309 -0
- package/src/routes/today.tsx +22 -8
- package/src/styles.css +22 -0
package/src/lib/period-digest.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { Effect } from "effect";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
+
import { maybeAutoSyncBackupEffect } from "./backup";
|
|
4
5
|
import { runEffectPromise, tryPromise } from "./effect-runtime";
|
|
5
6
|
import { getLinkInsights } from "./link-insights";
|
|
7
|
+
import { syncMentionThreadsEffect } from "./mention-threads-live";
|
|
8
|
+
import { syncMentionsEffect } from "./mentions-live";
|
|
6
9
|
import { listDmConversations, listTimelineItems } from "./queries";
|
|
7
10
|
import { readSyncCache, writeSyncCache } from "./sync-cache";
|
|
8
|
-
import type
|
|
11
|
+
import { syncHomeTimelineEffect, type HomeTimelineMode } from "./timeline-live";
|
|
12
|
+
import type { ProfileRecord, TweetEntities } from "./types";
|
|
9
13
|
|
|
10
14
|
export type PeriodDigestPreset = "today" | "yesterday" | "24h" | "week";
|
|
11
15
|
export type PeriodDigestSourceKind =
|
|
@@ -24,11 +28,19 @@ export interface PeriodDigestOptions {
|
|
|
24
28
|
includeDms?: boolean;
|
|
25
29
|
refresh?: boolean;
|
|
26
30
|
model?: string;
|
|
31
|
+
language?: string;
|
|
27
32
|
reasoningEffort?: "minimal" | "low" | "medium" | "high";
|
|
28
33
|
serviceTier?: "default" | "flex" | "priority";
|
|
29
34
|
signal?: AbortSignal;
|
|
30
35
|
maxTweets?: number;
|
|
31
36
|
maxLinks?: number;
|
|
37
|
+
liveSync?: boolean;
|
|
38
|
+
liveSyncMode?: HomeTimelineMode;
|
|
39
|
+
liveTimelineLimit?: number;
|
|
40
|
+
liveTimelineMaxPages?: number;
|
|
41
|
+
liveMentionsLimit?: number;
|
|
42
|
+
liveMentionsMaxPages?: number;
|
|
43
|
+
liveThreadLimit?: number;
|
|
32
44
|
}
|
|
33
45
|
|
|
34
46
|
export interface PeriodDigestWindow {
|
|
@@ -54,6 +66,7 @@ export interface PeriodDigestStreamHandlers {
|
|
|
54
66
|
}
|
|
55
67
|
|
|
56
68
|
export type PeriodDigestStreamEvent =
|
|
69
|
+
| { type: "status"; label: string; detail?: string }
|
|
57
70
|
| { type: "start"; context: PeriodDigestContext; cached: boolean }
|
|
58
71
|
| { type: "delta"; delta: string }
|
|
59
72
|
| { type: "done"; result: PeriodDigestRunResult }
|
|
@@ -96,6 +109,32 @@ const PeriodDigestSchema = z.object({
|
|
|
96
109
|
sourceTweetIds: z.array(z.string()).default([]),
|
|
97
110
|
});
|
|
98
111
|
|
|
112
|
+
const MAX_DIGEST_LANGUAGE_LENGTH = 64;
|
|
113
|
+
|
|
114
|
+
export function normalizeDigestLanguage(
|
|
115
|
+
value: string | undefined,
|
|
116
|
+
): string | undefined {
|
|
117
|
+
const trimmed = value?.trim();
|
|
118
|
+
if (!trimmed) return undefined;
|
|
119
|
+
if (
|
|
120
|
+
trimmed.length > MAX_DIGEST_LANGUAGE_LENGTH ||
|
|
121
|
+
!/^[a-z0-9]+(?:-[a-z0-9]+)*$/i.test(trimmed)
|
|
122
|
+
) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
"Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
try {
|
|
128
|
+
const [canonical] = Intl.getCanonicalLocales(trimmed);
|
|
129
|
+
if (!canonical) throw new Error("missing canonical locale");
|
|
130
|
+
return canonical;
|
|
131
|
+
} catch {
|
|
132
|
+
throw new Error(
|
|
133
|
+
"Digest language must be a valid Unicode locale identifier such as en, zh-CN, or pt-BR",
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
99
138
|
export type PeriodDigest = z.infer<typeof PeriodDigestSchema>;
|
|
100
139
|
|
|
101
140
|
interface CompactTweet {
|
|
@@ -107,10 +146,20 @@ interface CompactTweet {
|
|
|
107
146
|
authorProfile: ProfileRecord;
|
|
108
147
|
createdAt: string;
|
|
109
148
|
text: string;
|
|
149
|
+
entities?: TweetEntities;
|
|
110
150
|
likeCount: number;
|
|
111
151
|
liked: boolean;
|
|
112
152
|
bookmarked: boolean;
|
|
113
153
|
needsReply: boolean;
|
|
154
|
+
replyToId?: string | null;
|
|
155
|
+
replyToTweet?: {
|
|
156
|
+
id: string;
|
|
157
|
+
url: string;
|
|
158
|
+
author: string;
|
|
159
|
+
name: string;
|
|
160
|
+
createdAt: string;
|
|
161
|
+
text: string;
|
|
162
|
+
} | null;
|
|
114
163
|
}
|
|
115
164
|
|
|
116
165
|
interface CompactDm {
|
|
@@ -166,8 +215,14 @@ interface OpenAIStreamState {
|
|
|
166
215
|
const DEFAULT_MODEL = "gpt-5.5";
|
|
167
216
|
const DEFAULT_REASONING_EFFORT = "medium";
|
|
168
217
|
const DEFAULT_SERVICE_TIER = "priority";
|
|
169
|
-
const DEFAULT_MAX_TWEETS =
|
|
218
|
+
const DEFAULT_MAX_TWEETS = 2_500;
|
|
170
219
|
const DEFAULT_MAX_LINKS = 12;
|
|
220
|
+
const DEFAULT_LIVE_TIMELINE_MAX_PAGES = undefined;
|
|
221
|
+
const DEFAULT_LIVE_MENTIONS_LIMIT = 100;
|
|
222
|
+
const DEFAULT_LIVE_MENTIONS_MAX_PAGES = undefined;
|
|
223
|
+
const DEFAULT_LIVE_THREAD_LIMIT = 12;
|
|
224
|
+
const DEFAULT_LIVE_THREAD_TIMEOUT_MS = 5_000;
|
|
225
|
+
const MAX_PROMPT_DATA_CHARS = 1_200_000;
|
|
171
226
|
const DELIMITER_PATTERN = /\n---\s*\n/;
|
|
172
227
|
const VISIBLE_DELIMITER_HOLD = 8;
|
|
173
228
|
|
|
@@ -204,6 +259,12 @@ function parseDate(value: string | undefined) {
|
|
|
204
259
|
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
205
260
|
}
|
|
206
261
|
|
|
262
|
+
function floorIsoToHour(value: string) {
|
|
263
|
+
const date = new Date(value);
|
|
264
|
+
date.setUTCMinutes(0, 0, 0);
|
|
265
|
+
return date.toISOString();
|
|
266
|
+
}
|
|
267
|
+
|
|
207
268
|
function normalizePeriod(value: string | undefined): PeriodDigestPreset {
|
|
208
269
|
const normalized = value?.trim().toLowerCase();
|
|
209
270
|
if (normalized === "yesterday") return "yesterday";
|
|
@@ -273,6 +334,16 @@ function compactTweet(
|
|
|
273
334
|
source: PeriodDigestSourceKind,
|
|
274
335
|
item: ReturnType<typeof listTimelineItems>[number],
|
|
275
336
|
): CompactTweet {
|
|
337
|
+
const replyToTweet = item.replyToTweet
|
|
338
|
+
? {
|
|
339
|
+
id: item.replyToTweet.id,
|
|
340
|
+
url: tweetUrl(item.replyToTweet.author.handle, item.replyToTweet.id),
|
|
341
|
+
author: item.replyToTweet.author.handle,
|
|
342
|
+
name: item.replyToTweet.author.displayName,
|
|
343
|
+
createdAt: item.replyToTweet.createdAt,
|
|
344
|
+
text: item.replyToTweet.text,
|
|
345
|
+
}
|
|
346
|
+
: null;
|
|
276
347
|
return {
|
|
277
348
|
id: item.id,
|
|
278
349
|
url: tweetUrl(item.author.handle, item.id),
|
|
@@ -282,10 +353,13 @@ function compactTweet(
|
|
|
282
353
|
authorProfile: item.author,
|
|
283
354
|
createdAt: item.createdAt,
|
|
284
355
|
text: item.text,
|
|
356
|
+
entities: item.entities,
|
|
285
357
|
likeCount: item.likeCount,
|
|
286
358
|
liked: item.liked,
|
|
287
359
|
bookmarked: item.bookmarked,
|
|
288
360
|
needsReply: !item.isReplied,
|
|
361
|
+
replyToId: item.replyToId ?? null,
|
|
362
|
+
replyToTweet,
|
|
289
363
|
};
|
|
290
364
|
}
|
|
291
365
|
|
|
@@ -417,6 +491,9 @@ function contextHash(context: Omit<PeriodDigestContext, "hash">) {
|
|
|
417
491
|
tweet.liked,
|
|
418
492
|
tweet.bookmarked,
|
|
419
493
|
tweet.needsReply,
|
|
494
|
+
tweet.replyToId,
|
|
495
|
+
tweet.replyToTweet?.id,
|
|
496
|
+
tweet.replyToTweet?.text,
|
|
420
497
|
]),
|
|
421
498
|
dms: context.dms.map((dm) => [
|
|
422
499
|
dm.id,
|
|
@@ -513,6 +590,12 @@ export function collectPeriodDigestContext(
|
|
|
513
590
|
};
|
|
514
591
|
}
|
|
515
592
|
|
|
593
|
+
function languageFromOptions(options: PeriodDigestOptions) {
|
|
594
|
+
return normalizeDigestLanguage(
|
|
595
|
+
options.language ?? process.env.BIRDCLAW_DIGEST_LANGUAGE,
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
|
|
516
599
|
function modelFromOptions(options: PeriodDigestOptions) {
|
|
517
600
|
return options.model ?? process.env.BIRDCLAW_AI_MODEL ?? DEFAULT_MODEL;
|
|
518
601
|
}
|
|
@@ -537,20 +620,270 @@ function serviceTierFromOptions(options: PeriodDigestOptions) {
|
|
|
537
620
|
);
|
|
538
621
|
}
|
|
539
622
|
|
|
623
|
+
function boundedPositiveInteger(
|
|
624
|
+
value: number | undefined,
|
|
625
|
+
fallback: number,
|
|
626
|
+
max: number,
|
|
627
|
+
) {
|
|
628
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 1) {
|
|
629
|
+
return fallback;
|
|
630
|
+
}
|
|
631
|
+
return Math.min(max, Math.floor(value));
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
function emitDigestStatus(
|
|
635
|
+
handlers: PeriodDigestStreamHandlers,
|
|
636
|
+
label: string,
|
|
637
|
+
detail?: string,
|
|
638
|
+
) {
|
|
639
|
+
handlers.onEvent?.({
|
|
640
|
+
type: "status",
|
|
641
|
+
label,
|
|
642
|
+
...(detail ? { detail } : {}),
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function formatFetchedStatus({
|
|
647
|
+
fetched,
|
|
648
|
+
total,
|
|
649
|
+
noun,
|
|
650
|
+
}: {
|
|
651
|
+
fetched: number;
|
|
652
|
+
total: number;
|
|
653
|
+
noun: string;
|
|
654
|
+
}) {
|
|
655
|
+
const count = `${String(Math.min(fetched, total))}/${String(total)}`;
|
|
656
|
+
return `Fetched ${count} ${noun}`;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
function formatPageDetail({
|
|
660
|
+
source,
|
|
661
|
+
page,
|
|
662
|
+
maxPages,
|
|
663
|
+
done,
|
|
664
|
+
}: {
|
|
665
|
+
source: string;
|
|
666
|
+
page?: number;
|
|
667
|
+
maxPages?: number;
|
|
668
|
+
done: boolean;
|
|
669
|
+
}) {
|
|
670
|
+
const pageText =
|
|
671
|
+
page === undefined
|
|
672
|
+
? undefined
|
|
673
|
+
: `page ${String(page)}${maxPages === undefined ? "" : `/${String(maxPages)}`}`;
|
|
674
|
+
return [source, pageText, done ? "done" : undefined]
|
|
675
|
+
.filter(Boolean)
|
|
676
|
+
.join(" · ");
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
function refreshPeriodDigestInputsEffect(
|
|
680
|
+
options: PeriodDigestOptions,
|
|
681
|
+
phase: {
|
|
682
|
+
timeline?: boolean;
|
|
683
|
+
mentions?: boolean;
|
|
684
|
+
threads?: boolean;
|
|
685
|
+
threadTweetIds?: string[];
|
|
686
|
+
} = {},
|
|
687
|
+
handlers: PeriodDigestStreamHandlers = {},
|
|
688
|
+
): Effect.Effect<void, unknown> {
|
|
689
|
+
if (!options.liveSync) {
|
|
690
|
+
return Effect.void;
|
|
691
|
+
}
|
|
692
|
+
const includeTimeline = phase.timeline ?? true;
|
|
693
|
+
const includeMentions = phase.mentions ?? true;
|
|
694
|
+
const includeThreads = phase.threads ?? true;
|
|
695
|
+
const window = resolvePeriodDigestWindow(options);
|
|
696
|
+
const liveStartTime = floorIsoToHour(window.since);
|
|
697
|
+
const mode = options.liveSyncMode ?? "xurl";
|
|
698
|
+
const contextTweetBudget = Math.max(
|
|
699
|
+
20,
|
|
700
|
+
Math.trunc(options.maxTweets ?? DEFAULT_MAX_TWEETS),
|
|
701
|
+
);
|
|
702
|
+
const timelineLimit =
|
|
703
|
+
options.liveTimelineLimit === undefined
|
|
704
|
+
? undefined
|
|
705
|
+
: boundedPositiveInteger(options.liveTimelineLimit, 300, 100_000);
|
|
706
|
+
const mentionsLimit = boundedPositiveInteger(
|
|
707
|
+
options.liveMentionsLimit,
|
|
708
|
+
DEFAULT_LIVE_MENTIONS_LIMIT,
|
|
709
|
+
100,
|
|
710
|
+
);
|
|
711
|
+
const threadLimit = boundedPositiveInteger(
|
|
712
|
+
options.liveThreadLimit,
|
|
713
|
+
DEFAULT_LIVE_THREAD_LIMIT,
|
|
714
|
+
100,
|
|
715
|
+
);
|
|
716
|
+
const timelineMaxPages =
|
|
717
|
+
options.liveTimelineMaxPages === undefined
|
|
718
|
+
? DEFAULT_LIVE_TIMELINE_MAX_PAGES
|
|
719
|
+
: boundedPositiveInteger(options.liveTimelineMaxPages, 3, 1_000);
|
|
720
|
+
const mentionsMaxPages =
|
|
721
|
+
options.liveMentionsMaxPages === undefined
|
|
722
|
+
? DEFAULT_LIVE_MENTIONS_MAX_PAGES
|
|
723
|
+
: boundedPositiveInteger(options.liveMentionsMaxPages, 3, 1_000);
|
|
724
|
+
|
|
725
|
+
return Effect.gen(function* () {
|
|
726
|
+
if (includeTimeline) {
|
|
727
|
+
yield* Effect.sync(() =>
|
|
728
|
+
emitDigestStatus(
|
|
729
|
+
handlers,
|
|
730
|
+
"Fetching home timeline from X",
|
|
731
|
+
"Walking the selected time window with xurl.",
|
|
732
|
+
),
|
|
733
|
+
);
|
|
734
|
+
const result = yield* syncHomeTimelineEffect({
|
|
735
|
+
account: options.account,
|
|
736
|
+
mode,
|
|
737
|
+
limit: timelineLimit,
|
|
738
|
+
maxPages: timelineMaxPages,
|
|
739
|
+
startTime: liveStartTime,
|
|
740
|
+
following: true,
|
|
741
|
+
refresh: Boolean(options.refresh),
|
|
742
|
+
cacheTtlMs: 2 * 60_000,
|
|
743
|
+
timeoutMs: 30_000,
|
|
744
|
+
onProgress: (progress) =>
|
|
745
|
+
emitDigestStatus(
|
|
746
|
+
handlers,
|
|
747
|
+
formatFetchedStatus({
|
|
748
|
+
fetched: progress.fetched,
|
|
749
|
+
total: progress.total ?? contextTweetBudget,
|
|
750
|
+
noun: "home tweets",
|
|
751
|
+
}),
|
|
752
|
+
formatPageDetail({
|
|
753
|
+
source: progress.source,
|
|
754
|
+
page: progress.page,
|
|
755
|
+
maxPages: progress.maxPages,
|
|
756
|
+
done: progress.done,
|
|
757
|
+
}),
|
|
758
|
+
),
|
|
759
|
+
}).pipe(
|
|
760
|
+
Effect.match({
|
|
761
|
+
onFailure: () => null,
|
|
762
|
+
onSuccess: (value) => value,
|
|
763
|
+
}),
|
|
764
|
+
);
|
|
765
|
+
yield* Effect.sync(() =>
|
|
766
|
+
emitDigestStatus(
|
|
767
|
+
handlers,
|
|
768
|
+
result
|
|
769
|
+
? `Fetched ${String(result.count)} home tweets from ${result.source}`
|
|
770
|
+
: "Home timeline fetch failed; using local data",
|
|
771
|
+
),
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
if (includeMentions) {
|
|
775
|
+
yield* Effect.sync(() =>
|
|
776
|
+
emitDigestStatus(
|
|
777
|
+
handlers,
|
|
778
|
+
"Fetching mentions from X",
|
|
779
|
+
"Reading replies and mentions for the selected window.",
|
|
780
|
+
),
|
|
781
|
+
);
|
|
782
|
+
const result = yield* syncMentionsEffect({
|
|
783
|
+
account: options.account,
|
|
784
|
+
mode: "xurl",
|
|
785
|
+
limit: mentionsLimit,
|
|
786
|
+
maxPages: mentionsMaxPages,
|
|
787
|
+
startTime: liveStartTime,
|
|
788
|
+
refresh: Boolean(options.refresh),
|
|
789
|
+
cacheTtlMs: 2 * 60_000,
|
|
790
|
+
onProgress: (progress) =>
|
|
791
|
+
emitDigestStatus(
|
|
792
|
+
handlers,
|
|
793
|
+
formatFetchedStatus({
|
|
794
|
+
fetched: progress.fetched,
|
|
795
|
+
total: progress.total ?? contextTweetBudget,
|
|
796
|
+
noun: "mentions",
|
|
797
|
+
}),
|
|
798
|
+
formatPageDetail({
|
|
799
|
+
source: progress.source,
|
|
800
|
+
page: progress.page,
|
|
801
|
+
maxPages: progress.maxPages,
|
|
802
|
+
done: progress.done,
|
|
803
|
+
}),
|
|
804
|
+
),
|
|
805
|
+
}).pipe(
|
|
806
|
+
Effect.match({
|
|
807
|
+
onFailure: () => null,
|
|
808
|
+
onSuccess: (value) => value,
|
|
809
|
+
}),
|
|
810
|
+
);
|
|
811
|
+
yield* Effect.sync(() =>
|
|
812
|
+
emitDigestStatus(
|
|
813
|
+
handlers,
|
|
814
|
+
result
|
|
815
|
+
? `Fetched ${String(result.count)} mentions from ${result.source}`
|
|
816
|
+
: "Mention fetch failed; using local data",
|
|
817
|
+
),
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
if (includeThreads) {
|
|
821
|
+
yield* Effect.sync(() =>
|
|
822
|
+
emitDigestStatus(
|
|
823
|
+
handlers,
|
|
824
|
+
"Fetching mention conversations",
|
|
825
|
+
"Pulling parent tweets so the AI sees what replies refer to.",
|
|
826
|
+
),
|
|
827
|
+
);
|
|
828
|
+
const result = yield* syncMentionThreadsEffect({
|
|
829
|
+
account: options.account,
|
|
830
|
+
mode: "xurl",
|
|
831
|
+
limit: threadLimit,
|
|
832
|
+
tweetIds: phase.threadTweetIds,
|
|
833
|
+
delayMs: 100,
|
|
834
|
+
timeoutMs: DEFAULT_LIVE_THREAD_TIMEOUT_MS,
|
|
835
|
+
maxPages: 2,
|
|
836
|
+
onProgress: (progress) =>
|
|
837
|
+
emitDigestStatus(
|
|
838
|
+
handlers,
|
|
839
|
+
`Fetched conversations for ${String(progress.processed)}/${String(progress.total)} mentions`,
|
|
840
|
+
`${String(progress.fetched)} tweets · ${progress.source}${
|
|
841
|
+
progress.done ? " · done" : ""
|
|
842
|
+
}`,
|
|
843
|
+
),
|
|
844
|
+
}).pipe(
|
|
845
|
+
Effect.match({
|
|
846
|
+
onFailure: () => null,
|
|
847
|
+
onSuccess: (value) => value,
|
|
848
|
+
}),
|
|
849
|
+
);
|
|
850
|
+
yield* Effect.sync(() =>
|
|
851
|
+
emitDigestStatus(
|
|
852
|
+
handlers,
|
|
853
|
+
result
|
|
854
|
+
? `Fetched ${String(result.uniqueTweets)} conversation tweets`
|
|
855
|
+
: "Conversation fetch failed; using available context",
|
|
856
|
+
),
|
|
857
|
+
);
|
|
858
|
+
}
|
|
859
|
+
yield* Effect.sync(() =>
|
|
860
|
+
emitDigestStatus(handlers, "Preparing local AI context"),
|
|
861
|
+
);
|
|
862
|
+
yield* maybeAutoSyncBackupEffect().pipe(Effect.catchAll(() => Effect.void));
|
|
863
|
+
}).pipe(Effect.asVoid);
|
|
864
|
+
}
|
|
865
|
+
|
|
540
866
|
function digestCacheKey(
|
|
541
867
|
context: PeriodDigestContext,
|
|
542
868
|
options: PeriodDigestOptions,
|
|
543
869
|
) {
|
|
544
|
-
|
|
870
|
+
const parts = [
|
|
545
871
|
"period-digest:v2",
|
|
546
872
|
modelFromOptions(options),
|
|
547
873
|
reasoningEffortFromOptions(options),
|
|
548
874
|
serviceTierFromOptions(options),
|
|
549
875
|
context.hash,
|
|
550
|
-
]
|
|
876
|
+
];
|
|
877
|
+
const lang = languageFromOptions(options);
|
|
878
|
+
if (lang) parts.push(`lang:${lang}`);
|
|
879
|
+
return parts.join(":");
|
|
551
880
|
}
|
|
552
881
|
|
|
553
|
-
function buildPrompt(
|
|
882
|
+
function buildPrompt(
|
|
883
|
+
context: PeriodDigestContext,
|
|
884
|
+
options?: { language?: string },
|
|
885
|
+
) {
|
|
886
|
+
const language = normalizeDigestLanguage(options?.language);
|
|
554
887
|
const promptTweets = context.tweets.map((tweet) => ({
|
|
555
888
|
id: tweet.id,
|
|
556
889
|
url: tweet.url,
|
|
@@ -565,12 +898,69 @@ function buildPrompt(context: PeriodDigestContext) {
|
|
|
565
898
|
liked: tweet.liked,
|
|
566
899
|
bookmarked: tweet.bookmarked,
|
|
567
900
|
needsReply: tweet.needsReply,
|
|
901
|
+
replyToId: tweet.replyToId,
|
|
902
|
+
replyToTweet: tweet.replyToTweet,
|
|
568
903
|
}));
|
|
904
|
+
const fitDataset = () => {
|
|
905
|
+
let tweetCount = promptTweets.length;
|
|
906
|
+
let dmCount = context.dms.length;
|
|
907
|
+
let linkCount = context.links.length;
|
|
908
|
+
const datasetFor = (tweets: number, dms: number, links: number) => ({
|
|
909
|
+
tweets: promptTweets.slice(0, tweets),
|
|
910
|
+
dms: context.dms.slice(0, dms),
|
|
911
|
+
links: context.links.slice(0, links),
|
|
912
|
+
});
|
|
913
|
+
const lengthFor = (tweets: number, dms: number, links: number) =>
|
|
914
|
+
JSON.stringify(datasetFor(tweets, dms, links)).length;
|
|
915
|
+
const fitCount = (max: number, fits: (count: number) => boolean) => {
|
|
916
|
+
let low = 0;
|
|
917
|
+
let high = max;
|
|
918
|
+
let best = 0;
|
|
919
|
+
while (low <= high) {
|
|
920
|
+
const mid = Math.floor((low + high) / 2);
|
|
921
|
+
if (fits(mid)) {
|
|
922
|
+
best = mid;
|
|
923
|
+
low = mid + 1;
|
|
924
|
+
} else {
|
|
925
|
+
high = mid - 1;
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return best;
|
|
929
|
+
};
|
|
930
|
+
if (lengthFor(tweetCount, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS) {
|
|
931
|
+
return {
|
|
932
|
+
dataset: datasetFor(tweetCount, dmCount, linkCount),
|
|
933
|
+
tweetCount,
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
dmCount = fitCount(
|
|
937
|
+
dmCount,
|
|
938
|
+
(count) =>
|
|
939
|
+
lengthFor(tweetCount, count, linkCount) <= MAX_PROMPT_DATA_CHARS,
|
|
940
|
+
);
|
|
941
|
+
if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
|
|
942
|
+
linkCount = fitCount(
|
|
943
|
+
linkCount,
|
|
944
|
+
(count) =>
|
|
945
|
+
lengthFor(tweetCount, dmCount, count) <= MAX_PROMPT_DATA_CHARS,
|
|
946
|
+
);
|
|
947
|
+
}
|
|
948
|
+
if (lengthFor(tweetCount, dmCount, linkCount) > MAX_PROMPT_DATA_CHARS) {
|
|
949
|
+
tweetCount = fitCount(
|
|
950
|
+
tweetCount,
|
|
951
|
+
(count) =>
|
|
952
|
+
lengthFor(count, dmCount, linkCount) <= MAX_PROMPT_DATA_CHARS,
|
|
953
|
+
);
|
|
954
|
+
}
|
|
955
|
+
return { dataset: datasetFor(tweetCount, dmCount, linkCount), tweetCount };
|
|
956
|
+
};
|
|
957
|
+
const { dataset, tweetCount } = fitDataset();
|
|
569
958
|
|
|
570
959
|
return `Window: ${context.window.label}
|
|
571
960
|
Since: ${context.window.since}
|
|
572
961
|
Until: ${context.window.until}
|
|
573
962
|
Sources: ${JSON.stringify(context.counts)}
|
|
963
|
+
Prompt tweets: ${String(tweetCount)} of ${String(context.tweets.length)} selected context tweets
|
|
574
964
|
|
|
575
965
|
Write a high-signal "what happened" report from this local Twitter/X dataset.
|
|
576
966
|
|
|
@@ -578,7 +968,8 @@ Requirements:
|
|
|
578
968
|
- Stream one readable Markdown report first. The UI will show this text directly; do not rely on separate cards or structured summaries.
|
|
579
969
|
- Target 700-1100 words when there is enough data.
|
|
580
970
|
- Start with a 2-3 sentence lead that immediately says what people are talking about.
|
|
581
|
-
- Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies.
|
|
971
|
+
- Use sections named "What people are talking about", "Important links shared", and "Worth opening". Add "Worth replying to" only if there are clearly high-signal replies. Translate these section titles when a report language is requested.
|
|
972
|
+
- When a tweet has replyToTweet, use that parent context to understand what the author was replying to and whether Peter already joined the conversation.
|
|
582
973
|
- Use bullets under each section. Each bullet should be specific and explain why it matters.
|
|
583
974
|
- For tweets: cite every claim with inline tweet ids at the end of the relevant sentence or bullet, e.g. (tweet_123, tweet_456). These citations become hoverable source links.
|
|
584
975
|
- For links: emit normal Markdown links with no space between the label and URL, e.g. [title](https://example.com), then cite the sharing tweet ids in the same bullet.
|
|
@@ -591,28 +982,31 @@ Requirements:
|
|
|
591
982
|
- Keep actionItems empty unless you wrote a "Worth replying to" section.
|
|
592
983
|
- Put every tweet id cited in the Markdown into sourceTweetIds.
|
|
593
984
|
- JSON shape: { "title": string, "summary": string, "keyTopics": [{ "title": string, "summary": string, "tweetIds": string[], "handles": string[] }], "notableLinks": [{ "title": string, "url": string, "why": string, "sourceTweetIds": string[] }], "people": [{ "handle": string, "name"?: string, "why": string }], "actionItems": [{ "kind": "reply"|"follow_up"|"read"|"sync", "label": string, "tweetId"?: string, "dmConversationId"?: string }], "sourceTweetIds": string[] }
|
|
985
|
+
${language ? `- Write all human-readable prose, including section titles and JSON prose fields, in ${language}.\n- Preserve handles, URLs, tweet ids, and JSON property names exactly.` : ""}
|
|
594
986
|
|
|
595
987
|
Dataset:
|
|
596
|
-
${JSON.stringify(
|
|
597
|
-
{
|
|
598
|
-
tweets: promptTweets,
|
|
599
|
-
dms: context.dms,
|
|
600
|
-
links: context.links,
|
|
601
|
-
},
|
|
602
|
-
null,
|
|
603
|
-
2,
|
|
604
|
-
)}`;
|
|
988
|
+
${JSON.stringify(dataset)}`;
|
|
605
989
|
}
|
|
606
990
|
|
|
607
991
|
function fallbackDigest(
|
|
608
992
|
context: PeriodDigestContext,
|
|
609
993
|
markdown: string,
|
|
994
|
+
language?: string,
|
|
610
995
|
): PeriodDigest {
|
|
611
|
-
const
|
|
996
|
+
const normalized = markdown.replaceAll(/\s+/g, " ").trim();
|
|
997
|
+
const heading = markdown
|
|
998
|
+
.split("\n")
|
|
999
|
+
.map((line) => line.match(/^#{1,6}\s+(.+)$/)?.[1]?.trim())
|
|
1000
|
+
.find(Boolean);
|
|
1001
|
+
const neutralFallback = language ? `[${language}]` : undefined;
|
|
612
1002
|
return {
|
|
613
|
-
title
|
|
1003
|
+
title:
|
|
1004
|
+
heading?.slice(0, 160) ??
|
|
1005
|
+
neutralFallback ??
|
|
1006
|
+
`${context.window.label} digest`,
|
|
614
1007
|
summary:
|
|
615
|
-
|
|
1008
|
+
normalized.slice(0, 280) ||
|
|
1009
|
+
neutralFallback ||
|
|
616
1010
|
"No model summary was returned.",
|
|
617
1011
|
keyTopics: [],
|
|
618
1012
|
notableLinks: [],
|
|
@@ -625,6 +1019,7 @@ function fallbackDigest(
|
|
|
625
1019
|
function parseDigestFromHybridText(
|
|
626
1020
|
context: PeriodDigestContext,
|
|
627
1021
|
rawText: string,
|
|
1022
|
+
language?: string,
|
|
628
1023
|
): { digest: PeriodDigest; markdown: string } {
|
|
629
1024
|
const [markdownPart, jsonPart] = rawText.split(DELIMITER_PATTERN);
|
|
630
1025
|
const markdown = (markdownPart ?? rawText).trim();
|
|
@@ -639,10 +1034,13 @@ function parseDigestFromHybridText(
|
|
|
639
1034
|
digest: PeriodDigestSchema.parse(JSON.parse(candidate)),
|
|
640
1035
|
};
|
|
641
1036
|
} catch {
|
|
642
|
-
return {
|
|
1037
|
+
return {
|
|
1038
|
+
markdown,
|
|
1039
|
+
digest: fallbackDigest(context, markdown, language),
|
|
1040
|
+
};
|
|
643
1041
|
}
|
|
644
1042
|
}
|
|
645
|
-
return { markdown, digest: fallbackDigest(context, markdown) };
|
|
1043
|
+
return { markdown, digest: fallbackDigest(context, markdown, language) };
|
|
646
1044
|
}
|
|
647
1045
|
|
|
648
1046
|
function emitVisibleDelta(
|
|
@@ -786,7 +1184,9 @@ function createOpenAIRequestBody(
|
|
|
786
1184
|
},
|
|
787
1185
|
{
|
|
788
1186
|
role: "user",
|
|
789
|
-
content: buildPrompt(context
|
|
1187
|
+
content: buildPrompt(context, {
|
|
1188
|
+
language: languageFromOptions(options),
|
|
1189
|
+
}),
|
|
790
1190
|
},
|
|
791
1191
|
],
|
|
792
1192
|
};
|
|
@@ -829,7 +1229,11 @@ function readOpenAIStreamEffect(
|
|
|
829
1229
|
}
|
|
830
1230
|
|
|
831
1231
|
const parsed = yield* tryDigestSync(() =>
|
|
832
|
-
parseDigestFromHybridText(
|
|
1232
|
+
parseDigestFromHybridText(
|
|
1233
|
+
context,
|
|
1234
|
+
state.rawText,
|
|
1235
|
+
languageFromOptions(options),
|
|
1236
|
+
),
|
|
833
1237
|
);
|
|
834
1238
|
const cacheKey = digestCacheKey(context, options);
|
|
835
1239
|
const updatedAt = yield* tryDigestSync(() =>
|
|
@@ -870,11 +1274,20 @@ export function streamPeriodDigestEffect(
|
|
|
870
1274
|
handlers: PeriodDigestStreamHandlers = {},
|
|
871
1275
|
): Effect.Effect<PeriodDigestRunResult, Error> {
|
|
872
1276
|
return Effect.gen(function* () {
|
|
873
|
-
const
|
|
874
|
-
|
|
1277
|
+
const resolvedOptions = {
|
|
1278
|
+
...options,
|
|
1279
|
+
language: yield* tryDigestSync(() => languageFromOptions(options)),
|
|
1280
|
+
};
|
|
1281
|
+
yield* refreshPeriodDigestInputsEffect(
|
|
1282
|
+
resolvedOptions,
|
|
1283
|
+
{ threads: false },
|
|
1284
|
+
handlers,
|
|
1285
|
+
).pipe(Effect.catchAll(() => Effect.void));
|
|
1286
|
+
let context = yield* tryDigestSync(() =>
|
|
1287
|
+
collectPeriodDigestContext(resolvedOptions),
|
|
875
1288
|
);
|
|
876
|
-
|
|
877
|
-
const cached =
|
|
1289
|
+
let cacheKey = digestCacheKey(context, resolvedOptions);
|
|
1290
|
+
const cached = resolvedOptions.refresh
|
|
878
1291
|
? null
|
|
879
1292
|
: yield* tryDigestSync(() =>
|
|
880
1293
|
readSyncCache<{
|
|
@@ -904,21 +1317,39 @@ export function streamPeriodDigestEffect(
|
|
|
904
1317
|
return result;
|
|
905
1318
|
}
|
|
906
1319
|
|
|
1320
|
+
yield* refreshPeriodDigestInputsEffect(
|
|
1321
|
+
resolvedOptions,
|
|
1322
|
+
{
|
|
1323
|
+
timeline: false,
|
|
1324
|
+
mentions: false,
|
|
1325
|
+
threads: true,
|
|
1326
|
+
threadTweetIds: context.tweets
|
|
1327
|
+
.filter((tweet) => tweet.source === "mentions")
|
|
1328
|
+
.map((tweet) => tweet.id),
|
|
1329
|
+
},
|
|
1330
|
+
handlers,
|
|
1331
|
+
).pipe(Effect.catchAll(() => Effect.void));
|
|
1332
|
+
context = yield* tryDigestSync(() =>
|
|
1333
|
+
collectPeriodDigestContext(resolvedOptions),
|
|
1334
|
+
);
|
|
1335
|
+
cacheKey = digestCacheKey(context, resolvedOptions);
|
|
1336
|
+
|
|
907
1337
|
const apiKey = process.env.OPENAI_API_KEY;
|
|
908
1338
|
if (!apiKey) {
|
|
909
1339
|
return yield* Effect.fail(new Error("OPENAI_API_KEY is not set"));
|
|
910
1340
|
}
|
|
911
1341
|
|
|
912
1342
|
handlers.onEvent?.({ type: "start", context, cached: false });
|
|
1343
|
+
emitDigestStatus(handlers, "Streaming AI summary");
|
|
913
1344
|
const response = yield* tryDigestPromise(() =>
|
|
914
1345
|
fetch("https://api.openai.com/v1/responses", {
|
|
915
1346
|
method: "POST",
|
|
916
|
-
signal:
|
|
1347
|
+
signal: resolvedOptions.signal,
|
|
917
1348
|
headers: {
|
|
918
1349
|
authorization: `Bearer ${apiKey}`,
|
|
919
1350
|
"content-type": "application/json",
|
|
920
1351
|
},
|
|
921
|
-
body: JSON.stringify(createOpenAIRequestBody(context,
|
|
1352
|
+
body: JSON.stringify(createOpenAIRequestBody(context, resolvedOptions)),
|
|
922
1353
|
}),
|
|
923
1354
|
);
|
|
924
1355
|
if (!response.ok) {
|
|
@@ -932,7 +1363,12 @@ export function streamPeriodDigestEffect(
|
|
|
932
1363
|
),
|
|
933
1364
|
);
|
|
934
1365
|
}
|
|
935
|
-
return yield* readOpenAIStreamEffect(
|
|
1366
|
+
return yield* readOpenAIStreamEffect(
|
|
1367
|
+
response,
|
|
1368
|
+
context,
|
|
1369
|
+
resolvedOptions,
|
|
1370
|
+
handlers,
|
|
1371
|
+
);
|
|
936
1372
|
});
|
|
937
1373
|
}
|
|
938
1374
|
|
|
@@ -946,6 +1382,9 @@ export function streamPeriodDigest(
|
|
|
946
1382
|
export const __test__ = {
|
|
947
1383
|
PeriodDigestSchema,
|
|
948
1384
|
buildPrompt,
|
|
1385
|
+
digestCacheKey,
|
|
1386
|
+
languageFromOptions,
|
|
1387
|
+
normalizeDigestLanguage,
|
|
949
1388
|
readOpenAIStreamEffect,
|
|
950
1389
|
parseDigestFromHybridText,
|
|
951
1390
|
processSseChunk,
|