birdclaw 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +32 -2
  3. package/package.json +29 -30
  4. package/scripts/build-docs-site.mjs +39 -12
  5. package/src/cli.ts +457 -26
  6. package/src/components/AppNav.tsx +10 -0
  7. package/src/components/MarkdownViewer.tsx +438 -72
  8. package/src/components/ProfileAnalysisStream.tsx +428 -0
  9. package/src/components/ProfilePreview.tsx +120 -9
  10. package/src/components/SavedTimelineView.tsx +30 -8
  11. package/src/components/SyncNowButton.tsx +5 -2
  12. package/src/components/TimelineCard.tsx +20 -4
  13. package/src/components/TimelineRouteFrame.tsx +16 -0
  14. package/src/components/TweetRichText.tsx +36 -12
  15. package/src/components/useTimelineRouteData.ts +74 -6
  16. package/src/lib/account-sync-job.ts +15 -3
  17. package/src/lib/archive-finder.ts +1 -1
  18. package/src/lib/archive-import.ts +245 -7
  19. package/src/lib/authored-live.ts +1 -0
  20. package/src/lib/avatar-cache.ts +50 -0
  21. package/src/lib/backup.ts +4 -3
  22. package/src/lib/bird.ts +33 -0
  23. package/src/lib/config.ts +35 -2
  24. package/src/lib/data-sources.ts +219 -0
  25. package/src/lib/db.ts +62 -1
  26. package/src/lib/geocoding.ts +296 -0
  27. package/src/lib/link-insights.ts +2 -0
  28. package/src/lib/location.ts +137 -0
  29. package/src/lib/mention-threads-live.ts +94 -1
  30. package/src/lib/mentions-live.ts +187 -40
  31. package/src/lib/network-map.ts +382 -0
  32. package/src/lib/period-digest.ts +468 -29
  33. package/src/lib/profile-analysis.ts +1272 -0
  34. package/src/lib/profile-bio-entities.ts +1 -1
  35. package/src/lib/queries.ts +14 -4
  36. package/src/lib/search-discussion.ts +1016 -0
  37. package/src/lib/timeline-live.ts +272 -19
  38. package/src/lib/tweet-account-edges.ts +2 -0
  39. package/src/lib/tweet-render.ts +141 -1
  40. package/src/lib/tweet-search-live.ts +565 -0
  41. package/src/lib/types.ts +37 -2
  42. package/src/lib/ui.ts +1 -1
  43. package/src/lib/web-sync.ts +7 -2
  44. package/src/lib/xurl-rate-limits.ts +267 -0
  45. package/src/lib/xurl.ts +551 -41
  46. package/src/routeTree.gen.ts +231 -0
  47. package/src/routes/__root.tsx +5 -6
  48. package/src/routes/api/data-sources.tsx +24 -0
  49. package/src/routes/api/network-map.tsx +55 -0
  50. package/src/routes/api/period-digest.tsx +29 -3
  51. package/src/routes/api/profile-analysis.tsx +152 -0
  52. package/src/routes/api/query.tsx +1 -0
  53. package/src/routes/api/search-discussion.tsx +169 -0
  54. package/src/routes/api/xurl-rate-limits.tsx +24 -0
  55. package/src/routes/data-sources.tsx +257 -0
  56. package/src/routes/discuss.tsx +419 -0
  57. package/src/routes/network-map.tsx +1035 -0
  58. package/src/routes/profile-analyze.tsx +112 -0
  59. package/src/routes/profiles.$handle.tsx +228 -0
  60. package/src/routes/rate-limits.tsx +309 -0
  61. package/src/routes/today.tsx +22 -8
  62. package/src/styles.css +22 -0
@@ -6,8 +6,10 @@ import {
6
6
  Image,
7
7
  MessageCircle,
8
8
  Repeat2,
9
+ UserSearch,
9
10
  } from "lucide-react";
10
11
  import { formatCompactNumber, formatShortTimestamp } from "#/lib/present";
12
+ import { normalizeTweetUrlEntityRangeForText } from "#/lib/tweet-render";
11
13
  import type {
12
14
  TimelineItem,
13
15
  TweetEntities,
@@ -98,10 +100,10 @@ function isUnresolvedShortUrlEntity(entry: TweetUrlEntity) {
98
100
  return !entry.displayUrl && isShortUrl(entry.url);
99
101
  }
100
102
 
101
- function unresolvedShortUrlRanges(entities: TweetEntities) {
103
+ function unresolvedShortUrlRanges(text: string, entities: TweetEntities) {
102
104
  return (entities.urls ?? [])
103
105
  .filter(isUnresolvedShortUrlEntity)
104
- .map((entry) => ({ start: entry.start, end: entry.end }));
106
+ .map((entry) => normalizeTweetUrlEntityRangeForText(text, entry));
105
107
  }
106
108
 
107
109
  function textOutsideRanges(
@@ -133,7 +135,7 @@ function shouldHideUnresolvedShortUrls(
133
135
  mediaUrls: Set<string>,
134
136
  ) {
135
137
  if (mediaUrls.size === 0) return false;
136
- const ranges = unresolvedShortUrlRanges(entities);
138
+ const ranges = unresolvedShortUrlRanges(text, entities);
137
139
  if (ranges.length === 0) return false;
138
140
  return textOutsideRanges(text, ranges).trim().length === 0;
139
141
  }
@@ -201,7 +203,7 @@ function getHiddenMediaUrlRanges(
201
203
  isMediaUrlEntity(entry, mediaUrls, tweetId) ||
202
204
  (hideUnresolvedShortUrls && isUnresolvedShortUrlEntity(entry)),
203
205
  )
204
- .map((entry) => ({ start: entry.start, end: entry.end }));
206
+ .map((entry) => normalizeTweetUrlEntityRangeForText(text, entry));
205
207
  }
206
208
 
207
209
  function getVisibleUrlCards(
@@ -444,6 +446,20 @@ export function TimelineCard({
444
446
  <span className="text-[13px]">Reply</span>
445
447
  </button>
446
448
  ) : null}
449
+ <a
450
+ aria-label={`Analyse @${displayAuthor.handle}`}
451
+ className={feedActionButtonClass}
452
+ href={`/profiles/${encodeURIComponent(displayAuthor.handle)}`}
453
+ onClick={(event) => {
454
+ event.stopPropagation();
455
+ }}
456
+ title={`Analyse @${displayAuthor.handle}`}
457
+ >
458
+ <span className={feedActionIconWrapClass}>
459
+ <UserSearch className={feedActionIconClass} strokeWidth={1.7} />
460
+ </span>
461
+ <span className="text-[13px]">Analyse</span>
462
+ </a>
447
463
  {showLikeIndicator ? (
448
464
  <span
449
465
  aria-label={`${formatCompactNumber(displayLikeCount)} likes`}
@@ -78,6 +78,9 @@ export function TimelineRouteFrame({
78
78
  retry,
79
79
  refreshLocalView,
80
80
  replyToTweet,
81
+ hasMore,
82
+ loadingMore,
83
+ loadMore,
81
84
  } = useTimelineRouteData({
82
85
  resource,
83
86
  replyFilter,
@@ -99,6 +102,7 @@ export function TimelineRouteFrame({
99
102
  kind={syncKind}
100
103
  label={syncLabel}
101
104
  onSynced={refreshLocalView}
105
+ showAccountPicker
102
106
  />
103
107
  </div>
104
108
  <div className="px-4 pb-3">
@@ -163,6 +167,18 @@ export function TimelineRouteFrame({
163
167
  {items.map((item) => (
164
168
  <TimelineCard key={item.id} item={item} onReply={replyToTweet} />
165
169
  ))}
170
+ {!loading && !error && hasMore ? (
171
+ <div className="flex justify-center py-4">
172
+ <button
173
+ className="rounded-full bg-[var(--accent)] px-5 py-1.5 text-[14px] font-bold text-white disabled:opacity-60"
174
+ disabled={loadingMore}
175
+ onClick={loadMore}
176
+ type="button"
177
+ >
178
+ {loadingMore ? "Loading…" : "Load more"}
179
+ </button>
180
+ </div>
181
+ ) : null}
166
182
  </section>
167
183
  </ConversationSurfaceScope>
168
184
  </>
@@ -1,8 +1,9 @@
1
1
  import { Fragment } from "react";
2
2
  import type { ReactNode } from "react";
3
3
  import {
4
- collectTweetSegments,
4
+ collectTweetSegmentsForText,
5
5
  enrichFallbackUrlEntities,
6
+ normalizeTweetUrlEntityRangeForText,
6
7
  } from "#/lib/tweet-render";
7
8
  import type { TweetEntities } from "#/lib/types";
8
9
  import {
@@ -14,23 +15,40 @@ import {
14
15
  import { safeHttpUrl } from "#/lib/url-safety";
15
16
  import { ProfilePreview } from "./ProfilePreview";
16
17
 
18
+ function rangeKey(range: { start: number; end: number }) {
19
+ return `${range.start}:${range.end}`;
20
+ }
21
+
17
22
  export function TweetRichText({
18
23
  text,
19
24
  entities,
20
25
  className = "body-copy",
21
26
  hiddenUrlRanges = [],
27
+ urlLabel = "display",
28
+ as = "p",
22
29
  }: {
23
30
  text: string;
24
31
  entities: TweetEntities;
25
32
  className?: string;
26
33
  hiddenUrlRanges?: Array<{ start: number; end: number }>;
34
+ urlLabel?: "display" | "expanded";
35
+ as?: "p" | "span";
27
36
  }) {
28
37
  const richEntities = enrichFallbackUrlEntities(text, entities);
29
- const segments = collectTweetSegments(richEntities);
38
+ const segments = collectTweetSegmentsForText(text, richEntities);
39
+ const hiddenRawRangeKeys = new Set(hiddenUrlRanges.map(rangeKey));
40
+ const hiddenRangeKeys = new Set(hiddenRawRangeKeys);
41
+ for (const entry of richEntities.urls ?? []) {
42
+ if (!hiddenRawRangeKeys.has(rangeKey(entry))) continue;
43
+ hiddenRangeKeys.add(
44
+ rangeKey(normalizeTweetUrlEntityRangeForText(text, entry)),
45
+ );
46
+ }
47
+ const Wrapper = as;
30
48
  let cursor = 0;
31
49
 
32
50
  return (
33
- <p className={className === "body-copy" ? bodyCopyClass : className}>
51
+ <Wrapper className={className === "body-copy" ? bodyCopyClass : className}>
34
52
  {segments.map((segment, index) => {
35
53
  if (
36
54
  segment.start < cursor ||
@@ -48,13 +66,7 @@ export function TweetRichText({
48
66
  {text.slice(segment.start, segment.end)}
49
67
  </Fragment>
50
68
  );
51
- if (
52
- segment.kind === "url" &&
53
- hiddenUrlRanges.some(
54
- (range) =>
55
- range.start === segment.start && range.end === segment.end,
56
- )
57
- ) {
69
+ if (segment.kind === "url" && hiddenRangeKeys.has(rangeKey(segment))) {
58
70
  node = null;
59
71
  } else if (segment.kind === "mention" && segment.profile) {
60
72
  node = (
@@ -65,6 +77,16 @@ export function TweetRichText({
65
77
  <span className={tweetMentionClass}>@{segment.username}</span>
66
78
  </ProfilePreview>
67
79
  );
80
+ } else if (segment.kind === "mention") {
81
+ node = (
82
+ <a
83
+ key={`segment-${String(index)}`}
84
+ className={tweetMentionClass}
85
+ href={`/profiles/${encodeURIComponent(segment.username)}`}
86
+ >
87
+ @{segment.username}
88
+ </a>
89
+ );
68
90
  } else if (segment.kind === "url") {
69
91
  const href = safeHttpUrl(segment.expandedUrl);
70
92
  if (href) {
@@ -76,7 +98,9 @@ export function TweetRichText({
76
98
  rel="noreferrer"
77
99
  target="_blank"
78
100
  >
79
- {segment.displayUrl}
101
+ {urlLabel === "expanded"
102
+ ? segment.expandedUrl
103
+ : segment.displayUrl}
80
104
  </a>
81
105
  );
82
106
  }
@@ -99,6 +123,6 @@ export function TweetRichText({
99
123
  );
100
124
  })}
101
125
  {text.slice(cursor)}
102
- </p>
126
+ </Wrapper>
103
127
  );
104
128
  }
@@ -1,4 +1,4 @@
1
- import { useEffect, useState } from "react";
1
+ import { useEffect, useRef, useState } from "react";
2
2
  import type {
3
3
  QueryEnvelope,
4
4
  ReplyFilter,
@@ -12,6 +12,8 @@ import {
12
12
  } from "#/lib/api-client";
13
13
  import { useSelectedAccountId } from "./account-selection";
14
14
 
15
+ const PAGE_SIZE = 50;
16
+
15
17
  interface UseTimelineRouteDataOptions {
16
18
  resource: Exclude<ResourceKind, "dms">;
17
19
  search: string;
@@ -35,7 +37,14 @@ export function useTimelineRouteData({
35
37
  const [error, setError] = useState<string | null>(null);
36
38
  const [replyError, setReplyError] = useState<string | null>(null);
37
39
  const [refreshTick, setRefreshTick] = useState(0);
40
+ const [hasMore, setHasMore] = useState(false);
41
+ const [loadingMore, setLoadingMore] = useState(false);
38
42
  const selectedAccountId = useSelectedAccountId(meta?.accounts);
43
+ // Bumped whenever the active filters change. A `loadMore` request that
44
+ // resolves against a stale generation is discarded so its older page is
45
+ // never appended to a freshly loaded feed.
46
+ const generationRef = useRef(0);
47
+ const loadMoreControllerRef = useRef<AbortController | null>(null);
39
48
 
40
49
  async function loadStatus() {
41
50
  setMeta(await fetchQueryEnvelope());
@@ -45,10 +54,14 @@ export function useTimelineRouteData({
45
54
  void loadStatus();
46
55
  }, []);
47
56
 
48
- useEffect(() => {
57
+ // Build the /api/query URL for the current filters. Passing the last item's
58
+ // (createdAt, id) requests the next (older) page via the server's keyset
59
+ // cursor, which is deterministic across duplicate timestamps.
60
+ function buildQueryUrl(until?: string, untilId?: string) {
49
61
  const url = new URL("/api/query", window.location.origin);
50
62
  url.searchParams.set("resource", resource);
51
63
  url.searchParams.set("refresh", String(refreshTick));
64
+ url.searchParams.set("limit", String(PAGE_SIZE));
52
65
  if (selectedAccountId) {
53
66
  url.searchParams.set("account", selectedAccountId);
54
67
  }
@@ -64,16 +77,29 @@ export function useTimelineRouteData({
64
77
  if (search.trim()) {
65
78
  url.searchParams.set("search", search.trim());
66
79
  }
80
+ if (until) {
81
+ url.searchParams.set("until", until);
82
+ }
83
+ if (untilId) {
84
+ url.searchParams.set("untilId", untilId);
85
+ }
86
+ return url;
87
+ }
67
88
 
89
+ useEffect(() => {
90
+ generationRef.current += 1;
91
+ loadMoreControllerRef.current?.abort();
68
92
  const controller = new AbortController();
69
93
  let active = true;
70
94
  setError(null);
71
95
  setLoading(true);
72
- fetchQueryResponse(url, { signal: controller.signal })
96
+ setLoadingMore(false);
97
+ fetchQueryResponse(buildQueryUrl(), { signal: controller.signal })
73
98
  .then((data) => {
74
- if (active) {
75
- setItems(data.items as TimelineItem[]);
76
- }
99
+ if (!active) return;
100
+ const next = data.items as TimelineItem[];
101
+ setItems(next);
102
+ setHasMore(next.length >= PAGE_SIZE);
77
103
  })
78
104
  .catch((fetchError: unknown) => {
79
105
  if (
@@ -87,6 +113,7 @@ export function useTimelineRouteData({
87
113
  fetchError instanceof Error ? fetchError.message : errorFallback,
88
114
  );
89
115
  setItems([]);
116
+ setHasMore(false);
90
117
  })
91
118
  .finally(() => {
92
119
  if (active) {
@@ -109,6 +136,44 @@ export function useTimelineRouteData({
109
136
  selectedAccountId,
110
137
  ]);
111
138
 
139
+ async function loadMore() {
140
+ if (loading || loadingMore || !hasMore || items.length === 0) return;
141
+ const lastItem = items[items.length - 1];
142
+ const until = lastItem?.createdAt;
143
+ const untilId = lastItem?.id;
144
+ if (!until || !untilId) return;
145
+ const generation = generationRef.current;
146
+ const controller = new AbortController();
147
+ loadMoreControllerRef.current = controller;
148
+ setLoadingMore(true);
149
+ try {
150
+ const data = await fetchQueryResponse(buildQueryUrl(until, untilId), {
151
+ signal: controller.signal,
152
+ });
153
+ // Discard if the filters changed (new generation) while in flight.
154
+ if (generation !== generationRef.current) return;
155
+ const page = data.items as TimelineItem[];
156
+ setItems((prev) => {
157
+ const seen = new Set(prev.map((item) => item.id));
158
+ return [...prev, ...page.filter((item) => !seen.has(item.id))];
159
+ });
160
+ setHasMore(page.length >= PAGE_SIZE);
161
+ } catch (loadError) {
162
+ if (
163
+ loadError instanceof DOMException &&
164
+ loadError.name === "AbortError"
165
+ ) {
166
+ return;
167
+ }
168
+ if (generation !== generationRef.current) return;
169
+ setError(loadError instanceof Error ? loadError.message : errorFallback);
170
+ } finally {
171
+ if (generation === generationRef.current) {
172
+ setLoadingMore(false);
173
+ }
174
+ }
175
+ }
176
+
112
177
  function retry() {
113
178
  setRefreshTick((value) => value + 1);
114
179
  }
@@ -149,5 +214,8 @@ export function useTimelineRouteData({
149
214
  refreshLocalView,
150
215
  replyToTweet,
151
216
  selectedAccountId,
217
+ hasMore,
218
+ loadingMore,
219
+ loadMore,
152
220
  };
153
221
  }
@@ -249,12 +249,22 @@ async function runStep({
249
249
  }): Promise<AccountSyncAuditStep> {
250
250
  try {
251
251
  if (kind === "timeline") {
252
- if (!allowBirdAccount) {
252
+ const timelineMode =
253
+ mode === "auto"
254
+ ? account
255
+ ? allowBirdAccount
256
+ ? "bird"
257
+ : "xurl"
258
+ : "auto"
259
+ : mode;
260
+ if (timelineMode === "bird" && !allowBirdAccount) {
253
261
  return { kind, ok: false, count: 0, error: birdAccountError(kind) };
254
262
  }
255
263
  const result = await syncHomeTimeline({
256
264
  account,
265
+ mode: timelineMode,
257
266
  limit,
267
+ maxPages,
258
268
  following: true,
259
269
  refresh,
260
270
  cacheTtlMs,
@@ -267,12 +277,14 @@ async function runStep({
267
277
  };
268
278
  }
269
279
  if (kind === "mentions") {
270
- if (!allowBirdAccount) {
280
+ if (mode === "bird" && !allowBirdAccount) {
271
281
  return { kind, ok: false, count: 0, error: birdAccountError(kind) };
272
282
  }
283
+ const mentionMode =
284
+ mode === "auto" ? (allowBirdAccount ? "auto" : "xurl") : mode;
273
285
  const result = await syncMentions({
274
286
  account,
275
- mode: "bird",
287
+ mode: mentionMode,
276
288
  limit,
277
289
  maxPages,
278
290
  refresh,
@@ -75,7 +75,7 @@ function searchDirectoryEffect(directoryPath: string) {
75
75
  );
76
76
 
77
77
  return candidates.filter((item) => item !== null);
78
- });
78
+ }).pipe(Effect.catchAll(() => Effect.succeed([])));
79
79
  }
80
80
 
81
81
  function searchSpotlightEffect(query: string) {