birdclaw 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +25 -0
- package/package.json +6 -1
- package/src/cli.ts +438 -26
- package/src/components/AppNav.tsx +10 -0
- package/src/components/MarkdownViewer.tsx +438 -72
- package/src/components/ProfileAnalysisStream.tsx +428 -0
- package/src/components/ProfilePreview.tsx +120 -9
- package/src/components/SavedTimelineView.tsx +30 -8
- package/src/components/SyncNowButton.tsx +5 -2
- package/src/components/TimelineCard.tsx +20 -4
- package/src/components/TimelineRouteFrame.tsx +16 -0
- package/src/components/TweetRichText.tsx +36 -12
- package/src/components/useTimelineRouteData.ts +74 -6
- package/src/lib/account-sync-job.ts +15 -3
- package/src/lib/archive-finder.ts +1 -1
- package/src/lib/archive-import.ts +245 -7
- package/src/lib/authored-live.ts +1 -0
- package/src/lib/avatar-cache.ts +50 -0
- package/src/lib/backup.ts +4 -3
- package/src/lib/bird.ts +33 -0
- package/src/lib/config.ts +35 -2
- package/src/lib/data-sources.ts +219 -0
- package/src/lib/db.ts +62 -1
- package/src/lib/geocoding.ts +296 -0
- package/src/lib/location.ts +137 -0
- package/src/lib/mention-threads-live.ts +94 -1
- package/src/lib/mentions-live.ts +187 -40
- package/src/lib/network-map.ts +382 -0
- package/src/lib/period-digest.ts +377 -13
- package/src/lib/profile-analysis.ts +1272 -0
- package/src/lib/profile-bio-entities.ts +1 -1
- package/src/lib/queries.ts +14 -4
- package/src/lib/search-discussion.ts +1016 -0
- package/src/lib/timeline-live.ts +272 -19
- package/src/lib/tweet-account-edges.ts +2 -0
- package/src/lib/tweet-render.ts +141 -1
- package/src/lib/tweet-search-live.ts +565 -0
- package/src/lib/types.ts +37 -2
- package/src/lib/ui.ts +1 -1
- package/src/lib/web-sync.ts +7 -2
- package/src/lib/xurl-rate-limits.ts +267 -0
- package/src/lib/xurl.ts +551 -41
- package/src/routeTree.gen.ts +231 -0
- package/src/routes/__root.tsx +5 -6
- package/src/routes/api/data-sources.tsx +24 -0
- package/src/routes/api/network-map.tsx +55 -0
- package/src/routes/api/period-digest.tsx +11 -1
- package/src/routes/api/profile-analysis.tsx +152 -0
- package/src/routes/api/query.tsx +1 -0
- package/src/routes/api/search-discussion.tsx +169 -0
- package/src/routes/api/xurl-rate-limits.tsx +24 -0
- package/src/routes/data-sources.tsx +255 -0
- package/src/routes/discuss.tsx +419 -0
- package/src/routes/network-map.tsx +1035 -0
- package/src/routes/profile-analyze.tsx +112 -0
- package/src/routes/profiles.$handle.tsx +228 -0
- package/src/routes/rate-limits.tsx +309 -0
- package/src/routes/today.tsx +22 -8
- package/src/styles.css +22 -0
|
@@ -6,8 +6,10 @@ import {
|
|
|
6
6
|
Image,
|
|
7
7
|
MessageCircle,
|
|
8
8
|
Repeat2,
|
|
9
|
+
UserSearch,
|
|
9
10
|
} from "lucide-react";
|
|
10
11
|
import { formatCompactNumber, formatShortTimestamp } from "#/lib/present";
|
|
12
|
+
import { normalizeTweetUrlEntityRangeForText } from "#/lib/tweet-render";
|
|
11
13
|
import type {
|
|
12
14
|
TimelineItem,
|
|
13
15
|
TweetEntities,
|
|
@@ -98,10 +100,10 @@ function isUnresolvedShortUrlEntity(entry: TweetUrlEntity) {
|
|
|
98
100
|
return !entry.displayUrl && isShortUrl(entry.url);
|
|
99
101
|
}
|
|
100
102
|
|
|
101
|
-
function unresolvedShortUrlRanges(entities: TweetEntities) {
|
|
103
|
+
function unresolvedShortUrlRanges(text: string, entities: TweetEntities) {
|
|
102
104
|
return (entities.urls ?? [])
|
|
103
105
|
.filter(isUnresolvedShortUrlEntity)
|
|
104
|
-
.map((entry) => (
|
|
106
|
+
.map((entry) => normalizeTweetUrlEntityRangeForText(text, entry));
|
|
105
107
|
}
|
|
106
108
|
|
|
107
109
|
function textOutsideRanges(
|
|
@@ -133,7 +135,7 @@ function shouldHideUnresolvedShortUrls(
|
|
|
133
135
|
mediaUrls: Set<string>,
|
|
134
136
|
) {
|
|
135
137
|
if (mediaUrls.size === 0) return false;
|
|
136
|
-
const ranges = unresolvedShortUrlRanges(entities);
|
|
138
|
+
const ranges = unresolvedShortUrlRanges(text, entities);
|
|
137
139
|
if (ranges.length === 0) return false;
|
|
138
140
|
return textOutsideRanges(text, ranges).trim().length === 0;
|
|
139
141
|
}
|
|
@@ -201,7 +203,7 @@ function getHiddenMediaUrlRanges(
|
|
|
201
203
|
isMediaUrlEntity(entry, mediaUrls, tweetId) ||
|
|
202
204
|
(hideUnresolvedShortUrls && isUnresolvedShortUrlEntity(entry)),
|
|
203
205
|
)
|
|
204
|
-
.map((entry) => (
|
|
206
|
+
.map((entry) => normalizeTweetUrlEntityRangeForText(text, entry));
|
|
205
207
|
}
|
|
206
208
|
|
|
207
209
|
function getVisibleUrlCards(
|
|
@@ -444,6 +446,20 @@ export function TimelineCard({
|
|
|
444
446
|
<span className="text-[13px]">Reply</span>
|
|
445
447
|
</button>
|
|
446
448
|
) : null}
|
|
449
|
+
<a
|
|
450
|
+
aria-label={`Analyse @${displayAuthor.handle}`}
|
|
451
|
+
className={feedActionButtonClass}
|
|
452
|
+
href={`/profiles/${encodeURIComponent(displayAuthor.handle)}`}
|
|
453
|
+
onClick={(event) => {
|
|
454
|
+
event.stopPropagation();
|
|
455
|
+
}}
|
|
456
|
+
title={`Analyse @${displayAuthor.handle}`}
|
|
457
|
+
>
|
|
458
|
+
<span className={feedActionIconWrapClass}>
|
|
459
|
+
<UserSearch className={feedActionIconClass} strokeWidth={1.7} />
|
|
460
|
+
</span>
|
|
461
|
+
<span className="text-[13px]">Analyse</span>
|
|
462
|
+
</a>
|
|
447
463
|
{showLikeIndicator ? (
|
|
448
464
|
<span
|
|
449
465
|
aria-label={`${formatCompactNumber(displayLikeCount)} likes`}
|
|
@@ -78,6 +78,9 @@ export function TimelineRouteFrame({
|
|
|
78
78
|
retry,
|
|
79
79
|
refreshLocalView,
|
|
80
80
|
replyToTweet,
|
|
81
|
+
hasMore,
|
|
82
|
+
loadingMore,
|
|
83
|
+
loadMore,
|
|
81
84
|
} = useTimelineRouteData({
|
|
82
85
|
resource,
|
|
83
86
|
replyFilter,
|
|
@@ -99,6 +102,7 @@ export function TimelineRouteFrame({
|
|
|
99
102
|
kind={syncKind}
|
|
100
103
|
label={syncLabel}
|
|
101
104
|
onSynced={refreshLocalView}
|
|
105
|
+
showAccountPicker
|
|
102
106
|
/>
|
|
103
107
|
</div>
|
|
104
108
|
<div className="px-4 pb-3">
|
|
@@ -163,6 +167,18 @@ export function TimelineRouteFrame({
|
|
|
163
167
|
{items.map((item) => (
|
|
164
168
|
<TimelineCard key={item.id} item={item} onReply={replyToTweet} />
|
|
165
169
|
))}
|
|
170
|
+
{!loading && !error && hasMore ? (
|
|
171
|
+
<div className="flex justify-center py-4">
|
|
172
|
+
<button
|
|
173
|
+
className="rounded-full bg-[var(--accent)] px-5 py-1.5 text-[14px] font-bold text-white disabled:opacity-60"
|
|
174
|
+
disabled={loadingMore}
|
|
175
|
+
onClick={loadMore}
|
|
176
|
+
type="button"
|
|
177
|
+
>
|
|
178
|
+
{loadingMore ? "Loading…" : "Load more"}
|
|
179
|
+
</button>
|
|
180
|
+
</div>
|
|
181
|
+
) : null}
|
|
166
182
|
</section>
|
|
167
183
|
</ConversationSurfaceScope>
|
|
168
184
|
</>
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { Fragment } from "react";
|
|
2
2
|
import type { ReactNode } from "react";
|
|
3
3
|
import {
|
|
4
|
-
|
|
4
|
+
collectTweetSegmentsForText,
|
|
5
5
|
enrichFallbackUrlEntities,
|
|
6
|
+
normalizeTweetUrlEntityRangeForText,
|
|
6
7
|
} from "#/lib/tweet-render";
|
|
7
8
|
import type { TweetEntities } from "#/lib/types";
|
|
8
9
|
import {
|
|
@@ -14,23 +15,40 @@ import {
|
|
|
14
15
|
import { safeHttpUrl } from "#/lib/url-safety";
|
|
15
16
|
import { ProfilePreview } from "./ProfilePreview";
|
|
16
17
|
|
|
18
|
+
function rangeKey(range: { start: number; end: number }) {
|
|
19
|
+
return `${range.start}:${range.end}`;
|
|
20
|
+
}
|
|
21
|
+
|
|
17
22
|
export function TweetRichText({
|
|
18
23
|
text,
|
|
19
24
|
entities,
|
|
20
25
|
className = "body-copy",
|
|
21
26
|
hiddenUrlRanges = [],
|
|
27
|
+
urlLabel = "display",
|
|
28
|
+
as = "p",
|
|
22
29
|
}: {
|
|
23
30
|
text: string;
|
|
24
31
|
entities: TweetEntities;
|
|
25
32
|
className?: string;
|
|
26
33
|
hiddenUrlRanges?: Array<{ start: number; end: number }>;
|
|
34
|
+
urlLabel?: "display" | "expanded";
|
|
35
|
+
as?: "p" | "span";
|
|
27
36
|
}) {
|
|
28
37
|
const richEntities = enrichFallbackUrlEntities(text, entities);
|
|
29
|
-
const segments =
|
|
38
|
+
const segments = collectTweetSegmentsForText(text, richEntities);
|
|
39
|
+
const hiddenRawRangeKeys = new Set(hiddenUrlRanges.map(rangeKey));
|
|
40
|
+
const hiddenRangeKeys = new Set(hiddenRawRangeKeys);
|
|
41
|
+
for (const entry of richEntities.urls ?? []) {
|
|
42
|
+
if (!hiddenRawRangeKeys.has(rangeKey(entry))) continue;
|
|
43
|
+
hiddenRangeKeys.add(
|
|
44
|
+
rangeKey(normalizeTweetUrlEntityRangeForText(text, entry)),
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
const Wrapper = as;
|
|
30
48
|
let cursor = 0;
|
|
31
49
|
|
|
32
50
|
return (
|
|
33
|
-
<
|
|
51
|
+
<Wrapper className={className === "body-copy" ? bodyCopyClass : className}>
|
|
34
52
|
{segments.map((segment, index) => {
|
|
35
53
|
if (
|
|
36
54
|
segment.start < cursor ||
|
|
@@ -48,13 +66,7 @@ export function TweetRichText({
|
|
|
48
66
|
{text.slice(segment.start, segment.end)}
|
|
49
67
|
</Fragment>
|
|
50
68
|
);
|
|
51
|
-
if (
|
|
52
|
-
segment.kind === "url" &&
|
|
53
|
-
hiddenUrlRanges.some(
|
|
54
|
-
(range) =>
|
|
55
|
-
range.start === segment.start && range.end === segment.end,
|
|
56
|
-
)
|
|
57
|
-
) {
|
|
69
|
+
if (segment.kind === "url" && hiddenRangeKeys.has(rangeKey(segment))) {
|
|
58
70
|
node = null;
|
|
59
71
|
} else if (segment.kind === "mention" && segment.profile) {
|
|
60
72
|
node = (
|
|
@@ -65,6 +77,16 @@ export function TweetRichText({
|
|
|
65
77
|
<span className={tweetMentionClass}>@{segment.username}</span>
|
|
66
78
|
</ProfilePreview>
|
|
67
79
|
);
|
|
80
|
+
} else if (segment.kind === "mention") {
|
|
81
|
+
node = (
|
|
82
|
+
<a
|
|
83
|
+
key={`segment-${String(index)}`}
|
|
84
|
+
className={tweetMentionClass}
|
|
85
|
+
href={`/profiles/${encodeURIComponent(segment.username)}`}
|
|
86
|
+
>
|
|
87
|
+
@{segment.username}
|
|
88
|
+
</a>
|
|
89
|
+
);
|
|
68
90
|
} else if (segment.kind === "url") {
|
|
69
91
|
const href = safeHttpUrl(segment.expandedUrl);
|
|
70
92
|
if (href) {
|
|
@@ -76,7 +98,9 @@ export function TweetRichText({
|
|
|
76
98
|
rel="noreferrer"
|
|
77
99
|
target="_blank"
|
|
78
100
|
>
|
|
79
|
-
{
|
|
101
|
+
{urlLabel === "expanded"
|
|
102
|
+
? segment.expandedUrl
|
|
103
|
+
: segment.displayUrl}
|
|
80
104
|
</a>
|
|
81
105
|
);
|
|
82
106
|
}
|
|
@@ -99,6 +123,6 @@ export function TweetRichText({
|
|
|
99
123
|
);
|
|
100
124
|
})}
|
|
101
125
|
{text.slice(cursor)}
|
|
102
|
-
</
|
|
126
|
+
</Wrapper>
|
|
103
127
|
);
|
|
104
128
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { useEffect, useState } from "react";
|
|
1
|
+
import { useEffect, useRef, useState } from "react";
|
|
2
2
|
import type {
|
|
3
3
|
QueryEnvelope,
|
|
4
4
|
ReplyFilter,
|
|
@@ -12,6 +12,8 @@ import {
|
|
|
12
12
|
} from "#/lib/api-client";
|
|
13
13
|
import { useSelectedAccountId } from "./account-selection";
|
|
14
14
|
|
|
15
|
+
const PAGE_SIZE = 50;
|
|
16
|
+
|
|
15
17
|
interface UseTimelineRouteDataOptions {
|
|
16
18
|
resource: Exclude<ResourceKind, "dms">;
|
|
17
19
|
search: string;
|
|
@@ -35,7 +37,14 @@ export function useTimelineRouteData({
|
|
|
35
37
|
const [error, setError] = useState<string | null>(null);
|
|
36
38
|
const [replyError, setReplyError] = useState<string | null>(null);
|
|
37
39
|
const [refreshTick, setRefreshTick] = useState(0);
|
|
40
|
+
const [hasMore, setHasMore] = useState(false);
|
|
41
|
+
const [loadingMore, setLoadingMore] = useState(false);
|
|
38
42
|
const selectedAccountId = useSelectedAccountId(meta?.accounts);
|
|
43
|
+
// Bumped whenever the active filters change. A `loadMore` request that
|
|
44
|
+
// resolves against a stale generation is discarded so its older page is
|
|
45
|
+
// never appended to a freshly loaded feed.
|
|
46
|
+
const generationRef = useRef(0);
|
|
47
|
+
const loadMoreControllerRef = useRef<AbortController | null>(null);
|
|
39
48
|
|
|
40
49
|
async function loadStatus() {
|
|
41
50
|
setMeta(await fetchQueryEnvelope());
|
|
@@ -45,10 +54,14 @@ export function useTimelineRouteData({
|
|
|
45
54
|
void loadStatus();
|
|
46
55
|
}, []);
|
|
47
56
|
|
|
48
|
-
|
|
57
|
+
// Build the /api/query URL for the current filters. Passing the last item's
|
|
58
|
+
// (createdAt, id) requests the next (older) page via the server's keyset
|
|
59
|
+
// cursor, which is deterministic across duplicate timestamps.
|
|
60
|
+
function buildQueryUrl(until?: string, untilId?: string) {
|
|
49
61
|
const url = new URL("/api/query", window.location.origin);
|
|
50
62
|
url.searchParams.set("resource", resource);
|
|
51
63
|
url.searchParams.set("refresh", String(refreshTick));
|
|
64
|
+
url.searchParams.set("limit", String(PAGE_SIZE));
|
|
52
65
|
if (selectedAccountId) {
|
|
53
66
|
url.searchParams.set("account", selectedAccountId);
|
|
54
67
|
}
|
|
@@ -64,16 +77,29 @@ export function useTimelineRouteData({
|
|
|
64
77
|
if (search.trim()) {
|
|
65
78
|
url.searchParams.set("search", search.trim());
|
|
66
79
|
}
|
|
80
|
+
if (until) {
|
|
81
|
+
url.searchParams.set("until", until);
|
|
82
|
+
}
|
|
83
|
+
if (untilId) {
|
|
84
|
+
url.searchParams.set("untilId", untilId);
|
|
85
|
+
}
|
|
86
|
+
return url;
|
|
87
|
+
}
|
|
67
88
|
|
|
89
|
+
useEffect(() => {
|
|
90
|
+
generationRef.current += 1;
|
|
91
|
+
loadMoreControllerRef.current?.abort();
|
|
68
92
|
const controller = new AbortController();
|
|
69
93
|
let active = true;
|
|
70
94
|
setError(null);
|
|
71
95
|
setLoading(true);
|
|
72
|
-
|
|
96
|
+
setLoadingMore(false);
|
|
97
|
+
fetchQueryResponse(buildQueryUrl(), { signal: controller.signal })
|
|
73
98
|
.then((data) => {
|
|
74
|
-
if (active)
|
|
75
|
-
|
|
76
|
-
|
|
99
|
+
if (!active) return;
|
|
100
|
+
const next = data.items as TimelineItem[];
|
|
101
|
+
setItems(next);
|
|
102
|
+
setHasMore(next.length >= PAGE_SIZE);
|
|
77
103
|
})
|
|
78
104
|
.catch((fetchError: unknown) => {
|
|
79
105
|
if (
|
|
@@ -87,6 +113,7 @@ export function useTimelineRouteData({
|
|
|
87
113
|
fetchError instanceof Error ? fetchError.message : errorFallback,
|
|
88
114
|
);
|
|
89
115
|
setItems([]);
|
|
116
|
+
setHasMore(false);
|
|
90
117
|
})
|
|
91
118
|
.finally(() => {
|
|
92
119
|
if (active) {
|
|
@@ -109,6 +136,44 @@ export function useTimelineRouteData({
|
|
|
109
136
|
selectedAccountId,
|
|
110
137
|
]);
|
|
111
138
|
|
|
139
|
+
async function loadMore() {
|
|
140
|
+
if (loading || loadingMore || !hasMore || items.length === 0) return;
|
|
141
|
+
const lastItem = items[items.length - 1];
|
|
142
|
+
const until = lastItem?.createdAt;
|
|
143
|
+
const untilId = lastItem?.id;
|
|
144
|
+
if (!until || !untilId) return;
|
|
145
|
+
const generation = generationRef.current;
|
|
146
|
+
const controller = new AbortController();
|
|
147
|
+
loadMoreControllerRef.current = controller;
|
|
148
|
+
setLoadingMore(true);
|
|
149
|
+
try {
|
|
150
|
+
const data = await fetchQueryResponse(buildQueryUrl(until, untilId), {
|
|
151
|
+
signal: controller.signal,
|
|
152
|
+
});
|
|
153
|
+
// Discard if the filters changed (new generation) while in flight.
|
|
154
|
+
if (generation !== generationRef.current) return;
|
|
155
|
+
const page = data.items as TimelineItem[];
|
|
156
|
+
setItems((prev) => {
|
|
157
|
+
const seen = new Set(prev.map((item) => item.id));
|
|
158
|
+
return [...prev, ...page.filter((item) => !seen.has(item.id))];
|
|
159
|
+
});
|
|
160
|
+
setHasMore(page.length >= PAGE_SIZE);
|
|
161
|
+
} catch (loadError) {
|
|
162
|
+
if (
|
|
163
|
+
loadError instanceof DOMException &&
|
|
164
|
+
loadError.name === "AbortError"
|
|
165
|
+
) {
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (generation !== generationRef.current) return;
|
|
169
|
+
setError(loadError instanceof Error ? loadError.message : errorFallback);
|
|
170
|
+
} finally {
|
|
171
|
+
if (generation === generationRef.current) {
|
|
172
|
+
setLoadingMore(false);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
112
177
|
function retry() {
|
|
113
178
|
setRefreshTick((value) => value + 1);
|
|
114
179
|
}
|
|
@@ -149,5 +214,8 @@ export function useTimelineRouteData({
|
|
|
149
214
|
refreshLocalView,
|
|
150
215
|
replyToTweet,
|
|
151
216
|
selectedAccountId,
|
|
217
|
+
hasMore,
|
|
218
|
+
loadingMore,
|
|
219
|
+
loadMore,
|
|
152
220
|
};
|
|
153
221
|
}
|
|
@@ -249,12 +249,22 @@ async function runStep({
|
|
|
249
249
|
}): Promise<AccountSyncAuditStep> {
|
|
250
250
|
try {
|
|
251
251
|
if (kind === "timeline") {
|
|
252
|
-
|
|
252
|
+
const timelineMode =
|
|
253
|
+
mode === "auto"
|
|
254
|
+
? account
|
|
255
|
+
? allowBirdAccount
|
|
256
|
+
? "bird"
|
|
257
|
+
: "xurl"
|
|
258
|
+
: "auto"
|
|
259
|
+
: mode;
|
|
260
|
+
if (timelineMode === "bird" && !allowBirdAccount) {
|
|
253
261
|
return { kind, ok: false, count: 0, error: birdAccountError(kind) };
|
|
254
262
|
}
|
|
255
263
|
const result = await syncHomeTimeline({
|
|
256
264
|
account,
|
|
265
|
+
mode: timelineMode,
|
|
257
266
|
limit,
|
|
267
|
+
maxPages,
|
|
258
268
|
following: true,
|
|
259
269
|
refresh,
|
|
260
270
|
cacheTtlMs,
|
|
@@ -267,12 +277,14 @@ async function runStep({
|
|
|
267
277
|
};
|
|
268
278
|
}
|
|
269
279
|
if (kind === "mentions") {
|
|
270
|
-
if (!allowBirdAccount) {
|
|
280
|
+
if (mode === "bird" && !allowBirdAccount) {
|
|
271
281
|
return { kind, ok: false, count: 0, error: birdAccountError(kind) };
|
|
272
282
|
}
|
|
283
|
+
const mentionMode =
|
|
284
|
+
mode === "auto" ? (allowBirdAccount ? "auto" : "xurl") : mode;
|
|
273
285
|
const result = await syncMentions({
|
|
274
286
|
account,
|
|
275
|
-
mode:
|
|
287
|
+
mode: mentionMode,
|
|
276
288
|
limit,
|
|
277
289
|
maxPages,
|
|
278
290
|
refresh,
|