birdclaw 0.8.1 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/package.json +2 -4
  3. package/scripts/browser-perf.mjs +27 -0
  4. package/src/cli/command-context.ts +17 -0
  5. package/src/cli/register-analysis.ts +500 -0
  6. package/src/cli/register-compose.ts +40 -0
  7. package/src/cli/register-graph.ts +132 -0
  8. package/src/cli/register-inbox.ts +41 -0
  9. package/src/cli/register-storage.ts +106 -0
  10. package/src/cli.ts +30 -750
  11. package/src/components/AccountSwitcher.tsx +7 -15
  12. package/src/components/AvatarChip.tsx +1 -1
  13. package/src/components/AvatarPreload.ts +149 -0
  14. package/src/components/ConversationThread.tsx +4 -0
  15. package/src/components/EmbeddedTweetCard.tsx +4 -0
  16. package/src/components/FloatingPreview.tsx +382 -0
  17. package/src/components/MarkdownCitations.tsx +680 -0
  18. package/src/components/MarkdownViewer.tsx +7 -715
  19. package/src/components/ProfileAnalysisClient.ts +191 -0
  20. package/src/components/ProfileAnalysisStream.tsx +16 -185
  21. package/src/components/ProfilePreview.tsx +41 -81
  22. package/src/components/TimelineCard.tsx +18 -2
  23. package/src/components/TweetArticleCard.tsx +66 -0
  24. package/src/components/TweetRichText.tsx +33 -2
  25. package/src/components/links-controller.ts +162 -0
  26. package/src/components/links-model.ts +198 -0
  27. package/src/components/network-map-controller.ts +84 -0
  28. package/src/components/network-map-model.ts +255 -0
  29. package/src/components/useDebouncedValue.ts +12 -0
  30. package/src/components/useTimelineRouteData.ts +142 -170
  31. package/src/lib/analysis-runtime.ts +238 -0
  32. package/src/lib/api-client.ts +16 -100
  33. package/src/lib/api-contracts.ts +328 -0
  34. package/src/lib/archive-finder.ts +38 -0
  35. package/src/lib/archive-import-plan.ts +102 -0
  36. package/src/lib/archive-import.ts +170 -239
  37. package/src/lib/authored-live.ts +77 -182
  38. package/src/lib/backup.ts +335 -424
  39. package/src/lib/bird.ts +32 -1
  40. package/src/lib/blocks-write.ts +30 -26
  41. package/src/lib/blocks.ts +18 -20
  42. package/src/lib/database-metrics.ts +88 -0
  43. package/src/lib/database-migrations.ts +34 -0
  44. package/src/lib/database-schema.ts +312 -0
  45. package/src/lib/database-writer.ts +69 -0
  46. package/src/lib/db.ts +141 -334
  47. package/src/lib/dm-read-model.ts +533 -0
  48. package/src/lib/dms-live.ts +34 -97
  49. package/src/lib/follow-graph.ts +17 -27
  50. package/src/lib/import-repository.ts +138 -0
  51. package/src/lib/inbox.ts +2 -1
  52. package/src/lib/live-sync-engine.ts +209 -0
  53. package/src/lib/live-transport-gateway.ts +128 -0
  54. package/src/lib/mention-threads-live.ts +90 -176
  55. package/src/lib/mentions-export.ts +1 -1
  56. package/src/lib/mentions-live.ts +57 -225
  57. package/src/lib/moderation-target.ts +15 -4
  58. package/src/lib/moderation-write.ts +1 -1
  59. package/src/lib/mutes-write.ts +30 -26
  60. package/src/lib/openai-response-runtime.ts +251 -0
  61. package/src/lib/paginated-sync.ts +93 -0
  62. package/src/lib/period-digest.ts +116 -304
  63. package/src/lib/profile-analysis.ts +37 -111
  64. package/src/lib/queries.ts +6 -2380
  65. package/src/lib/query-actions.ts +437 -0
  66. package/src/lib/query-client.tsx +47 -0
  67. package/src/lib/query-read-model-shared.ts +52 -0
  68. package/src/lib/query-read-models.ts +5 -0
  69. package/src/lib/query-resource.ts +41 -0
  70. package/src/lib/query-status.ts +164 -0
  71. package/src/lib/research.ts +1 -1
  72. package/src/lib/runtime-services.ts +20 -0
  73. package/src/lib/search-discussion.ts +75 -279
  74. package/src/lib/server-runtime-services.ts +30 -0
  75. package/src/lib/sqlite.ts +53 -14
  76. package/src/lib/streaming-ingestion.ts +240 -0
  77. package/src/lib/sync-cache.ts +6 -1
  78. package/src/lib/sync-plan.ts +175 -0
  79. package/src/lib/timeline-collections-live.ts +83 -256
  80. package/src/lib/timeline-live.ts +86 -235
  81. package/src/lib/timeline-read-model.ts +1191 -0
  82. package/src/lib/tweet-render.ts +78 -0
  83. package/src/lib/tweet-repository.ts +156 -0
  84. package/src/lib/tweet-search-live.ts +63 -166
  85. package/src/lib/types.ts +8 -0
  86. package/src/lib/ui.ts +1 -1
  87. package/src/lib/web-sync.ts +67 -50
  88. package/src/lib/whois.ts +2 -1
  89. package/src/router.tsx +1 -1
  90. package/src/routes/__root.tsx +11 -21
  91. package/src/routes/api/action.tsx +1 -1
  92. package/src/routes/api/conversation.tsx +1 -1
  93. package/src/routes/api/query.tsx +32 -26
  94. package/src/routes/api/status.tsx +6 -2
  95. package/src/routes/api/sync.tsx +5 -2
  96. package/src/routes/blocks.tsx +97 -131
  97. package/src/routes/data-sources.tsx +17 -25
  98. package/src/routes/dms.tsx +168 -167
  99. package/src/routes/inbox.tsx +63 -57
  100. package/src/routes/links.tsx +31 -329
  101. package/src/routes/network-map.tsx +41 -344
  102. package/src/routes/rate-limits.tsx +17 -21
  103. package/vite.config.ts +0 -2
@@ -9,7 +9,7 @@ import {
9
9
  Search,
10
10
  Users,
11
11
  } from "lucide-react";
12
- import { useCallback, useEffect, useMemo, useRef, useState } from "react";
12
+ import { useState } from "react";
13
13
  import { AvatarChip } from "#/components/AvatarChip";
14
14
  import {
15
15
  FeedEmpty,
@@ -19,17 +19,12 @@ import {
19
19
  } from "#/components/FeedState";
20
20
  import { ProfilePreview } from "#/components/ProfilePreview";
21
21
  import { SmartTimestamp } from "#/components/SmartTimestamp";
22
+ import { useLinksController } from "#/components/links-controller";
22
23
  import { formatCompactNumber } from "#/lib/present";
23
24
  import type {
24
25
  LinkInsightItem,
25
- LinkInsightKind,
26
26
  LinkInsightMention,
27
- LinkInsightRange,
28
- LinkInsightResponse,
29
- LinkInsightSort,
30
- LinkInsightSource,
31
27
  ProfileRecord,
32
- TweetMediaItem,
33
28
  } from "#/lib/types";
34
29
  import {
35
30
  cx,
@@ -50,170 +45,20 @@ export const Route = createFileRoute("/links")({
50
45
  component: LinksRoute,
51
46
  });
52
47
 
53
- const INITIAL_VISIBLE_COMMENTS = 3;
54
- const MORE_COMMENTS_BATCH = 6;
55
- const LINK_INSIGHTS_LIMIT = 30;
56
- const LINK_INSIGHTS_COMMENTS_LIMIT = 30;
57
- const PROFILE_HYDRATION_LIMIT = 30;
58
- const PROFILE_HYDRATION_DELAY_MS = 1200;
59
-
60
- const ranges: Array<{ value: LinkInsightRange; label: string }> = [
61
- { value: "today", label: "Today" },
62
- { value: "week", label: "Week" },
63
- { value: "month", label: "Month" },
64
- { value: "year", label: "Year" },
65
- { value: "all", label: "All" },
66
- ];
67
-
68
- function itemTitle(item: LinkInsightItem) {
69
- return item.title?.trim() || item.displayUrl;
70
- }
71
-
72
- function itemSubtitle(item: LinkInsightItem) {
73
- const description = item.description?.trim();
74
- if (description) {
75
- return description;
76
- }
77
- return item.displayUrl.split("?")[0] || item.displayUrl;
78
- }
79
-
80
- function insightCacheKey(
81
- kind: LinkInsightKind,
82
- range: LinkInsightRange,
83
- sort: LinkInsightSort,
84
- source: LinkInsightSource,
85
- refreshTick: number,
86
- ) {
87
- return `${kind}:${range}:${sort}:${source}:${refreshTick}`;
88
- }
89
-
90
- function linkInsightsUrl(
91
- kind: LinkInsightKind,
92
- range: LinkInsightRange,
93
- sort: LinkInsightSort,
94
- source: LinkInsightSource,
95
- refreshTick: number,
96
- ) {
97
- const url = new URL("/api/link-insights", window.location.origin);
98
- url.searchParams.set("kind", kind);
99
- url.searchParams.set("range", range);
100
- url.searchParams.set("sort", sort);
101
- url.searchParams.set("source", source);
102
- url.searchParams.set("limit", String(LINK_INSIGHTS_LIMIT));
103
- url.searchParams.set("commentsLimit", String(LINK_INSIGHTS_COMMENTS_LIMIT));
104
- url.searchParams.set("refresh", String(refreshTick));
105
- return url;
106
- }
107
-
108
- function mentionHref(mention: LinkInsightMention, item: LinkInsightItem) {
109
- return mention.sourceUrl || mention.contentTweetUrl || item.url;
110
- }
111
-
112
- function mentionCopy(mention: LinkInsightMention) {
113
- return (
114
- mention.commentText ||
115
- mention.sharedContentText ||
116
- mention.rawText ||
117
- "Shared without comment"
118
- );
119
- }
120
-
121
- function isSameProfile(
122
- left: ProfileRecord | null | undefined,
123
- right: ProfileRecord | null | undefined,
124
- ) {
125
- return Boolean(left && right && left.id === right.id);
126
- }
127
-
128
- function mediaImage(media: TweetMediaItem[]) {
129
- return media.find((item) => item.thumbnailUrl || item.url) ?? null;
130
- }
131
-
132
- function youtubeVideoId(rawUrl: string) {
133
- try {
134
- const url = new URL(rawUrl);
135
- const host = url.hostname.toLowerCase().replace(/^www\./, "");
136
- if (host === "youtu.be") {
137
- return url.pathname.split("/").filter(Boolean)[0] ?? null;
138
- }
139
- if (!host.endsWith("youtube.com")) {
140
- return null;
141
- }
142
- if (url.pathname === "/watch") {
143
- return url.searchParams.get("v");
144
- }
145
- const parts = url.pathname.split("/").filter(Boolean);
146
- if (parts[0] === "embed" || parts[0] === "shorts" || parts[0] === "live") {
147
- return parts[1] ?? null;
148
- }
149
- return null;
150
- } catch {
151
- return null;
152
- }
153
- }
154
-
155
- function youtubeThumbnailUrl(rawUrl: string) {
156
- const id = youtubeVideoId(rawUrl);
157
- if (!id || !/^[\w-]{6,}$/.test(id)) {
158
- return null;
159
- }
160
- return `https://i.ytimg.com/vi/${encodeURIComponent(id)}/hqdefault.jpg`;
161
- }
162
-
163
- function commentCount(item: LinkInsightItem) {
164
- return (
165
- item.commentCount ??
166
- item.mentions.filter((mention) => mention.hasComment).length
167
- );
168
- }
169
-
170
- function pureShareCount(item: LinkInsightItem) {
171
- return (
172
- item.pureShareCount ??
173
- item.mentions.filter(
174
- (mention) => mention.isPureShare || !mention.hasComment,
175
- ).length
176
- );
177
- }
178
-
179
- function isArchivePlaceholderProfile(profile: ProfileRecord) {
180
- return (
181
- /^id\d+$/i.test(profile.handle) &&
182
- profile.displayName === profile.handle &&
183
- profile.id === `profile_user_${profile.handle.slice(2)}`
184
- );
185
- }
186
-
187
- function profileNeedsHydration(profile: ProfileRecord | null | undefined) {
188
- if (!profile?.handle || isArchivePlaceholderProfile(profile)) {
189
- return false;
190
- }
191
- return !profile.avatarUrl || profile.followersCount === 0;
192
- }
193
-
194
- function collectProfilesForHydration(data: LinkInsightResponse | null) {
195
- const handles = new Set<string>();
196
- for (const item of data?.items ?? []) {
197
- for (const profile of [
198
- item.topSharer,
199
- ...item.sharers,
200
- ...item.mentions.flatMap((mention) => [
201
- mention.sharedBy,
202
- mention.contentAuthor,
203
- mention.participant,
204
- ]),
205
- ]) {
206
- if (!profile || !profileNeedsHydration(profile)) {
207
- continue;
208
- }
209
- handles.add(profile.handle.replace(/^@/, ""));
210
- if (handles.size >= PROFILE_HYDRATION_LIMIT) {
211
- return [...handles];
212
- }
213
- }
214
- }
215
- return [...handles];
216
- }
48
+ import {
49
+ INITIAL_VISIBLE_COMMENTS,
50
+ MORE_COMMENTS_BATCH,
51
+ commentCount,
52
+ isSameProfile,
53
+ itemSubtitle,
54
+ itemTitle,
55
+ mediaImage,
56
+ mentionCopy,
57
+ mentionHref,
58
+ pureShareCount,
59
+ ranges,
60
+ youtubeThumbnailUrl,
61
+ } from "#/components/links-model";
217
62
 
218
63
  function ProfilePill({
219
64
  profile,
@@ -645,166 +490,23 @@ function LinkInsightRow({
645
490
  }
646
491
 
647
492
  function LinksRoute() {
648
- const [kind, setKind] = useState<LinkInsightKind>("links");
649
- const [range, setRange] = useState<LinkInsightRange>("week");
650
- const [source, setSource] = useState<LinkInsightSource>("all");
651
- const [sort, setSort] = useState<LinkInsightSort>("rank");
652
- const [search, setSearch] = useState("");
653
- const [refreshTick, setRefreshTick] = useState(0);
654
- const hydratedHandlesRef = useRef(new Set<string>());
655
- const cacheRef = useRef(new Map<string, LinkInsightResponse>());
656
- const inFlightRef = useRef(new Set<string>());
657
- const mountedRef = useRef(true);
658
- const [errorByKey, setErrorByKey] = useState<Record<string, string>>({});
659
- const [, bumpCacheVersion] = useState(0);
660
- const currentCacheKey = insightCacheKey(
493
+ const {
661
494
  kind,
495
+ setKind,
662
496
  range,
663
- sort,
497
+ setRange,
664
498
  source,
665
- refreshTick,
666
- );
667
- const data = cacheRef.current.get(currentCacheKey) ?? null;
668
- const error = errorByKey[currentCacheKey] ?? null;
669
- const loading = !data && !error;
670
-
671
- useEffect(() => {
672
- return () => {
673
- mountedRef.current = false;
674
- };
675
- }, []);
676
-
677
- const fetchInsights = useCallback(
678
- (fetchKind: LinkInsightKind) => {
679
- const key = insightCacheKey(fetchKind, range, sort, source, refreshTick);
680
- if (cacheRef.current.has(key) || inFlightRef.current.has(key)) {
681
- return;
682
- }
683
- inFlightRef.current.add(key);
684
- setErrorByKey((current) => {
685
- const rest = { ...current };
686
- delete rest[key];
687
- return rest;
688
- });
689
- fetch(linkInsightsUrl(fetchKind, range, sort, source, refreshTick))
690
- .then((response) => response.json())
691
- .then((response: LinkInsightResponse) => {
692
- cacheRef.current.set(key, response);
693
- if (mountedRef.current) {
694
- bumpCacheVersion((value) => value + 1);
695
- }
696
- })
697
- .catch((error: unknown) => {
698
- if (error instanceof DOMException && error.name === "AbortError") {
699
- return;
700
- }
701
- if (!mountedRef.current) {
702
- return;
703
- }
704
- console.warn("Link insights failed", error);
705
- setErrorByKey((current) => ({
706
- ...current,
707
- [key]:
708
- error instanceof Error
709
- ? error.message
710
- : "Link insights unavailable",
711
- }));
712
- })
713
- .finally(() => {
714
- inFlightRef.current.delete(key);
715
- });
716
- },
717
- [range, refreshTick, sort, source],
718
- );
719
-
720
- useEffect(() => {
721
- fetchInsights(kind);
722
- }, [fetchInsights, kind]);
723
-
724
- useEffect(() => {
725
- if (!data) {
726
- return;
727
- }
728
- const prefetchKind = kind === "links" ? "videos" : "links";
729
- const timer = window.setTimeout(() => fetchInsights(prefetchKind), 250);
730
- return () => window.clearTimeout(timer);
731
- }, [data, fetchInsights, kind]);
732
-
733
- useEffect(() => {
734
- const handles = collectProfilesForHydration(data).filter(
735
- (handle) => !hydratedHandlesRef.current.has(handle.toLowerCase()),
736
- );
737
- if (handles.length === 0) {
738
- return;
739
- }
740
-
741
- const controller = new AbortController();
742
- const url = new URL("/api/profile-hydrate", window.location.origin);
743
- url.searchParams.set("handles", handles.join(","));
744
- for (const handle of handles) {
745
- hydratedHandlesRef.current.add(handle.toLowerCase());
746
- }
747
-
748
- let idleId: number | null = null;
749
- const runHydration = () => {
750
- fetch(url, { signal: controller.signal })
751
- .then((response) => response.json())
752
- .then((response: { hydratedProfiles?: number }) => {
753
- if ((response.hydratedProfiles ?? 0) > 0) {
754
- setRefreshTick((value) => value + 1);
755
- }
756
- })
757
- .catch((error: unknown) => {
758
- if (error instanceof DOMException && error.name === "AbortError") {
759
- return;
760
- }
761
- console.warn("Profile hydration failed", error);
762
- });
763
- };
764
- const timer = window.setTimeout(() => {
765
- if ("requestIdleCallback" in window) {
766
- idleId = window.requestIdleCallback(runHydration, { timeout: 2500 });
767
- } else {
768
- runHydration();
769
- }
770
- }, PROFILE_HYDRATION_DELAY_MS);
771
-
772
- return () => {
773
- controller.abort();
774
- window.clearTimeout(timer);
775
- if (idleId !== null && "cancelIdleCallback" in window) {
776
- window.cancelIdleCallback(idleId);
777
- }
778
- };
779
- }, [data]);
780
-
781
- const items = useMemo(() => {
782
- const query = search.trim().toLowerCase();
783
- const filtered = (data?.items ?? []).filter((item) => {
784
- if (!query) {
785
- return true;
786
- }
787
- return [
788
- item.title,
789
- item.description,
790
- item.displayUrl,
791
- item.host,
792
- item.topSharer?.handle,
793
- ...item.mentions.map((mention) => mention.commentText),
794
- ]
795
- .filter(Boolean)
796
- .some((value) => String(value).toLowerCase().includes(query));
797
- });
798
- return filtered;
799
- }, [data?.items, search]);
800
-
801
- const subtitle = useMemo(() => {
802
- if (!data) {
803
- return "Loading link memory...";
804
- }
805
- const label = kind === "videos" ? "video URLs" : "URLs";
806
- return `${formatCompactNumber(data.stats.occurrences)} ${label} across ${formatCompactNumber(data.stats.groups)} groups`;
807
- }, [data, kind]);
499
+ setSource,
500
+ sort,
501
+ setSort,
502
+ search,
503
+ setSearch,
504
+ items,
505
+ subtitle,
506
+ loading,
507
+ error,
508
+ retry,
509
+ } = useLinksController();
808
510
 
809
511
  return (
810
512
  <>
@@ -901,7 +603,7 @@ function LinksRoute() {
901
603
  action={
902
604
  <button
903
605
  className="rounded-full bg-[var(--accent)] px-4 py-1.5 text-[14px] font-bold text-white"
904
- onClick={() => setRefreshTick((value) => value + 1)}
606
+ onClick={() => void retry()}
905
607
  type="button"
906
608
  >
907
609
  Retry