birdclaw 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +10 -8
- package/bin/birdclaw.mjs +4 -66
- package/dist/cli/birdclaw.js +28165 -0
- package/dist/client/assets/AvatarChip-DLoqM5xD.js +1 -0
- package/dist/client/assets/ProfilePreview-BejAuYsJ.js +1 -0
- package/dist/client/assets/SavedTimelineView-BIS7HKpp.js +1 -0
- package/dist/client/assets/SmartTimestamp-BR0-9se2.js +1 -0
- package/dist/client/assets/TimelineRouteFrame-CyGPj0eM.js +1 -0
- package/dist/client/assets/blocks-7UDKQp8H.js +1 -0
- package/dist/client/assets/bookmarks-CbBkMvXk.js +1 -0
- package/dist/client/assets/chunk-C0phanGV.js +1 -0
- package/dist/client/assets/circle-check-C_Vq3ElD.js +1 -0
- package/dist/client/assets/client-stream-contracts-ys3XhWG4.js +5 -0
- package/dist/client/assets/createLucideIcon-JV8isFFR.js +1 -0
- package/dist/client/assets/data-sources-Cm2bpNvg.js +1 -0
- package/dist/client/assets/discuss-DTjaOugw.js +1 -0
- package/dist/client/assets/dms-CGd6Eogq.js +1 -0
- package/dist/client/assets/inbox-BHlGuOy2.js +1 -0
- package/dist/client/assets/index-CEIG7S6s.js +31 -0
- package/dist/client/assets/index-bpFrQ5yG.css +1 -0
- package/dist/client/assets/likes-cLrcAQem.js +1 -0
- package/dist/client/assets/links-CM9QO1LC.js +1 -0
- package/dist/client/assets/mapbox-BxTlqXt9.js +2 -0
- package/dist/client/assets/mapbox-gl-dgf_hX3a.js +3215 -0
- package/dist/client/assets/mentions-CxKCgzIf.js +1 -0
- package/dist/client/assets/network-map-B5a3whqj.js +2 -0
- package/dist/client/assets/profile-analyze-CRtNEwlb.js +1 -0
- package/dist/client/assets/profile-hydration-client-YEtJsNTv.js +1 -0
- package/dist/client/assets/profiles._handle-C82_HoKX.js +1 -0
- package/dist/client/assets/query-client-fYu2rzRC.js +17 -0
- package/dist/client/assets/rate-limits-ScaEwdVK.js +1 -0
- package/dist/client/assets/react-dom-DHl6Ug7F.js +1 -0
- package/dist/client/assets/react-nBylq1U5.js +1 -0
- package/dist/client/assets/repeat-2-xCwHM9hD.js +1 -0
- package/dist/client/assets/routes-q7ZaW4En.js +1 -0
- package/dist/client/assets/schemas-I9r6F6ya.js +64 -0
- package/dist/client/assets/search-Dq5kYsxA.js +1 -0
- package/dist/client/assets/sparkles-CQVlvHsZ.js +1 -0
- package/dist/client/assets/styles-B8s3tt3r.css +2 -0
- package/dist/client/assets/today-CeG1O7Py.js +1 -0
- package/dist/client/assets/triangle-alert-CBL7s2pf.js +1 -0
- package/dist/client/assets/useTimelineRouteData-1HsqgvUi.js +1 -0
- package/dist/client/assets/users-CHbKlqUr.js +1 -0
- package/dist/server/assets/AvatarChip-zUDAEmjL.js +105 -0
- package/dist/server/assets/FeedState-ntW0Q2GB.js +214 -0
- package/dist/server/assets/ProfileAnalysisStream-BFvhVdhn.js +198 -0
- package/dist/server/assets/ProfilePreview-C6gAor_j.js +752 -0
- package/dist/server/assets/SavedTimelineView-BG1jnBp1.js +64 -0
- package/dist/server/assets/SmartTimestamp-DfwE1NGf.js +45 -0
- package/dist/server/assets/SyncNowButton-BOn_IBXS.js +76 -0
- package/dist/server/assets/TimelineRouteFrame-CU7ilM7U.js +85 -0
- package/dist/server/assets/TweetRichText-DhCVv1-6.js +74 -0
- package/dist/server/assets/_tanstack-start-manifest_v-C-ZRnG_n.js +148 -0
- package/dist/server/assets/account-selection-CTdFGWRk.js +44 -0
- package/dist/server/assets/blocks-C1mgGTB3.js +299 -0
- package/dist/server/assets/blocks-DSgQghqP.js +17 -0
- package/dist/server/assets/bookmarks-BJN8tykY.js +14 -0
- package/dist/server/assets/client-stream-contracts-C2xbWVjm.js +769 -0
- package/dist/server/assets/data-sources-BnA1Q94p.js +180 -0
- package/dist/server/assets/discuss-CJ1dau1j.js +15 -0
- package/dist/server/assets/discuss-QTo08NCN.js +338 -0
- package/dist/server/assets/dms-BuLj4bp8.js +368 -0
- package/dist/server/assets/dms-DAyQrVM2.js +240 -0
- package/dist/server/assets/empty-plugin-adapters-DzrEkdBF.js +5 -0
- package/dist/server/assets/inbox-CkGOaW1q.js +214 -0
- package/dist/server/assets/inbox-DpWd1EzC.js +146 -0
- package/dist/server/assets/likes-DBOAtsJU.js +14 -0
- package/dist/server/assets/links-DAhSjynu.js +240 -0
- package/dist/server/assets/links-DmH5lxHO.js +468 -0
- package/dist/server/assets/mentions-BH2MzKtB.js +26 -0
- package/dist/server/assets/network-map-D3CzRcC4.js +224 -0
- package/dist/server/assets/network-map-D5DoLOwR.js +582 -0
- package/dist/server/assets/profile-analyze-BP65PED0.js +90 -0
- package/dist/server/assets/profile-analyze-C1NoM0tV.js +9 -0
- package/dist/server/assets/profile-hydration-client-GJ4kwJS8.js +133 -0
- package/dist/server/assets/profiles._handle-DjwlN_ac.js +184 -0
- package/dist/server/assets/profiles._handle-DsLNj2Ca.js +12 -0
- package/dist/server/assets/query-client-BdauHBgd.js +923 -0
- package/dist/server/assets/rate-limits-VAr2qzYD.js +262 -0
- package/dist/server/assets/route-search-BUlQ5lGM.js +117 -0
- package/dist/server/assets/router-DqkwSPiM.js +17026 -0
- package/dist/server/assets/routes-A9c9ZhwN.js +26 -0
- package/dist/server/assets/start-FaDOTmto.js +4 -0
- package/dist/server/assets/today-B5hVM57Z.js +17 -0
- package/dist/server/assets/today-fvGiZ7ud.js +303 -0
- package/dist/server/assets/ui-qbmcMSSX.js +137 -0
- package/{src/components/useDebouncedValue.ts → dist/server/assets/useDebouncedValue-DOnA6Xcy.js} +4 -4
- package/dist/server/assets/useTimelineRouteData-27cxynxa.js +1101 -0
- package/dist/server/server.js +5520 -0
- package/package.json +27 -34
- package/playwright.config.ts +0 -35
- package/scripts/browser-perf.mjs +0 -427
- package/scripts/build-docs-site.mjs +0 -967
- package/scripts/docs-site-assets.mjs +0 -311
- package/scripts/run-vitest.mjs +0 -21
- package/scripts/sanitize-node-options.mjs +0 -23
- package/scripts/start-test-server.mjs +0 -42
- package/src/cli-moderation.ts +0 -210
- package/src/cli.ts +0 -2336
- package/src/components/AccountSwitcher.tsx +0 -186
- package/src/components/AppNav.tsx +0 -118
- package/src/components/AvatarChip.tsx +0 -58
- package/src/components/BrandMark.tsx +0 -67
- package/src/components/ConversationThread.tsx +0 -131
- package/src/components/DmWorkspace.tsx +0 -286
- package/src/components/EmbeddedTweetCard.tsx +0 -55
- package/src/components/FeedState.tsx +0 -147
- package/src/components/FloatingPreview.tsx +0 -382
- package/src/components/InboxCard.tsx +0 -152
- package/src/components/LinkPreviewCard.tsx +0 -292
- package/src/components/MarkdownViewer.tsx +0 -777
- package/src/components/ProfileAnalysisStream.tsx +0 -428
- package/src/components/ProfilePreview.tsx +0 -129
- package/src/components/SavedTimelineView.tsx +0 -155
- package/src/components/SmartTimestamp.tsx +0 -72
- package/src/components/SyncNowButton.tsx +0 -140
- package/src/components/ThemeSlider.tsx +0 -80
- package/src/components/TimelineCard.tsx +0 -533
- package/src/components/TimelineRouteFrame.tsx +0 -186
- package/src/components/TweetArticleCard.tsx +0 -66
- package/src/components/TweetMediaGrid.tsx +0 -185
- package/src/components/TweetRichText.tsx +0 -159
- package/src/components/account-selection.ts +0 -64
- package/src/components/useTimelineRouteData.ts +0 -323
- package/src/lib/account-sync-job.ts +0 -666
- package/src/lib/actions-transport.ts +0 -300
- package/src/lib/api-client.ts +0 -419
- package/src/lib/archive-finder.ts +0 -185
- package/src/lib/archive-import.ts +0 -2764
- package/src/lib/authored-live.ts +0 -1072
- package/src/lib/avatar-cache.ts +0 -349
- package/src/lib/backup.ts +0 -2741
- package/src/lib/bird-actions.ts +0 -294
- package/src/lib/bird-command.ts +0 -130
- package/src/lib/bird.ts +0 -1140
- package/src/lib/blocklist.ts +0 -136
- package/src/lib/blocks-write.ts +0 -167
- package/src/lib/blocks.ts +0 -389
- package/src/lib/bookmark-sync-job.ts +0 -536
- package/src/lib/client-cache.ts +0 -109
- package/src/lib/config.ts +0 -224
- package/src/lib/conversation-surface.ts +0 -205
- package/src/lib/data-sources.ts +0 -219
- package/src/lib/db.ts +0 -1135
- package/src/lib/dms-live.ts +0 -933
- package/src/lib/effect-runtime.ts +0 -45
- package/src/lib/follow-graph.ts +0 -1097
- package/src/lib/geocoding.ts +0 -296
- package/src/lib/http-effect.ts +0 -222
- package/src/lib/identity-search-index.ts +0 -369
- package/src/lib/inbox.ts +0 -241
- package/src/lib/link-index.ts +0 -641
- package/src/lib/link-insights.ts +0 -860
- package/src/lib/link-preview-metadata.ts +0 -754
- package/src/lib/location.ts +0 -137
- package/src/lib/media-fetch.ts +0 -860
- package/src/lib/media-includes.ts +0 -165
- package/src/lib/mention-threads-live.ts +0 -917
- package/src/lib/mentions-export.ts +0 -147
- package/src/lib/mentions-live.ts +0 -1238
- package/src/lib/moderation-target.ts +0 -216
- package/src/lib/moderation-write.ts +0 -131
- package/src/lib/mutes-write.ts +0 -167
- package/src/lib/mutes.ts +0 -84
- package/src/lib/ndjson-stream.ts +0 -106
- package/src/lib/network-map.ts +0 -382
- package/src/lib/openai.ts +0 -117
- package/src/lib/period-digest.ts +0 -1552
- package/src/lib/present.ts +0 -118
- package/src/lib/profile-affiliation-hydration.ts +0 -228
- package/src/lib/profile-affiliations.ts +0 -262
- package/src/lib/profile-analysis.ts +0 -1272
- package/src/lib/profile-bio-entities.ts +0 -318
- package/src/lib/profile-history.ts +0 -164
- package/src/lib/profile-hydration.ts +0 -180
- package/src/lib/profile-replies.ts +0 -77
- package/src/lib/profile-resolver.ts +0 -650
- package/src/lib/queries.ts +0 -2381
- package/src/lib/research.ts +0 -652
- package/src/lib/search-discussion.ts +0 -1016
- package/src/lib/seed.ts +0 -613
- package/src/lib/sqlite.ts +0 -147
- package/src/lib/sync-cache.ts +0 -65
- package/src/lib/theme-transition.ts +0 -113
- package/src/lib/theme.tsx +0 -165
- package/src/lib/timeline-collections-live.ts +0 -581
- package/src/lib/timeline-live.ts +0 -452
- package/src/lib/tweet-account-edges.ts +0 -48
- package/src/lib/tweet-lookup.ts +0 -46
- package/src/lib/tweet-render.ts +0 -430
- package/src/lib/tweet-search-live.ts +0 -566
- package/src/lib/types.ts +0 -723
- package/src/lib/ui.ts +0 -529
- package/src/lib/url-expansion-store.ts +0 -110
- package/src/lib/url-expansion.ts +0 -343
- package/src/lib/url-safety.ts +0 -220
- package/src/lib/web-sync.ts +0 -516
- package/src/lib/whois.ts +0 -1079
- package/src/lib/x-profile.ts +0 -365
- package/src/lib/x-web.ts +0 -199
- package/src/lib/xurl-rate-limits.ts +0 -267
- package/src/lib/xurl.ts +0 -1677
- package/src/routeTree.gen.ts +0 -723
- package/src/router.tsx +0 -20
- package/src/routes/__root.tsx +0 -79
- package/src/routes/api/action.tsx +0 -147
- package/src/routes/api/avatar.tsx +0 -50
- package/src/routes/api/blocks.tsx +0 -35
- package/src/routes/api/conversation.tsx +0 -45
- package/src/routes/api/data-sources.tsx +0 -24
- package/src/routes/api/inbox.tsx +0 -43
- package/src/routes/api/link-insights.tsx +0 -85
- package/src/routes/api/link-preview.tsx +0 -47
- package/src/routes/api/network-map.tsx +0 -55
- package/src/routes/api/period-digest.tsx +0 -101
- package/src/routes/api/profile-analysis.tsx +0 -97
- package/src/routes/api/profile-hydrate.tsx +0 -54
- package/src/routes/api/query.tsx +0 -115
- package/src/routes/api/search-discussion.tsx +0 -113
- package/src/routes/api/status.tsx +0 -28
- package/src/routes/api/sync.tsx +0 -105
- package/src/routes/api/xurl-rate-limits.tsx +0 -24
- package/src/routes/blocks.tsx +0 -377
- package/src/routes/bookmarks.tsx +0 -18
- package/src/routes/data-sources.tsx +0 -257
- package/src/routes/discuss.tsx +0 -419
- package/src/routes/dms.tsx +0 -432
- package/src/routes/inbox.tsx +0 -226
- package/src/routes/index.tsx +0 -32
- package/src/routes/likes.tsx +0 -18
- package/src/routes/links.tsx +0 -994
- package/src/routes/mentions.tsx +0 -32
- package/src/routes/network-map.tsx +0 -1035
- package/src/routes/profile-analyze.tsx +0 -112
- package/src/routes/profiles.$handle.tsx +0 -228
- package/src/routes/rate-limits.tsx +0 -309
- package/src/routes/today.tsx +0 -511
- package/src/styles.css +0 -257
- package/tsconfig.json +0 -29
- package/vite.config.ts +0 -29
- /package/{public → dist/client}/birdclaw-mark.png +0 -0
- /package/{public → dist/client}/favicon.ico +0 -0
- /package/{public → dist/client}/logo192.png +0 -0
- /package/{public → dist/client}/logo512.png +0 -0
- /package/{public → dist/client}/manifest.json +0 -0
- /package/{public → dist/client}/robots.txt +0 -0
package/src/lib/follow-graph.ts
DELETED
|
@@ -1,1097 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { Effect } from "effect";
|
|
3
|
-
import { getNativeDb } from "./db";
|
|
4
|
-
import { listFollowUsersViaBirdEffect } from "./bird";
|
|
5
|
-
import { runEffectPromise, tryPromise } from "./effect-runtime";
|
|
6
|
-
import type { Database } from "./sqlite";
|
|
7
|
-
import { readSyncCache, writeSyncCache } from "./sync-cache";
|
|
8
|
-
import type {
|
|
9
|
-
FollowEventKind,
|
|
10
|
-
FollowDirection,
|
|
11
|
-
FollowGraphEvent,
|
|
12
|
-
FollowGraphProfile,
|
|
13
|
-
FollowGraphSummary,
|
|
14
|
-
XurlFollowUsersResponse,
|
|
15
|
-
XurlMentionUser,
|
|
16
|
-
XurlPublicMetrics,
|
|
17
|
-
} from "./types";
|
|
18
|
-
import { upsertProfileFromXUser } from "./x-profile";
|
|
19
|
-
import { listFollowUsersViaXurl } from "./xurl";
|
|
20
|
-
|
|
21
|
-
const DEFAULT_FOLLOW_CACHE_TTL_MS = 24 * 60 * 60_000;
|
|
22
|
-
const DEFAULT_FOLLOW_PAGE_LIMIT = 1000;
|
|
23
|
-
const MIN_FOLLOW_PAGE_LIMIT = 1;
|
|
24
|
-
const MAX_FOLLOW_PAGE_LIMIT = 1000;
|
|
25
|
-
const BIRD_FOLLOW_PAGE_LIMIT = 100;
|
|
26
|
-
|
|
27
|
-
export interface SyncFollowGraphOptions {
|
|
28
|
-
direction: FollowDirection;
|
|
29
|
-
account?: string;
|
|
30
|
-
mode?: FollowGraphSyncMode;
|
|
31
|
-
limit?: number;
|
|
32
|
-
maxPages?: number;
|
|
33
|
-
maxResources?: number;
|
|
34
|
-
yes?: boolean;
|
|
35
|
-
refresh?: boolean;
|
|
36
|
-
allowPartial?: boolean;
|
|
37
|
-
cacheTtlMs?: number;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
type FollowGraphSyncMode = "auto" | "bird" | "xurl";
|
|
41
|
-
type FollowGraphLiveSource = "bird" | "xurl";
|
|
42
|
-
|
|
43
|
-
function toError(error: unknown) {
|
|
44
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function trySync<T>(try_: () => T) {
|
|
48
|
-
return Effect.try({
|
|
49
|
-
try: try_,
|
|
50
|
-
catch: toError,
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
interface ResolvedAccount {
|
|
55
|
-
accountId: string;
|
|
56
|
-
username: string;
|
|
57
|
-
externalUserId?: string;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
interface MergedFollowPayload {
|
|
61
|
-
data: XurlMentionUser[];
|
|
62
|
-
meta: Record<string, unknown>;
|
|
63
|
-
complete: boolean;
|
|
64
|
-
pageCount: number;
|
|
65
|
-
truncatedByMaxResources: boolean;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function parseLimit(value = DEFAULT_FOLLOW_PAGE_LIMIT) {
|
|
69
|
-
if (
|
|
70
|
-
!Number.isFinite(value) ||
|
|
71
|
-
value < MIN_FOLLOW_PAGE_LIMIT ||
|
|
72
|
-
value > MAX_FOLLOW_PAGE_LIMIT
|
|
73
|
-
) {
|
|
74
|
-
throw new Error("--limit must be between 1 and 1000 for follow sync");
|
|
75
|
-
}
|
|
76
|
-
return Math.floor(value);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function parseOptionalPositiveInteger(name: string, value?: number) {
|
|
80
|
-
if (value === undefined) {
|
|
81
|
-
return undefined;
|
|
82
|
-
}
|
|
83
|
-
if (!Number.isFinite(value) || value < 1) {
|
|
84
|
-
throw new Error(`${name} must be at least 1`);
|
|
85
|
-
}
|
|
86
|
-
return Math.floor(value);
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
function parseCacheTtlMs(value?: number) {
|
|
90
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
91
|
-
return DEFAULT_FOLLOW_CACHE_TTL_MS;
|
|
92
|
-
}
|
|
93
|
-
return Math.floor(value);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function parseJsonField<T>(value: unknown, fallback: T): T {
|
|
97
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
98
|
-
return fallback;
|
|
99
|
-
}
|
|
100
|
-
try {
|
|
101
|
-
return JSON.parse(value) as T;
|
|
102
|
-
} catch {
|
|
103
|
-
return fallback;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function resolveAccount(db: Database, accountId?: string): ResolvedAccount {
|
|
108
|
-
const row = accountId
|
|
109
|
-
? (db
|
|
110
|
-
.prepare(
|
|
111
|
-
"select id, handle, external_user_id from accounts where id = ?",
|
|
112
|
-
)
|
|
113
|
-
.get(accountId) as
|
|
114
|
-
| { id: string; handle: string; external_user_id: string | null }
|
|
115
|
-
| undefined)
|
|
116
|
-
: (db
|
|
117
|
-
.prepare(
|
|
118
|
-
`
|
|
119
|
-
select id, handle, external_user_id
|
|
120
|
-
from accounts
|
|
121
|
-
order by is_default desc, created_at asc
|
|
122
|
-
limit 1
|
|
123
|
-
`,
|
|
124
|
-
)
|
|
125
|
-
.get() as
|
|
126
|
-
| { id: string; handle: string; external_user_id: string | null }
|
|
127
|
-
| undefined);
|
|
128
|
-
|
|
129
|
-
if (!row) {
|
|
130
|
-
throw new Error(`Unknown account: ${accountId ?? "default"}`);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
return {
|
|
134
|
-
accountId: row.id,
|
|
135
|
-
username: row.handle.replace(/^@/, ""),
|
|
136
|
-
externalUserId:
|
|
137
|
-
typeof row.external_user_id === "string" &&
|
|
138
|
-
row.external_user_id.length > 0
|
|
139
|
-
? row.external_user_id
|
|
140
|
-
: undefined,
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
function buildCacheKey({
|
|
145
|
-
direction,
|
|
146
|
-
accountId,
|
|
147
|
-
mode,
|
|
148
|
-
limit,
|
|
149
|
-
maxPages,
|
|
150
|
-
maxResources,
|
|
151
|
-
}: {
|
|
152
|
-
direction: FollowDirection;
|
|
153
|
-
accountId: string;
|
|
154
|
-
mode: FollowGraphSyncMode;
|
|
155
|
-
limit: number;
|
|
156
|
-
maxPages?: number;
|
|
157
|
-
maxResources?: number;
|
|
158
|
-
}) {
|
|
159
|
-
return [
|
|
160
|
-
"follow-graph",
|
|
161
|
-
direction,
|
|
162
|
-
accountId,
|
|
163
|
-
`mode:${mode}`,
|
|
164
|
-
`limit:${String(limit)}`,
|
|
165
|
-
`pages:${maxPages === undefined ? "all" : String(maxPages)}`,
|
|
166
|
-
`resources:${maxResources === undefined ? "all" : String(maxResources)}`,
|
|
167
|
-
].join(":");
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function readCurrentCount(
|
|
171
|
-
db: Database,
|
|
172
|
-
accountId: string,
|
|
173
|
-
direction: FollowDirection,
|
|
174
|
-
) {
|
|
175
|
-
const row = db
|
|
176
|
-
.prepare(
|
|
177
|
-
`
|
|
178
|
-
select count(*) as count
|
|
179
|
-
from follow_edges
|
|
180
|
-
where account_id = ? and direction = ? and current = 1
|
|
181
|
-
`,
|
|
182
|
-
)
|
|
183
|
-
.get(accountId, direction) as { count: number };
|
|
184
|
-
return Number(row.count);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function getLastSnapshot(
|
|
188
|
-
db: Database,
|
|
189
|
-
accountId: string,
|
|
190
|
-
direction: FollowDirection,
|
|
191
|
-
status: "complete" | "incomplete",
|
|
192
|
-
) {
|
|
193
|
-
const row = db
|
|
194
|
-
.prepare(
|
|
195
|
-
`
|
|
196
|
-
select completed_at
|
|
197
|
-
from follow_snapshots
|
|
198
|
-
where account_id = ? and direction = ? and status = ?
|
|
199
|
-
order by completed_at desc
|
|
200
|
-
limit 1
|
|
201
|
-
`,
|
|
202
|
-
)
|
|
203
|
-
.get(accountId, direction, status) as { completed_at: string } | undefined;
|
|
204
|
-
return row?.completed_at;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
function mergePages(
|
|
208
|
-
pages: XurlFollowUsersResponse[],
|
|
209
|
-
nextToken: string | undefined,
|
|
210
|
-
maxResources?: number,
|
|
211
|
-
): MergedFollowPayload {
|
|
212
|
-
const users: XurlMentionUser[] = [];
|
|
213
|
-
const seen = new Set<string>();
|
|
214
|
-
let truncatedByMaxResources = false;
|
|
215
|
-
|
|
216
|
-
for (const page of pages) {
|
|
217
|
-
for (const user of page.data) {
|
|
218
|
-
if (seen.has(user.id)) {
|
|
219
|
-
continue;
|
|
220
|
-
}
|
|
221
|
-
if (maxResources !== undefined && users.length >= maxResources) {
|
|
222
|
-
truncatedByMaxResources = true;
|
|
223
|
-
break;
|
|
224
|
-
}
|
|
225
|
-
seen.add(user.id);
|
|
226
|
-
users.push(user);
|
|
227
|
-
}
|
|
228
|
-
if (truncatedByMaxResources) {
|
|
229
|
-
break;
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
const lastPage = pages.at(-1);
|
|
234
|
-
const payloadPageCount = Number(lastPage?.meta?.page_count);
|
|
235
|
-
const pageCount =
|
|
236
|
-
Number.isFinite(payloadPageCount) && payloadPageCount > pages.length
|
|
237
|
-
? payloadPageCount
|
|
238
|
-
: pages.length;
|
|
239
|
-
return {
|
|
240
|
-
data: users,
|
|
241
|
-
meta: {
|
|
242
|
-
...lastPage?.meta,
|
|
243
|
-
result_count: users.length,
|
|
244
|
-
page_count: pageCount,
|
|
245
|
-
next_token: nextToken ?? null,
|
|
246
|
-
truncated_by_max_resources: truncatedByMaxResources,
|
|
247
|
-
},
|
|
248
|
-
complete: !nextToken && !truncatedByMaxResources,
|
|
249
|
-
pageCount,
|
|
250
|
-
truncatedByMaxResources,
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
function fetchFollowGraphViaXurlEffect({
|
|
255
|
-
direction,
|
|
256
|
-
username,
|
|
257
|
-
userId,
|
|
258
|
-
limit,
|
|
259
|
-
maxPages,
|
|
260
|
-
maxResources,
|
|
261
|
-
}: {
|
|
262
|
-
direction: FollowDirection;
|
|
263
|
-
username: string;
|
|
264
|
-
userId?: string;
|
|
265
|
-
limit: number;
|
|
266
|
-
maxPages?: number;
|
|
267
|
-
maxResources?: number;
|
|
268
|
-
}): Effect.Effect<MergedFollowPayload, unknown> {
|
|
269
|
-
return Effect.gen(function* () {
|
|
270
|
-
const pages: XurlFollowUsersResponse[] = [];
|
|
271
|
-
let nextToken: string | undefined;
|
|
272
|
-
let pageCount = 0;
|
|
273
|
-
let collectedCount = 0;
|
|
274
|
-
|
|
275
|
-
do {
|
|
276
|
-
const payload = yield* tryPromise(() =>
|
|
277
|
-
listFollowUsersViaXurl({
|
|
278
|
-
direction,
|
|
279
|
-
username,
|
|
280
|
-
userId,
|
|
281
|
-
maxResults: limit,
|
|
282
|
-
paginationToken: nextToken,
|
|
283
|
-
}),
|
|
284
|
-
);
|
|
285
|
-
pages.push(payload);
|
|
286
|
-
collectedCount += payload.data.length;
|
|
287
|
-
nextToken =
|
|
288
|
-
typeof payload.meta?.next_token === "string"
|
|
289
|
-
? String(payload.meta.next_token)
|
|
290
|
-
: undefined;
|
|
291
|
-
pageCount += 1;
|
|
292
|
-
} while (
|
|
293
|
-
nextToken &&
|
|
294
|
-
(maxPages === undefined || pageCount < maxPages) &&
|
|
295
|
-
(maxResources === undefined || collectedCount < maxResources)
|
|
296
|
-
);
|
|
297
|
-
|
|
298
|
-
return mergePages(pages, nextToken, maxResources);
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function fetchFollowGraphViaBirdEffect({
|
|
303
|
-
direction,
|
|
304
|
-
userId,
|
|
305
|
-
limit,
|
|
306
|
-
maxPages,
|
|
307
|
-
maxResources,
|
|
308
|
-
}: {
|
|
309
|
-
direction: FollowDirection;
|
|
310
|
-
userId?: string;
|
|
311
|
-
limit: number;
|
|
312
|
-
maxPages?: number;
|
|
313
|
-
maxResources?: number;
|
|
314
|
-
}): Effect.Effect<MergedFollowPayload, unknown> {
|
|
315
|
-
return Effect.gen(function* () {
|
|
316
|
-
const birdLimit = Math.min(limit, BIRD_FOLLOW_PAGE_LIMIT);
|
|
317
|
-
const cappedMaxPages =
|
|
318
|
-
maxResources === undefined
|
|
319
|
-
? maxPages
|
|
320
|
-
: Math.min(
|
|
321
|
-
maxPages ?? Number.POSITIVE_INFINITY,
|
|
322
|
-
Math.ceil(maxResources / birdLimit),
|
|
323
|
-
);
|
|
324
|
-
const payload = yield* listFollowUsersViaBirdEffect({
|
|
325
|
-
direction,
|
|
326
|
-
userId,
|
|
327
|
-
maxResults: Math.min(birdLimit, maxResources ?? birdLimit),
|
|
328
|
-
all: true,
|
|
329
|
-
maxPages: Number.isFinite(cappedMaxPages) ? cappedMaxPages : undefined,
|
|
330
|
-
});
|
|
331
|
-
return mergePages(
|
|
332
|
-
[payload],
|
|
333
|
-
typeof payload.meta?.next_token === "string"
|
|
334
|
-
? String(payload.meta.next_token)
|
|
335
|
-
: undefined,
|
|
336
|
-
maxResources,
|
|
337
|
-
);
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
function fetchFollowGraphEffect({
|
|
342
|
-
mode,
|
|
343
|
-
direction,
|
|
344
|
-
username,
|
|
345
|
-
userId,
|
|
346
|
-
limit,
|
|
347
|
-
maxPages,
|
|
348
|
-
maxResources,
|
|
349
|
-
}: {
|
|
350
|
-
mode: FollowGraphSyncMode;
|
|
351
|
-
direction: FollowDirection;
|
|
352
|
-
username: string;
|
|
353
|
-
userId?: string;
|
|
354
|
-
limit: number;
|
|
355
|
-
maxPages?: number;
|
|
356
|
-
maxResources?: number;
|
|
357
|
-
}): Effect.Effect<
|
|
358
|
-
{ source: FollowGraphLiveSource; payload: MergedFollowPayload },
|
|
359
|
-
unknown
|
|
360
|
-
> {
|
|
361
|
-
return Effect.gen(function* () {
|
|
362
|
-
if (mode === "bird") {
|
|
363
|
-
return {
|
|
364
|
-
source: "bird",
|
|
365
|
-
payload: yield* fetchFollowGraphViaBirdEffect({
|
|
366
|
-
direction,
|
|
367
|
-
userId,
|
|
368
|
-
limit,
|
|
369
|
-
maxPages,
|
|
370
|
-
maxResources,
|
|
371
|
-
}),
|
|
372
|
-
};
|
|
373
|
-
}
|
|
374
|
-
if (mode === "xurl") {
|
|
375
|
-
return {
|
|
376
|
-
source: "xurl",
|
|
377
|
-
payload: yield* fetchFollowGraphViaXurlEffect({
|
|
378
|
-
direction,
|
|
379
|
-
username,
|
|
380
|
-
userId,
|
|
381
|
-
limit,
|
|
382
|
-
maxPages,
|
|
383
|
-
maxResources,
|
|
384
|
-
}),
|
|
385
|
-
};
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
const birdResult = yield* fetchFollowGraphViaBirdEffect({
|
|
389
|
-
direction,
|
|
390
|
-
userId,
|
|
391
|
-
limit,
|
|
392
|
-
maxPages,
|
|
393
|
-
maxResources,
|
|
394
|
-
}).pipe(
|
|
395
|
-
Effect.map((payload) => ({ ok: true as const, payload })),
|
|
396
|
-
Effect.catchAll((error) => Effect.succeed({ ok: false as const, error })),
|
|
397
|
-
);
|
|
398
|
-
if (birdResult.ok) {
|
|
399
|
-
return {
|
|
400
|
-
source: "bird",
|
|
401
|
-
payload: birdResult.payload,
|
|
402
|
-
};
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
const xurlResult = yield* fetchFollowGraphViaXurlEffect({
|
|
406
|
-
direction,
|
|
407
|
-
username,
|
|
408
|
-
userId,
|
|
409
|
-
limit,
|
|
410
|
-
maxPages,
|
|
411
|
-
maxResources,
|
|
412
|
-
}).pipe(
|
|
413
|
-
Effect.map((payload) => ({ ok: true as const, payload })),
|
|
414
|
-
Effect.catchAll((error) => Effect.succeed({ ok: false as const, error })),
|
|
415
|
-
);
|
|
416
|
-
if (xurlResult.ok) {
|
|
417
|
-
return {
|
|
418
|
-
source: "xurl",
|
|
419
|
-
payload: xurlResult.payload,
|
|
420
|
-
};
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
return yield* Effect.fail(
|
|
424
|
-
new Error(
|
|
425
|
-
`follow graph sync failed via bird and xurl: bird: ${errorMessage(
|
|
426
|
-
birdResult.error,
|
|
427
|
-
)}; xurl: ${errorMessage(xurlResult.error)}`,
|
|
428
|
-
),
|
|
429
|
-
);
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
function getExistingEdges(
|
|
434
|
-
db: Database,
|
|
435
|
-
accountId: string,
|
|
436
|
-
direction: FollowDirection,
|
|
437
|
-
) {
|
|
438
|
-
const rows = db
|
|
439
|
-
.prepare(
|
|
440
|
-
`
|
|
441
|
-
select profile_id, external_user_id, current
|
|
442
|
-
from follow_edges
|
|
443
|
-
where account_id = ? and direction = ?
|
|
444
|
-
`,
|
|
445
|
-
)
|
|
446
|
-
.all(accountId, direction) as Array<{
|
|
447
|
-
profile_id: string;
|
|
448
|
-
external_user_id: string;
|
|
449
|
-
current: number;
|
|
450
|
-
}>;
|
|
451
|
-
return new Map(rows.map((row) => [row.profile_id, row]));
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
function mergeFollowPayloadIntoLocalStore({
|
|
455
|
-
db,
|
|
456
|
-
accountId,
|
|
457
|
-
direction,
|
|
458
|
-
payload,
|
|
459
|
-
source,
|
|
460
|
-
}: {
|
|
461
|
-
db: Database;
|
|
462
|
-
accountId: string;
|
|
463
|
-
direction: FollowDirection;
|
|
464
|
-
payload: MergedFollowPayload;
|
|
465
|
-
source: FollowGraphLiveSource | "cache";
|
|
466
|
-
}) {
|
|
467
|
-
const now = new Date().toISOString();
|
|
468
|
-
const snapshotId = `follow_snapshot_${randomUUID()}`;
|
|
469
|
-
const status = payload.complete ? "complete" : "incomplete";
|
|
470
|
-
|
|
471
|
-
const insertSnapshot = db.prepare(`
|
|
472
|
-
insert into follow_snapshots (
|
|
473
|
-
id, account_id, direction, source, status, page_count, result_count,
|
|
474
|
-
started_at, completed_at, raw_meta_json
|
|
475
|
-
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
476
|
-
`);
|
|
477
|
-
const insertMember = db.prepare(`
|
|
478
|
-
insert into follow_snapshot_members (
|
|
479
|
-
snapshot_id, profile_id, external_user_id, position
|
|
480
|
-
) values (?, ?, ?, ?)
|
|
481
|
-
`);
|
|
482
|
-
const insertEdge = db.prepare(`
|
|
483
|
-
insert into follow_edges (
|
|
484
|
-
account_id, direction, profile_id, external_user_id, source, current,
|
|
485
|
-
first_seen_at, last_seen_at, ended_at, updated_at
|
|
486
|
-
) values (?, ?, ?, ?, ?, 1, ?, ?, null, ?)
|
|
487
|
-
on conflict(account_id, direction, profile_id) do update set
|
|
488
|
-
external_user_id = excluded.external_user_id,
|
|
489
|
-
source = excluded.source,
|
|
490
|
-
current = 1,
|
|
491
|
-
last_seen_at = excluded.last_seen_at,
|
|
492
|
-
ended_at = null,
|
|
493
|
-
updated_at = excluded.updated_at
|
|
494
|
-
`);
|
|
495
|
-
const endEdge = db.prepare(`
|
|
496
|
-
update follow_edges
|
|
497
|
-
set current = 0, ended_at = ?, updated_at = ?
|
|
498
|
-
where account_id = ? and direction = ? and profile_id = ?
|
|
499
|
-
`);
|
|
500
|
-
const insertEvent = db.prepare(`
|
|
501
|
-
insert into follow_events (
|
|
502
|
-
id, account_id, direction, profile_id, external_user_id, kind, event_at, snapshot_id
|
|
503
|
-
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
504
|
-
`);
|
|
505
|
-
|
|
506
|
-
return db.transaction(() => {
|
|
507
|
-
const existingEdges = getExistingEdges(db, accountId, direction);
|
|
508
|
-
const currentProfileIds = new Set<string>();
|
|
509
|
-
|
|
510
|
-
insertSnapshot.run(
|
|
511
|
-
snapshotId,
|
|
512
|
-
accountId,
|
|
513
|
-
direction,
|
|
514
|
-
source,
|
|
515
|
-
status,
|
|
516
|
-
payload.pageCount,
|
|
517
|
-
payload.data.length,
|
|
518
|
-
now,
|
|
519
|
-
now,
|
|
520
|
-
JSON.stringify(payload.meta),
|
|
521
|
-
);
|
|
522
|
-
|
|
523
|
-
payload.data.forEach((user, index) => {
|
|
524
|
-
const resolved = upsertProfileFromXUser(db, user);
|
|
525
|
-
currentProfileIds.add(resolved.profile.id);
|
|
526
|
-
insertMember.run(
|
|
527
|
-
snapshotId,
|
|
528
|
-
resolved.profile.id,
|
|
529
|
-
resolved.externalUserId,
|
|
530
|
-
index,
|
|
531
|
-
);
|
|
532
|
-
|
|
533
|
-
if (!payload.complete) {
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
const previous = existingEdges.get(resolved.profile.id);
|
|
538
|
-
insertEdge.run(
|
|
539
|
-
accountId,
|
|
540
|
-
direction,
|
|
541
|
-
resolved.profile.id,
|
|
542
|
-
resolved.externalUserId,
|
|
543
|
-
source,
|
|
544
|
-
now,
|
|
545
|
-
now,
|
|
546
|
-
now,
|
|
547
|
-
);
|
|
548
|
-
if (!previous || previous.current === 0) {
|
|
549
|
-
insertEvent.run(
|
|
550
|
-
`follow_event_${randomUUID()}`,
|
|
551
|
-
accountId,
|
|
552
|
-
direction,
|
|
553
|
-
resolved.profile.id,
|
|
554
|
-
resolved.externalUserId,
|
|
555
|
-
"started",
|
|
556
|
-
now,
|
|
557
|
-
snapshotId,
|
|
558
|
-
);
|
|
559
|
-
}
|
|
560
|
-
});
|
|
561
|
-
|
|
562
|
-
if (payload.complete) {
|
|
563
|
-
for (const [profileId, previous] of existingEdges) {
|
|
564
|
-
if (previous.current === 1 && !currentProfileIds.has(profileId)) {
|
|
565
|
-
endEdge.run(now, now, accountId, direction, profileId);
|
|
566
|
-
insertEvent.run(
|
|
567
|
-
`follow_event_${randomUUID()}`,
|
|
568
|
-
accountId,
|
|
569
|
-
direction,
|
|
570
|
-
profileId,
|
|
571
|
-
previous.external_user_id,
|
|
572
|
-
"ended",
|
|
573
|
-
now,
|
|
574
|
-
snapshotId,
|
|
575
|
-
);
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
return {
|
|
581
|
-
snapshotId,
|
|
582
|
-
status,
|
|
583
|
-
count: payload.data.length,
|
|
584
|
-
pageCount: payload.pageCount,
|
|
585
|
-
};
|
|
586
|
-
})();
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
function makeDryRunResponse({
|
|
590
|
-
db,
|
|
591
|
-
account,
|
|
592
|
-
direction,
|
|
593
|
-
mode,
|
|
594
|
-
limit,
|
|
595
|
-
maxPages,
|
|
596
|
-
maxResources,
|
|
597
|
-
cacheKey,
|
|
598
|
-
cacheTtlMs,
|
|
599
|
-
}: {
|
|
600
|
-
db: Database;
|
|
601
|
-
account: ResolvedAccount;
|
|
602
|
-
direction: FollowDirection;
|
|
603
|
-
mode: FollowGraphSyncMode;
|
|
604
|
-
limit: number;
|
|
605
|
-
maxPages?: number;
|
|
606
|
-
maxResources?: number;
|
|
607
|
-
cacheKey: string;
|
|
608
|
-
cacheTtlMs: number;
|
|
609
|
-
}) {
|
|
610
|
-
const cached = readSyncCache<MergedFollowPayload>(cacheKey, db);
|
|
611
|
-
const cacheAgeMs = cached
|
|
612
|
-
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
613
|
-
: Number.POSITIVE_INFINITY;
|
|
614
|
-
const wouldCallLive = !cached || cacheAgeMs > cacheTtlMs;
|
|
615
|
-
return {
|
|
616
|
-
ok: true,
|
|
617
|
-
dryRun: true,
|
|
618
|
-
direction,
|
|
619
|
-
accountId: account.accountId,
|
|
620
|
-
mode,
|
|
621
|
-
wouldCallLive,
|
|
622
|
-
wouldCallX: mode === "bird" ? false : wouldCallLive,
|
|
623
|
-
requiredFlag: "--yes",
|
|
624
|
-
estimate: {
|
|
625
|
-
maxResultsPerPage: limit,
|
|
626
|
-
maxPages: maxPages ?? null,
|
|
627
|
-
maxResources: maxResources ?? null,
|
|
628
|
-
cacheTtlSeconds: Math.floor(cacheTtlMs / 1000),
|
|
629
|
-
},
|
|
630
|
-
cache: {
|
|
631
|
-
key: cacheKey,
|
|
632
|
-
hit: Boolean(cached),
|
|
633
|
-
fresh: Boolean(cached && cacheAgeMs <= cacheTtlMs),
|
|
634
|
-
updatedAt: cached?.updatedAt ?? null,
|
|
635
|
-
ageSeconds: Number.isFinite(cacheAgeMs)
|
|
636
|
-
? Math.floor(cacheAgeMs / 1000)
|
|
637
|
-
: null,
|
|
638
|
-
count: cached?.value.data.length ?? 0,
|
|
639
|
-
},
|
|
640
|
-
currentCount: readCurrentCount(db, account.accountId, direction),
|
|
641
|
-
message:
|
|
642
|
-
"Dry run only. Pass --yes to use a fresh cache or perform live follow graph sync.",
|
|
643
|
-
};
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
function parseMode(value?: FollowGraphSyncMode) {
|
|
647
|
-
if (!value || value === "auto" || value === "bird" || value === "xurl") {
|
|
648
|
-
return value ?? "auto";
|
|
649
|
-
}
|
|
650
|
-
throw new Error("--mode must be auto, bird, or xurl");
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
function errorMessage(error: unknown) {
|
|
654
|
-
return error instanceof Error ? error.message : String(error);
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
export function syncFollowGraphEffect(options: SyncFollowGraphOptions) {
|
|
658
|
-
return Effect.gen(function* () {
|
|
659
|
-
const db = yield* trySync(() => getNativeDb());
|
|
660
|
-
const mode = yield* trySync(() => parseMode(options.mode));
|
|
661
|
-
const limit = yield* trySync(() => parseLimit(options.limit));
|
|
662
|
-
const maxPages = yield* trySync(() =>
|
|
663
|
-
parseOptionalPositiveInteger("--max-pages", options.maxPages),
|
|
664
|
-
);
|
|
665
|
-
const maxResources = yield* trySync(() =>
|
|
666
|
-
parseOptionalPositiveInteger("--max-resources", options.maxResources),
|
|
667
|
-
);
|
|
668
|
-
const cacheTtlMs = parseCacheTtlMs(options.cacheTtlMs);
|
|
669
|
-
const account = yield* trySync(() => resolveAccount(db, options.account));
|
|
670
|
-
const cacheKey = buildCacheKey({
|
|
671
|
-
direction: options.direction,
|
|
672
|
-
accountId: account.accountId,
|
|
673
|
-
mode,
|
|
674
|
-
limit,
|
|
675
|
-
maxPages,
|
|
676
|
-
maxResources,
|
|
677
|
-
});
|
|
678
|
-
|
|
679
|
-
if (!options.yes) {
|
|
680
|
-
return yield* trySync(() =>
|
|
681
|
-
makeDryRunResponse({
|
|
682
|
-
db,
|
|
683
|
-
account,
|
|
684
|
-
direction: options.direction,
|
|
685
|
-
mode,
|
|
686
|
-
limit,
|
|
687
|
-
maxPages,
|
|
688
|
-
maxResources,
|
|
689
|
-
cacheKey,
|
|
690
|
-
cacheTtlMs,
|
|
691
|
-
}),
|
|
692
|
-
);
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
const cached = yield* trySync(() =>
|
|
696
|
-
readSyncCache<MergedFollowPayload>(cacheKey, db),
|
|
697
|
-
);
|
|
698
|
-
const cacheAgeMs = cached
|
|
699
|
-
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
700
|
-
: Number.POSITIVE_INFINITY;
|
|
701
|
-
const useCache = Boolean(
|
|
702
|
-
!options.refresh && cached && cacheAgeMs <= cacheTtlMs,
|
|
703
|
-
);
|
|
704
|
-
|
|
705
|
-
const liveResult = useCache
|
|
706
|
-
? undefined
|
|
707
|
-
: yield* fetchFollowGraphEffect({
|
|
708
|
-
mode,
|
|
709
|
-
direction: options.direction,
|
|
710
|
-
username: account.username,
|
|
711
|
-
userId: account.externalUserId,
|
|
712
|
-
limit,
|
|
713
|
-
maxPages,
|
|
714
|
-
maxResources,
|
|
715
|
-
});
|
|
716
|
-
const payload = useCache ? cached!.value : liveResult!.payload;
|
|
717
|
-
|
|
718
|
-
if (!useCache) {
|
|
719
|
-
yield* trySync(() => writeSyncCache(cacheKey, payload, db));
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
const mergeResult = yield* trySync(() =>
|
|
723
|
-
mergeFollowPayloadIntoLocalStore({
|
|
724
|
-
db,
|
|
725
|
-
accountId: account.accountId,
|
|
726
|
-
direction: options.direction,
|
|
727
|
-
payload,
|
|
728
|
-
source: useCache ? "cache" : liveResult!.source,
|
|
729
|
-
}),
|
|
730
|
-
);
|
|
731
|
-
|
|
732
|
-
return {
|
|
733
|
-
ok: true,
|
|
734
|
-
dryRun: false,
|
|
735
|
-
source: useCache ? "cache" : liveResult!.source,
|
|
736
|
-
mode,
|
|
737
|
-
direction: options.direction,
|
|
738
|
-
accountId: account.accountId,
|
|
739
|
-
status: mergeResult.status,
|
|
740
|
-
count: mergeResult.count,
|
|
741
|
-
pageCount: mergeResult.pageCount,
|
|
742
|
-
snapshotId: mergeResult.snapshotId,
|
|
743
|
-
partial: mergeResult.status === "incomplete",
|
|
744
|
-
cache: {
|
|
745
|
-
key: cacheKey,
|
|
746
|
-
reused: useCache,
|
|
747
|
-
updatedAt: useCache ? cached?.updatedAt : new Date().toISOString(),
|
|
748
|
-
},
|
|
749
|
-
warning:
|
|
750
|
-
mergeResult.status === "incomplete" && !options.allowPartial
|
|
751
|
-
? "Snapshot is incomplete because a page/resource cap stopped pagination. It was recorded but not used for churn events."
|
|
752
|
-
: undefined,
|
|
753
|
-
};
|
|
754
|
-
});
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
export function syncFollowGraph(options: SyncFollowGraphOptions) {
|
|
758
|
-
return runEffectPromise(syncFollowGraphEffect(options));
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
function toGraphProfile(row: Record<string, unknown>): FollowGraphProfile {
|
|
762
|
-
return {
|
|
763
|
-
id: String(row.profile_id ?? row.id),
|
|
764
|
-
externalUserId: String(row.external_user_id),
|
|
765
|
-
handle: String(row.handle),
|
|
766
|
-
displayName: String(row.display_name),
|
|
767
|
-
bio: String(row.bio ?? ""),
|
|
768
|
-
followersCount: Number(row.followers_count ?? 0),
|
|
769
|
-
publicMetrics: parseJsonField<XurlPublicMetrics>(
|
|
770
|
-
row.public_metrics_json,
|
|
771
|
-
{},
|
|
772
|
-
),
|
|
773
|
-
avatarUrl:
|
|
774
|
-
typeof row.avatar_url === "string" && row.avatar_url.length > 0
|
|
775
|
-
? String(row.avatar_url)
|
|
776
|
-
: undefined,
|
|
777
|
-
};
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
export function listTopFollowers({
|
|
781
|
-
account,
|
|
782
|
-
limit = 20,
|
|
783
|
-
}: {
|
|
784
|
-
account?: string;
|
|
785
|
-
limit?: number;
|
|
786
|
-
} = {}) {
|
|
787
|
-
const db = getNativeDb();
|
|
788
|
-
const resolved = resolveAccount(db, account);
|
|
789
|
-
const rows = db
|
|
790
|
-
.prepare(
|
|
791
|
-
`
|
|
792
|
-
select
|
|
793
|
-
e.profile_id,
|
|
794
|
-
e.external_user_id,
|
|
795
|
-
p.handle,
|
|
796
|
-
p.display_name,
|
|
797
|
-
p.bio,
|
|
798
|
-
p.followers_count,
|
|
799
|
-
p.public_metrics_json,
|
|
800
|
-
p.avatar_url
|
|
801
|
-
from follow_edges e
|
|
802
|
-
join profiles p on p.id = e.profile_id
|
|
803
|
-
where e.account_id = ? and e.direction = 'followers' and e.current = 1
|
|
804
|
-
order by p.followers_count desc, lower(p.handle) asc
|
|
805
|
-
limit ?
|
|
806
|
-
`,
|
|
807
|
-
)
|
|
808
|
-
.all(resolved.accountId, limit) as Array<Record<string, unknown>>;
|
|
809
|
-
return {
|
|
810
|
-
accountId: resolved.accountId,
|
|
811
|
-
items: rows.map(toGraphProfile),
|
|
812
|
-
};
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
export function listMutuals({
|
|
816
|
-
account,
|
|
817
|
-
limit = 100,
|
|
818
|
-
}: {
|
|
819
|
-
account?: string;
|
|
820
|
-
limit?: number;
|
|
821
|
-
} = {}) {
|
|
822
|
-
const db = getNativeDb();
|
|
823
|
-
const resolved = resolveAccount(db, account);
|
|
824
|
-
const rows = db
|
|
825
|
-
.prepare(
|
|
826
|
-
`
|
|
827
|
-
select
|
|
828
|
-
following.profile_id,
|
|
829
|
-
following.external_user_id,
|
|
830
|
-
p.handle,
|
|
831
|
-
p.display_name,
|
|
832
|
-
p.bio,
|
|
833
|
-
p.followers_count,
|
|
834
|
-
p.public_metrics_json,
|
|
835
|
-
p.avatar_url
|
|
836
|
-
from follow_edges following
|
|
837
|
-
join follow_edges followers
|
|
838
|
-
on followers.account_id = following.account_id
|
|
839
|
-
and followers.profile_id = following.profile_id
|
|
840
|
-
and followers.direction = 'followers'
|
|
841
|
-
and followers.current = 1
|
|
842
|
-
join profiles p on p.id = following.profile_id
|
|
843
|
-
where
|
|
844
|
-
following.account_id = ?
|
|
845
|
-
and following.direction = 'following'
|
|
846
|
-
and following.current = 1
|
|
847
|
-
order by p.followers_count desc, lower(p.handle) asc
|
|
848
|
-
limit ?
|
|
849
|
-
`,
|
|
850
|
-
)
|
|
851
|
-
.all(resolved.accountId, limit) as Array<Record<string, unknown>>;
|
|
852
|
-
return {
|
|
853
|
-
accountId: resolved.accountId,
|
|
854
|
-
items: rows.map(toGraphProfile),
|
|
855
|
-
};
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
export function listNonMutualFollowing({
|
|
859
|
-
account,
|
|
860
|
-
sort = "followers",
|
|
861
|
-
limit = 100,
|
|
862
|
-
}: {
|
|
863
|
-
account?: string;
|
|
864
|
-
sort?: "followers" | "handle";
|
|
865
|
-
limit?: number;
|
|
866
|
-
} = {}) {
|
|
867
|
-
const db = getNativeDb();
|
|
868
|
-
const resolved = resolveAccount(db, account);
|
|
869
|
-
const orderBy =
|
|
870
|
-
sort === "handle"
|
|
871
|
-
? "lower(p.handle) asc"
|
|
872
|
-
: "p.followers_count desc, lower(p.handle) asc";
|
|
873
|
-
const rows = db
|
|
874
|
-
.prepare(
|
|
875
|
-
`
|
|
876
|
-
select
|
|
877
|
-
following.profile_id,
|
|
878
|
-
following.external_user_id,
|
|
879
|
-
p.handle,
|
|
880
|
-
p.display_name,
|
|
881
|
-
p.bio,
|
|
882
|
-
p.followers_count,
|
|
883
|
-
p.public_metrics_json,
|
|
884
|
-
p.avatar_url
|
|
885
|
-
from follow_edges following
|
|
886
|
-
left join follow_edges followers
|
|
887
|
-
on followers.account_id = following.account_id
|
|
888
|
-
and followers.profile_id = following.profile_id
|
|
889
|
-
and followers.direction = 'followers'
|
|
890
|
-
and followers.current = 1
|
|
891
|
-
join profiles p on p.id = following.profile_id
|
|
892
|
-
where
|
|
893
|
-
following.account_id = ?
|
|
894
|
-
and following.direction = 'following'
|
|
895
|
-
and following.current = 1
|
|
896
|
-
and followers.profile_id is null
|
|
897
|
-
order by ${orderBy}
|
|
898
|
-
limit ?
|
|
899
|
-
`,
|
|
900
|
-
)
|
|
901
|
-
.all(resolved.accountId, limit) as Array<Record<string, unknown>>;
|
|
902
|
-
return {
|
|
903
|
-
accountId: resolved.accountId,
|
|
904
|
-
items: rows.map(toGraphProfile),
|
|
905
|
-
};
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
export function listUnfollowedSince({
|
|
909
|
-
account,
|
|
910
|
-
date,
|
|
911
|
-
direction = "followers",
|
|
912
|
-
limit = 100,
|
|
913
|
-
}: {
|
|
914
|
-
account?: string;
|
|
915
|
-
date: string;
|
|
916
|
-
direction?: FollowDirection;
|
|
917
|
-
limit?: number;
|
|
918
|
-
}) {
|
|
919
|
-
const db = getNativeDb();
|
|
920
|
-
const resolved = resolveAccount(db, account);
|
|
921
|
-
const since = date.includes("T") ? date : `${date}T00:00:00.000Z`;
|
|
922
|
-
const rows = db
|
|
923
|
-
.prepare(
|
|
924
|
-
`
|
|
925
|
-
select
|
|
926
|
-
ev.event_at,
|
|
927
|
-
ev.profile_id,
|
|
928
|
-
ev.external_user_id,
|
|
929
|
-
p.handle,
|
|
930
|
-
p.display_name,
|
|
931
|
-
p.bio,
|
|
932
|
-
p.followers_count,
|
|
933
|
-
p.public_metrics_json,
|
|
934
|
-
p.avatar_url
|
|
935
|
-
from follow_events ev
|
|
936
|
-
join profiles p on p.id = ev.profile_id
|
|
937
|
-
where
|
|
938
|
-
ev.account_id = ?
|
|
939
|
-
and ev.direction = ?
|
|
940
|
-
and ev.kind = 'ended'
|
|
941
|
-
and ev.event_at >= ?
|
|
942
|
-
order by ev.event_at desc, p.followers_count desc
|
|
943
|
-
limit ?
|
|
944
|
-
`,
|
|
945
|
-
)
|
|
946
|
-
.all(resolved.accountId, direction, since, limit) as Array<
|
|
947
|
-
Record<string, unknown>
|
|
948
|
-
>;
|
|
949
|
-
return {
|
|
950
|
-
accountId: resolved.accountId,
|
|
951
|
-
direction,
|
|
952
|
-
since,
|
|
953
|
-
items: rows.map((row) => ({
|
|
954
|
-
eventAt: String(row.event_at),
|
|
955
|
-
profile: toGraphProfile(row),
|
|
956
|
-
})),
|
|
957
|
-
};
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
export function listFollowEvents({
|
|
961
|
-
account,
|
|
962
|
-
direction,
|
|
963
|
-
kind,
|
|
964
|
-
since,
|
|
965
|
-
until,
|
|
966
|
-
limit = 100,
|
|
967
|
-
}: {
|
|
968
|
-
account?: string;
|
|
969
|
-
direction?: FollowDirection;
|
|
970
|
-
kind?: FollowEventKind;
|
|
971
|
-
since?: string;
|
|
972
|
-
until?: string;
|
|
973
|
-
limit?: number;
|
|
974
|
-
} = {}) {
|
|
975
|
-
const db = getNativeDb();
|
|
976
|
-
const resolved = resolveAccount(db, account);
|
|
977
|
-
const where = ["ev.account_id = ?"];
|
|
978
|
-
const params: unknown[] = [resolved.accountId];
|
|
979
|
-
|
|
980
|
-
if (direction) {
|
|
981
|
-
where.push("ev.direction = ?");
|
|
982
|
-
params.push(direction);
|
|
983
|
-
}
|
|
984
|
-
if (kind) {
|
|
985
|
-
where.push("ev.kind = ?");
|
|
986
|
-
params.push(kind);
|
|
987
|
-
}
|
|
988
|
-
if (since) {
|
|
989
|
-
where.push("ev.event_at >= ?");
|
|
990
|
-
params.push(since.includes("T") ? since : `${since}T00:00:00.000Z`);
|
|
991
|
-
}
|
|
992
|
-
if (until) {
|
|
993
|
-
where.push("ev.event_at < ?");
|
|
994
|
-
params.push(until.includes("T") ? until : `${until}T00:00:00.000Z`);
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
params.push(limit);
|
|
998
|
-
const rows = db
|
|
999
|
-
.prepare(
|
|
1000
|
-
`
|
|
1001
|
-
select
|
|
1002
|
-
ev.event_at,
|
|
1003
|
-
ev.direction,
|
|
1004
|
-
ev.kind,
|
|
1005
|
-
ev.snapshot_id,
|
|
1006
|
-
ev.profile_id,
|
|
1007
|
-
ev.external_user_id,
|
|
1008
|
-
p.handle,
|
|
1009
|
-
p.display_name,
|
|
1010
|
-
p.bio,
|
|
1011
|
-
p.followers_count,
|
|
1012
|
-
p.public_metrics_json,
|
|
1013
|
-
p.avatar_url
|
|
1014
|
-
from follow_events ev
|
|
1015
|
-
join profiles p on p.id = ev.profile_id
|
|
1016
|
-
where ${where.join(" and ")}
|
|
1017
|
-
order by ev.event_at desc, p.followers_count desc, lower(p.handle) asc
|
|
1018
|
-
limit ?
|
|
1019
|
-
`,
|
|
1020
|
-
)
|
|
1021
|
-
.all(...params) as Array<Record<string, unknown>>;
|
|
1022
|
-
|
|
1023
|
-
return {
|
|
1024
|
-
accountId: resolved.accountId,
|
|
1025
|
-
items: rows.map(
|
|
1026
|
-
(row): FollowGraphEvent => ({
|
|
1027
|
-
eventAt: String(row.event_at),
|
|
1028
|
-
direction: row.direction as FollowDirection,
|
|
1029
|
-
kind: row.kind as FollowEventKind,
|
|
1030
|
-
snapshotId: String(row.snapshot_id),
|
|
1031
|
-
profile: toGraphProfile(row),
|
|
1032
|
-
}),
|
|
1033
|
-
),
|
|
1034
|
-
};
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
export function getFollowGraphSummary({
|
|
1038
|
-
account,
|
|
1039
|
-
}: { account?: string } = {}): FollowGraphSummary {
|
|
1040
|
-
const db = getNativeDb();
|
|
1041
|
-
const resolved = resolveAccount(db, account);
|
|
1042
|
-
const followers = readCurrentCount(db, resolved.accountId, "followers");
|
|
1043
|
-
const following = readCurrentCount(db, resolved.accountId, "following");
|
|
1044
|
-
const mutualsRow = db
|
|
1045
|
-
.prepare(
|
|
1046
|
-
`
|
|
1047
|
-
select count(*) as count
|
|
1048
|
-
from follow_edges following
|
|
1049
|
-
join follow_edges followers
|
|
1050
|
-
on followers.account_id = following.account_id
|
|
1051
|
-
and followers.profile_id = following.profile_id
|
|
1052
|
-
and followers.direction = 'followers'
|
|
1053
|
-
and followers.current = 1
|
|
1054
|
-
where
|
|
1055
|
-
following.account_id = ?
|
|
1056
|
-
and following.direction = 'following'
|
|
1057
|
-
and following.current = 1
|
|
1058
|
-
`,
|
|
1059
|
-
)
|
|
1060
|
-
.get(resolved.accountId) as { count: number };
|
|
1061
|
-
|
|
1062
|
-
return {
|
|
1063
|
-
accountId: resolved.accountId,
|
|
1064
|
-
followers,
|
|
1065
|
-
following,
|
|
1066
|
-
mutuals: Number(mutualsRow.count),
|
|
1067
|
-
nonMutualFollowing: following - Number(mutualsRow.count),
|
|
1068
|
-
lastCompleteSnapshots: {
|
|
1069
|
-
followers: getLastSnapshot(
|
|
1070
|
-
db,
|
|
1071
|
-
resolved.accountId,
|
|
1072
|
-
"followers",
|
|
1073
|
-
"complete",
|
|
1074
|
-
),
|
|
1075
|
-
following: getLastSnapshot(
|
|
1076
|
-
db,
|
|
1077
|
-
resolved.accountId,
|
|
1078
|
-
"following",
|
|
1079
|
-
"complete",
|
|
1080
|
-
),
|
|
1081
|
-
},
|
|
1082
|
-
lastIncompleteSnapshots: {
|
|
1083
|
-
followers: getLastSnapshot(
|
|
1084
|
-
db,
|
|
1085
|
-
resolved.accountId,
|
|
1086
|
-
"followers",
|
|
1087
|
-
"incomplete",
|
|
1088
|
-
),
|
|
1089
|
-
following: getLastSnapshot(
|
|
1090
|
-
db,
|
|
1091
|
-
resolved.accountId,
|
|
1092
|
-
"following",
|
|
1093
|
-
"incomplete",
|
|
1094
|
-
),
|
|
1095
|
-
},
|
|
1096
|
-
};
|
|
1097
|
-
}
|