birdclaw 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/package.json +2 -1
- package/src/cli/command-context.ts +17 -0
- package/src/cli/register-analysis.ts +500 -0
- package/src/cli/register-compose.ts +40 -0
- package/src/cli/register-graph.ts +132 -0
- package/src/cli/register-inbox.ts +41 -0
- package/src/cli/register-storage.ts +106 -0
- package/src/cli.ts +30 -750
- package/src/components/AccountSwitcher.tsx +7 -15
- package/src/components/AvatarChip.tsx +1 -1
- package/src/components/AvatarPreload.ts +149 -0
- package/src/components/MarkdownCitations.tsx +680 -0
- package/src/components/MarkdownViewer.tsx +8 -674
- package/src/components/ProfileAnalysisClient.ts +191 -0
- package/src/components/ProfileAnalysisStream.tsx +16 -185
- package/src/components/ProfilePreview.tsx +2 -0
- package/src/components/links-controller.ts +162 -0
- package/src/components/links-model.ts +198 -0
- package/src/components/network-map-controller.ts +84 -0
- package/src/components/network-map-model.ts +255 -0
- package/src/components/useTimelineRouteData.ts +105 -235
- package/src/lib/analysis-runtime.ts +238 -0
- package/src/lib/api-client.ts +16 -215
- package/src/lib/api-contracts.ts +328 -0
- package/src/lib/archive-import-plan.ts +102 -0
- package/src/lib/archive-import.ts +170 -239
- package/src/lib/authored-live.ts +75 -120
- package/src/lib/backup.ts +335 -424
- package/src/lib/blocks-write.ts +30 -26
- package/src/lib/blocks.ts +18 -20
- package/src/lib/database-metrics.ts +88 -0
- package/src/lib/database-migrations.ts +34 -0
- package/src/lib/database-schema.ts +312 -0
- package/src/lib/database-writer.ts +69 -0
- package/src/lib/db.ts +84 -330
- package/src/lib/dm-read-model.ts +533 -0
- package/src/lib/dms-live.ts +34 -97
- package/src/lib/follow-graph.ts +17 -27
- package/src/lib/import-repository.ts +138 -0
- package/src/lib/inbox.ts +2 -1
- package/src/lib/live-sync-engine.ts +209 -0
- package/src/lib/live-transport-gateway.ts +128 -0
- package/src/lib/mention-threads-live.ts +90 -177
- package/src/lib/mentions-export.ts +1 -1
- package/src/lib/mentions-live.ts +57 -181
- package/src/lib/moderation-target.ts +15 -4
- package/src/lib/moderation-write.ts +1 -1
- package/src/lib/mutes-write.ts +30 -26
- package/src/lib/openai-response-runtime.ts +251 -0
- package/src/lib/paginated-sync.ts +93 -0
- package/src/lib/period-digest.ts +116 -304
- package/src/lib/profile-analysis.ts +36 -110
- package/src/lib/queries.ts +6 -2381
- package/src/lib/query-actions.ts +437 -0
- package/src/lib/query-client.tsx +47 -0
- package/src/lib/query-read-model-shared.ts +52 -0
- package/src/lib/query-read-models.ts +5 -0
- package/src/lib/query-resource.ts +41 -0
- package/src/lib/query-status.ts +164 -0
- package/src/lib/research.ts +1 -1
- package/src/lib/runtime-services.ts +20 -0
- package/src/lib/search-discussion.ts +75 -279
- package/src/lib/server-runtime-services.ts +30 -0
- package/src/lib/sqlite.ts +48 -12
- package/src/lib/streaming-ingestion.ts +240 -0
- package/src/lib/sync-cache.ts +6 -1
- package/src/lib/sync-plan.ts +175 -0
- package/src/lib/timeline-collections-live.ts +83 -257
- package/src/lib/timeline-live.ts +86 -236
- package/src/lib/timeline-read-model.ts +1191 -0
- package/src/lib/tweet-repository.ts +156 -0
- package/src/lib/tweet-search-live.ts +63 -167
- package/src/lib/web-sync.ts +67 -50
- package/src/lib/whois.ts +2 -1
- package/src/routes/__root.tsx +11 -8
- package/src/routes/api/action.tsx +1 -1
- package/src/routes/api/conversation.tsx +1 -1
- package/src/routes/api/query.tsx +32 -26
- package/src/routes/api/status.tsx +6 -4
- package/src/routes/api/sync.tsx +5 -2
- package/src/routes/blocks.tsx +97 -131
- package/src/routes/data-sources.tsx +17 -25
- package/src/routes/dms.tsx +167 -184
- package/src/routes/inbox.tsx +63 -57
- package/src/routes/links.tsx +31 -394
- package/src/routes/network-map.tsx +41 -344
- package/src/routes/rate-limits.tsx +17 -21
- package/src/lib/client-cache.ts +0 -109
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import type { Database } from "./sqlite";
|
|
2
|
+
import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
|
|
3
|
+
import { tweetEntitiesFromXurl } from "./tweet-render";
|
|
4
|
+
import type { XurlMentionData, XurlMentionsResponse } from "./types";
|
|
5
|
+
import {
|
|
6
|
+
type TweetAccountEdgeKind,
|
|
7
|
+
upsertTweetAccountEdge,
|
|
8
|
+
} from "./tweet-account-edges";
|
|
9
|
+
import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
|
|
10
|
+
|
|
11
|
+
export type CanonicalTweetKind =
|
|
12
|
+
| "authored"
|
|
13
|
+
| "bookmark"
|
|
14
|
+
| "home"
|
|
15
|
+
| "like"
|
|
16
|
+
| "mention"
|
|
17
|
+
| "search"
|
|
18
|
+
| "thread"
|
|
19
|
+
| "thread_context";
|
|
20
|
+
|
|
21
|
+
export interface IngestTweetPayloadOptions {
|
|
22
|
+
accountId: string;
|
|
23
|
+
payload: XurlMentionsResponse;
|
|
24
|
+
kind: CanonicalTweetKind;
|
|
25
|
+
source: string;
|
|
26
|
+
edgeKind?: TweetAccountEdgeKind;
|
|
27
|
+
collectionKind?: "likes" | "bookmarks";
|
|
28
|
+
markRepliesAsReplied?: boolean;
|
|
29
|
+
replaceSecondaryKind?: boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function getReferencedTweetId(tweet: XurlMentionData, type: string) {
|
|
33
|
+
return (
|
|
34
|
+
tweet.referenced_tweets?.find((item) => item.type === type)?.id ?? null
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function replaceTweetFts(db: Database, tweetId: string, text: string) {
|
|
39
|
+
db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
|
|
40
|
+
db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
|
|
41
|
+
tweetId,
|
|
42
|
+
text,
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function ingestTweetPayload(
|
|
47
|
+
db: Database,
|
|
48
|
+
{
|
|
49
|
+
accountId,
|
|
50
|
+
payload,
|
|
51
|
+
kind,
|
|
52
|
+
source,
|
|
53
|
+
edgeKind,
|
|
54
|
+
collectionKind,
|
|
55
|
+
markRepliesAsReplied = false,
|
|
56
|
+
replaceSecondaryKind = false,
|
|
57
|
+
}: IngestTweetPayloadOptions,
|
|
58
|
+
) {
|
|
59
|
+
const usersById = new Map(
|
|
60
|
+
(payload.includes?.users ?? []).map((user) => [user.id, user]),
|
|
61
|
+
);
|
|
62
|
+
const upsertTweet = db.prepare(`
|
|
63
|
+
insert into tweets (
|
|
64
|
+
id, account_id, author_profile_id, kind, text, created_at,
|
|
65
|
+
is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
|
|
66
|
+
entities_json, media_json, quoted_tweet_id
|
|
67
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
68
|
+
on conflict(id) do update set
|
|
69
|
+
account_id = tweets.account_id,
|
|
70
|
+
author_profile_id = excluded.author_profile_id,
|
|
71
|
+
kind = case
|
|
72
|
+
when ? = 1
|
|
73
|
+
and tweets.kind not in ('authored', 'home', 'mention')
|
|
74
|
+
then excluded.kind
|
|
75
|
+
else tweets.kind
|
|
76
|
+
end,
|
|
77
|
+
text = excluded.text,
|
|
78
|
+
created_at = excluded.created_at,
|
|
79
|
+
is_replied = max(tweets.is_replied, excluded.is_replied),
|
|
80
|
+
reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
|
|
81
|
+
like_count = excluded.like_count,
|
|
82
|
+
media_count = max(tweets.media_count, excluded.media_count),
|
|
83
|
+
entities_json = excluded.entities_json,
|
|
84
|
+
media_json = case
|
|
85
|
+
when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
|
|
86
|
+
else tweets.media_json
|
|
87
|
+
end,
|
|
88
|
+
bookmarked = tweets.bookmarked,
|
|
89
|
+
liked = tweets.liked,
|
|
90
|
+
quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id)
|
|
91
|
+
`);
|
|
92
|
+
const upsertCollection = collectionKind
|
|
93
|
+
? db.prepare(`
|
|
94
|
+
insert into tweet_collections (
|
|
95
|
+
account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
|
|
96
|
+
) values (?, ?, ?, null, ?, ?, ?)
|
|
97
|
+
on conflict(account_id, tweet_id, kind) do update set
|
|
98
|
+
source = excluded.source,
|
|
99
|
+
raw_json = excluded.raw_json,
|
|
100
|
+
updated_at = excluded.updated_at
|
|
101
|
+
`)
|
|
102
|
+
: undefined;
|
|
103
|
+
const tweetIds: string[] = [];
|
|
104
|
+
|
|
105
|
+
db.transaction(() => {
|
|
106
|
+
const observedAt = new Date().toISOString();
|
|
107
|
+
for (const tweet of payload.data) {
|
|
108
|
+
const author = usersById.get(tweet.author_id);
|
|
109
|
+
const profile = author
|
|
110
|
+
? upsertProfileFromXUser(db, author)
|
|
111
|
+
: ensureStubProfileForXUser(db, tweet.author_id);
|
|
112
|
+
const replyToId = getReferencedTweetId(tweet, "replied_to");
|
|
113
|
+
const quotedTweetId = getReferencedTweetId(tweet, "quoted");
|
|
114
|
+
upsertTweet.run(
|
|
115
|
+
tweet.id,
|
|
116
|
+
accountId,
|
|
117
|
+
profile.profile.id,
|
|
118
|
+
kind,
|
|
119
|
+
tweet.text,
|
|
120
|
+
tweet.created_at,
|
|
121
|
+
markRepliesAsReplied && replyToId ? 1 : 0,
|
|
122
|
+
replyToId,
|
|
123
|
+
Number(tweet.public_metrics?.like_count ?? 0),
|
|
124
|
+
countTweetMedia(tweet),
|
|
125
|
+
collectionKind === "bookmarks" ? 1 : 0,
|
|
126
|
+
collectionKind === "likes" ? 1 : 0,
|
|
127
|
+
JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
|
|
128
|
+
buildMediaJsonFromIncludes(tweet, payload.includes?.media),
|
|
129
|
+
quotedTweetId,
|
|
130
|
+
replaceSecondaryKind ? 1 : 0,
|
|
131
|
+
);
|
|
132
|
+
if (edgeKind) {
|
|
133
|
+
upsertTweetAccountEdge(db, {
|
|
134
|
+
accountId,
|
|
135
|
+
tweetId: tweet.id,
|
|
136
|
+
kind: edgeKind,
|
|
137
|
+
source,
|
|
138
|
+
seenAt: observedAt,
|
|
139
|
+
rawJson: JSON.stringify(tweet),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
upsertCollection?.run(
|
|
143
|
+
accountId,
|
|
144
|
+
tweet.id,
|
|
145
|
+
collectionKind,
|
|
146
|
+
source,
|
|
147
|
+
JSON.stringify(tweet),
|
|
148
|
+
observedAt,
|
|
149
|
+
);
|
|
150
|
+
replaceTweetFts(db, tweet.id, tweet.text);
|
|
151
|
+
tweetIds.push(tweet.id);
|
|
152
|
+
}
|
|
153
|
+
})();
|
|
154
|
+
|
|
155
|
+
return tweetIds;
|
|
156
|
+
}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { Effect } from "effect";
|
|
2
|
-
import { searchTweetsViaBirdEffect } from "./bird";
|
|
3
2
|
import type { Database } from "./sqlite";
|
|
4
3
|
import { getNativeDb } from "./db";
|
|
5
4
|
import { runEffectPromise } from "./effect-runtime";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
|
|
5
|
+
import { liveTransportGateway } from "./live-transport-gateway";
|
|
6
|
+
import {
|
|
7
|
+
normalizeCacheTtlMs,
|
|
8
|
+
resolveLiveSyncAccount,
|
|
9
|
+
runCachedLiveSyncEffect,
|
|
10
|
+
} from "./live-sync-engine";
|
|
9
11
|
import type { XurlMentionsResponse, XurlTweetsResponse } from "./types";
|
|
10
|
-
import {
|
|
11
|
-
import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
|
|
12
|
-
import { searchRecentTweetsEffect } from "./xurl";
|
|
12
|
+
import { ingestTweetPayload } from "./tweet-repository";
|
|
13
13
|
|
|
14
14
|
export type TweetSearchMode = "auto" | "bird" | "xurl" | "local";
|
|
15
15
|
|
|
@@ -85,130 +85,19 @@ function normalizeTime(value: string | undefined, optionName: string) {
|
|
|
85
85
|
return date.toISOString();
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
-
function normalizeCacheTtlMs(value: number | undefined) {
|
|
89
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
90
|
-
return DEFAULT_CACHE_TTL_MS;
|
|
91
|
-
}
|
|
92
|
-
return Math.floor(value);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
function resolveAccount(db: Database, accountId?: string) {
|
|
96
|
-
const row = accountId
|
|
97
|
-
? (db
|
|
98
|
-
.prepare("select id, handle from accounts where id = ?")
|
|
99
|
-
.get(accountId) as { id: string; handle: string } | undefined)
|
|
100
|
-
: (db
|
|
101
|
-
.prepare(
|
|
102
|
-
`
|
|
103
|
-
select id, handle
|
|
104
|
-
from accounts
|
|
105
|
-
order by is_default desc, created_at asc
|
|
106
|
-
limit 1
|
|
107
|
-
`,
|
|
108
|
-
)
|
|
109
|
-
.get() as { id: string; handle: string } | undefined);
|
|
110
|
-
|
|
111
|
-
if (!row) {
|
|
112
|
-
throw new Error(`Unknown account: ${accountId ?? "default"}`);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return {
|
|
116
|
-
accountId: row.id,
|
|
117
|
-
username: row.handle.replace(/^@/, ""),
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
function replaceTweetFts(db: Database, tweetId: string, text: string) {
|
|
122
|
-
db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
|
|
123
|
-
db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
|
|
124
|
-
tweetId,
|
|
125
|
-
text,
|
|
126
|
-
);
|
|
127
|
-
}
|
|
128
|
-
|
|
129
88
|
function mergeTweetSearchIntoLocalStore(
|
|
130
89
|
db: Database,
|
|
131
90
|
accountId: string,
|
|
132
91
|
payload: XurlMentionsResponse,
|
|
133
92
|
source: "bird" | "xurl" | "cache",
|
|
134
93
|
) {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
|
|
143
|
-
entities_json, media_json, quoted_tweet_id
|
|
144
|
-
) values (?, ?, ?, 'search', ?, ?, 0, ?, ?, ?, 0, 0, ?, ?, ?)
|
|
145
|
-
on conflict(id) do update set
|
|
146
|
-
account_id = tweets.account_id,
|
|
147
|
-
author_profile_id = excluded.author_profile_id,
|
|
148
|
-
kind = tweets.kind,
|
|
149
|
-
text = excluded.text,
|
|
150
|
-
created_at = excluded.created_at,
|
|
151
|
-
reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
|
|
152
|
-
like_count = excluded.like_count,
|
|
153
|
-
media_count = max(tweets.media_count, excluded.media_count),
|
|
154
|
-
entities_json = excluded.entities_json,
|
|
155
|
-
media_json = case
|
|
156
|
-
when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
|
|
157
|
-
else tweets.media_json
|
|
158
|
-
end,
|
|
159
|
-
quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id),
|
|
160
|
-
bookmarked = tweets.bookmarked,
|
|
161
|
-
liked = tweets.liked
|
|
162
|
-
`,
|
|
163
|
-
);
|
|
164
|
-
const tweetIds: string[] = [];
|
|
165
|
-
|
|
166
|
-
db.transaction(() => {
|
|
167
|
-
const seenAt = new Date().toISOString();
|
|
168
|
-
for (const tweet of payload.data) {
|
|
169
|
-
const author =
|
|
170
|
-
usersById.get(tweet.author_id) ??
|
|
171
|
-
({
|
|
172
|
-
id: tweet.author_id,
|
|
173
|
-
username: `user_${tweet.author_id}`,
|
|
174
|
-
name: `user_${tweet.author_id}`,
|
|
175
|
-
} as const);
|
|
176
|
-
const profile = usersById.has(tweet.author_id)
|
|
177
|
-
? upsertProfileFromXUser(db, author)
|
|
178
|
-
: ensureStubProfileForXUser(db, tweet.author_id);
|
|
179
|
-
const replyToId =
|
|
180
|
-
tweet.referenced_tweets?.find((item) => item.type === "replied_to")
|
|
181
|
-
?.id ?? null;
|
|
182
|
-
const quotedTweetId =
|
|
183
|
-
tweet.referenced_tweets?.find((item) => item.type === "quoted")?.id ??
|
|
184
|
-
null;
|
|
185
|
-
upsertTweet.run(
|
|
186
|
-
tweet.id,
|
|
187
|
-
accountId,
|
|
188
|
-
profile.profile.id,
|
|
189
|
-
tweet.text,
|
|
190
|
-
tweet.created_at,
|
|
191
|
-
replyToId,
|
|
192
|
-
Number(tweet.public_metrics?.like_count ?? 0),
|
|
193
|
-
countTweetMedia(tweet),
|
|
194
|
-
JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
|
|
195
|
-
buildMediaJsonFromIncludes(tweet, payload.includes?.media),
|
|
196
|
-
quotedTweetId,
|
|
197
|
-
);
|
|
198
|
-
upsertTweetAccountEdge(db, {
|
|
199
|
-
accountId,
|
|
200
|
-
tweetId: tweet.id,
|
|
201
|
-
kind: "search",
|
|
202
|
-
source,
|
|
203
|
-
seenAt,
|
|
204
|
-
rawJson: JSON.stringify(tweet),
|
|
205
|
-
});
|
|
206
|
-
replaceTweetFts(db, tweet.id, tweet.text);
|
|
207
|
-
tweetIds.push(tweet.id);
|
|
208
|
-
}
|
|
209
|
-
})();
|
|
210
|
-
|
|
211
|
-
return tweetIds;
|
|
94
|
+
return ingestTweetPayload(db, {
|
|
95
|
+
accountId,
|
|
96
|
+
payload,
|
|
97
|
+
kind: "search",
|
|
98
|
+
edgeKind: "search",
|
|
99
|
+
source,
|
|
100
|
+
});
|
|
212
101
|
}
|
|
213
102
|
|
|
214
103
|
function toMentionsResponse(payload: XurlTweetsResponse): XurlMentionsResponse {
|
|
@@ -302,7 +191,7 @@ function fetchBirdSearchEffect({
|
|
|
302
191
|
limit: number;
|
|
303
192
|
maxPages: number;
|
|
304
193
|
}) {
|
|
305
|
-
return
|
|
194
|
+
return liveTransportGateway.bird.searchTweets(query, {
|
|
306
195
|
maxResults: Math.min(limit, XURL_PAGE_SIZE),
|
|
307
196
|
all: maxPages > 1 || limit > XURL_PAGE_SIZE,
|
|
308
197
|
maxPages,
|
|
@@ -332,13 +221,16 @@ function fetchXurlSearchEffect({
|
|
|
332
221
|
limit -
|
|
333
222
|
responses.reduce((total, response) => total + response.data.length, 0);
|
|
334
223
|
if (remaining <= 0) break;
|
|
335
|
-
const response = yield*
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
224
|
+
const response = yield* liveTransportGateway.xurl.searchRecentTweets(
|
|
225
|
+
query,
|
|
226
|
+
{
|
|
227
|
+
maxResults: Math.max(10, Math.min(XURL_PAGE_SIZE, remaining)),
|
|
228
|
+
paginationToken: nextToken,
|
|
229
|
+
startTime: since,
|
|
230
|
+
endTime: until,
|
|
231
|
+
timeoutMs,
|
|
232
|
+
},
|
|
233
|
+
);
|
|
342
234
|
responses.push(toMentionsResponse(response));
|
|
343
235
|
nextToken =
|
|
344
236
|
typeof response.meta?.next_token === "string"
|
|
@@ -368,42 +260,44 @@ function runModeEffect(
|
|
|
368
260
|
return Effect.gen(function* () {
|
|
369
261
|
const db = getNativeDb();
|
|
370
262
|
const key = cacheKey({ ...options, mode });
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
).pipe(
|
|
263
|
+
const fetch =
|
|
264
|
+
mode === "bird"
|
|
265
|
+
? fetchBirdSearchEffect(options).pipe(Effect.mapError(toError))
|
|
266
|
+
: fetchXurlSearchEffect(options);
|
|
267
|
+
const syncResult = yield* runCachedLiveSyncEffect({
|
|
268
|
+
db,
|
|
269
|
+
cacheKey: key,
|
|
270
|
+
refresh: options.refresh,
|
|
271
|
+
cacheTtlMs: options.cacheTtlMs,
|
|
272
|
+
transports: [
|
|
273
|
+
{
|
|
274
|
+
source: mode,
|
|
275
|
+
fetch: fetch.pipe(
|
|
385
276
|
Effect.map((response) => limitResponse(response, options.limit)),
|
|
386
|
-
)
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
277
|
+
),
|
|
278
|
+
},
|
|
279
|
+
],
|
|
280
|
+
persistLive: (writeDb, payload, source) =>
|
|
281
|
+
mergeTweetSearchIntoLocalStore(
|
|
282
|
+
writeDb,
|
|
283
|
+
options.accountId,
|
|
284
|
+
payload,
|
|
285
|
+
source,
|
|
286
|
+
),
|
|
287
|
+
persistCached: (writeDb, payload) =>
|
|
288
|
+
mergeTweetSearchIntoLocalStore(
|
|
289
|
+
writeDb,
|
|
290
|
+
options.accountId,
|
|
291
|
+
payload,
|
|
292
|
+
"cache",
|
|
293
|
+
),
|
|
294
|
+
});
|
|
295
|
+
const { payload, source } = syncResult;
|
|
296
|
+
const tweetIds = syncResult.persisted ?? [];
|
|
400
297
|
|
|
401
298
|
return {
|
|
402
299
|
ok: true,
|
|
403
|
-
source
|
|
404
|
-
!options.refresh && cached && ageMs <= options.cacheTtlMs
|
|
405
|
-
? "cache"
|
|
406
|
-
: mode,
|
|
300
|
+
source,
|
|
407
301
|
accountId: options.accountId,
|
|
408
302
|
query: options.query,
|
|
409
303
|
count: tweetIds.length,
|
|
@@ -479,9 +373,11 @@ export function syncTweetSearchEffect({
|
|
|
479
373
|
const normalizedUntil = yield* trySync(() =>
|
|
480
374
|
normalizeTime(until, "--until"),
|
|
481
375
|
);
|
|
482
|
-
const ttlMs = normalizeCacheTtlMs(cacheTtlMs);
|
|
376
|
+
const ttlMs = normalizeCacheTtlMs(cacheTtlMs, DEFAULT_CACHE_TTL_MS);
|
|
483
377
|
const db = getNativeDb();
|
|
484
|
-
const resolvedAccount = yield* trySync(() =>
|
|
378
|
+
const resolvedAccount = yield* trySync(() =>
|
|
379
|
+
resolveLiveSyncAccount(db, account),
|
|
380
|
+
);
|
|
485
381
|
const accountId = resolvedAccount.accountId;
|
|
486
382
|
if (mode === "local") {
|
|
487
383
|
return {
|
package/src/lib/web-sync.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import { Effect } from "effect";
|
|
3
3
|
import { maybeAutoSyncBackupEffect } from "./backup";
|
|
4
|
-
import { getBirdclawPaths } from "./config";
|
|
5
4
|
import { syncDirectMessagesViaCachedBirdEffect } from "./dms-live";
|
|
6
5
|
import { runEffectBackground, runEffectPromise } from "./effect-runtime";
|
|
7
6
|
import { syncMentionThreadsEffect } from "./mention-threads-live";
|
|
8
7
|
import { syncMentionsEffect } from "./mentions-live";
|
|
8
|
+
import {
|
|
9
|
+
defaultServerRuntimeServices,
|
|
10
|
+
type ServerRuntimeServices,
|
|
11
|
+
} from "./server-runtime-services";
|
|
9
12
|
import NativeSqliteDatabase from "./sqlite";
|
|
10
13
|
import { syncTimelineCollectionEffect } from "./timeline-collections-live";
|
|
11
14
|
import { syncHomeTimelineEffect } from "./timeline-live";
|
|
@@ -69,6 +72,7 @@ interface WebSyncPlan {
|
|
|
69
72
|
run: (
|
|
70
73
|
accountId: string | undefined,
|
|
71
74
|
options: WebSyncOptions,
|
|
75
|
+
runtime: ServerRuntimeServices,
|
|
72
76
|
) => Effect.Effect<WebSyncStep[], unknown>;
|
|
73
77
|
}
|
|
74
78
|
|
|
@@ -132,12 +136,12 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
|
|
|
132
136
|
timeline: {
|
|
133
137
|
label: "Home timeline",
|
|
134
138
|
accountAware: true,
|
|
135
|
-
run: (account) =>
|
|
139
|
+
run: (account, _options, runtime) =>
|
|
136
140
|
Effect.gen(function* () {
|
|
137
141
|
const result = yield* syncHomeTimelineEffect({
|
|
138
142
|
account,
|
|
139
143
|
mode:
|
|
140
|
-
!account || account === resolveDefaultSyncAccountId()
|
|
144
|
+
!account || account === resolveDefaultSyncAccountId(runtime)
|
|
141
145
|
? "auto"
|
|
142
146
|
: "xurl",
|
|
143
147
|
limit: 100,
|
|
@@ -201,12 +205,14 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
|
|
|
201
205
|
likes: {
|
|
202
206
|
label: "Likes",
|
|
203
207
|
accountAware: true,
|
|
204
|
-
run: (account
|
|
208
|
+
run: (account, _options, runtime) =>
|
|
209
|
+
syncSavedCollection("likes", account, runtime),
|
|
205
210
|
},
|
|
206
211
|
bookmarks: {
|
|
207
212
|
label: "Bookmarks",
|
|
208
213
|
accountAware: true,
|
|
209
|
-
run: (account
|
|
214
|
+
run: (account, _options, runtime) =>
|
|
215
|
+
syncSavedCollection("bookmarks", account, runtime),
|
|
210
216
|
},
|
|
211
217
|
dms: {
|
|
212
218
|
label: "Direct messages",
|
|
@@ -242,10 +248,11 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
|
|
|
242
248
|
function syncSavedCollection(
|
|
243
249
|
kind: "likes" | "bookmarks",
|
|
244
250
|
account: string | undefined,
|
|
251
|
+
runtime: ServerRuntimeServices,
|
|
245
252
|
): Effect.Effect<WebSyncStep[], unknown> {
|
|
246
253
|
return Effect.gen(function* () {
|
|
247
254
|
const isNonDefaultAccount =
|
|
248
|
-
account !== undefined && account !== resolveDefaultSyncAccountId();
|
|
255
|
+
account !== undefined && account !== resolveDefaultSyncAccountId(runtime);
|
|
249
256
|
const result = yield* syncTimelineCollectionEffect({
|
|
250
257
|
kind,
|
|
251
258
|
account,
|
|
@@ -270,13 +277,14 @@ export function performWebSyncEffect(
|
|
|
270
277
|
kind: WebSyncKind,
|
|
271
278
|
accountId?: string,
|
|
272
279
|
options: WebSyncOptions = {},
|
|
280
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
273
281
|
) {
|
|
274
282
|
return Effect.gen(function* () {
|
|
275
|
-
const startedAt =
|
|
276
|
-
const steps = yield* WEB_SYNC_PLANS[kind].run(accountId, options);
|
|
283
|
+
const startedAt = runtime.now().toISOString();
|
|
284
|
+
const steps = yield* WEB_SYNC_PLANS[kind].run(accountId, options, runtime);
|
|
277
285
|
|
|
278
286
|
const backup = yield* maybeAutoSyncBackupEffect();
|
|
279
|
-
const finishedAt =
|
|
287
|
+
const finishedAt = runtime.now().toISOString();
|
|
280
288
|
return {
|
|
281
289
|
ok: true,
|
|
282
290
|
kind,
|
|
@@ -290,12 +298,12 @@ export function performWebSyncEffect(
|
|
|
290
298
|
});
|
|
291
299
|
}
|
|
292
300
|
|
|
293
|
-
function createWebSyncJobId(kind: WebSyncKind) {
|
|
294
|
-
return `sync_${kind}_${
|
|
301
|
+
function createWebSyncJobId(kind: WebSyncKind, runtime: ServerRuntimeServices) {
|
|
302
|
+
return `sync_${kind}_${runtime.now().getTime().toString(36)}_${runtime.random().toString(36).slice(2, 8)}`;
|
|
295
303
|
}
|
|
296
304
|
|
|
297
|
-
function resolveDefaultSyncAccountId() {
|
|
298
|
-
const dbPath =
|
|
305
|
+
function resolveDefaultSyncAccountId(runtime: ServerRuntimeServices) {
|
|
306
|
+
const dbPath = runtime.getPaths().dbPath;
|
|
299
307
|
if (!existsSync(dbPath)) {
|
|
300
308
|
return "acct_primary";
|
|
301
309
|
}
|
|
@@ -336,12 +344,13 @@ function getRunningSyncKey(
|
|
|
336
344
|
kind: WebSyncKind,
|
|
337
345
|
accountId: string | undefined,
|
|
338
346
|
options: WebSyncOptions = {},
|
|
347
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
339
348
|
) {
|
|
340
349
|
if (!WEB_SYNC_PLANS[kind].accountAware) {
|
|
341
350
|
const optionKey = serializeSyncOptions(kind, options);
|
|
342
351
|
return optionKey ? `${kind}:${optionKey}` : kind;
|
|
343
352
|
}
|
|
344
|
-
return `${kind}:${accountId ?? resolveDefaultSyncAccountId()}`;
|
|
353
|
+
return `${kind}:${accountId ?? resolveDefaultSyncAccountId(runtime)}`;
|
|
345
354
|
}
|
|
346
355
|
|
|
347
356
|
function getEffectiveAccountId(
|
|
@@ -380,8 +389,9 @@ function toFailedResponse(
|
|
|
380
389
|
startedAt: string,
|
|
381
390
|
error: unknown,
|
|
382
391
|
accountId?: string,
|
|
392
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
383
393
|
): WebSyncResponse {
|
|
384
|
-
const finishedAt =
|
|
394
|
+
const finishedAt = runtime.now().toISOString();
|
|
385
395
|
const message = messageFromError(error);
|
|
386
396
|
return {
|
|
387
397
|
ok: false,
|
|
@@ -399,17 +409,18 @@ export function startWebSync(
|
|
|
399
409
|
kind: WebSyncKind,
|
|
400
410
|
accountId?: string,
|
|
401
411
|
options: WebSyncOptions = {},
|
|
412
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
402
413
|
): WebSyncJobSnapshot {
|
|
403
414
|
const effectiveAccountId = getEffectiveAccountId(kind, accountId);
|
|
404
|
-
const syncKey = getRunningSyncKey(kind, effectiveAccountId, options);
|
|
415
|
+
const syncKey = getRunningSyncKey(kind, effectiveAccountId, options, runtime);
|
|
405
416
|
const current = runningSyncs.get(syncKey);
|
|
406
417
|
if (current) {
|
|
407
418
|
return current;
|
|
408
419
|
}
|
|
409
420
|
|
|
410
|
-
const startedAt =
|
|
421
|
+
const startedAt = runtime.now().toISOString();
|
|
411
422
|
const job: WebSyncJobSnapshot = {
|
|
412
|
-
id: createWebSyncJobId(kind),
|
|
423
|
+
id: createWebSyncJobId(kind, runtime),
|
|
413
424
|
kind,
|
|
414
425
|
...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
|
|
415
426
|
status: "running",
|
|
@@ -420,35 +431,39 @@ export function startWebSync(
|
|
|
420
431
|
webSyncJobKeys.set(job.id, syncKey);
|
|
421
432
|
setJobSnapshot(job);
|
|
422
433
|
|
|
423
|
-
runEffectBackground(
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
434
|
+
runEffectBackground(
|
|
435
|
+
performWebSyncEffect(kind, effectiveAccountId, options, runtime),
|
|
436
|
+
{
|
|
437
|
+
onSuccess: (result) => {
|
|
438
|
+
setJobSnapshot({
|
|
439
|
+
...job,
|
|
440
|
+
status: "succeeded",
|
|
441
|
+
finishedAt: result.finishedAt,
|
|
442
|
+
summary: result.summary,
|
|
443
|
+
inProgress: false,
|
|
444
|
+
result,
|
|
445
|
+
});
|
|
446
|
+
},
|
|
447
|
+
onFailure: (error) => {
|
|
448
|
+
const result = toFailedResponse(
|
|
449
|
+
kind,
|
|
450
|
+
startedAt,
|
|
451
|
+
error,
|
|
452
|
+
effectiveAccountId,
|
|
453
|
+
runtime,
|
|
454
|
+
);
|
|
455
|
+
setJobSnapshot({
|
|
456
|
+
...job,
|
|
457
|
+
status: "failed",
|
|
458
|
+
finishedAt: result.finishedAt,
|
|
459
|
+
summary: result.summary,
|
|
460
|
+
inProgress: false,
|
|
461
|
+
result,
|
|
462
|
+
error: result.error,
|
|
463
|
+
});
|
|
464
|
+
},
|
|
450
465
|
},
|
|
451
|
-
|
|
466
|
+
);
|
|
452
467
|
|
|
453
468
|
return job;
|
|
454
469
|
}
|
|
@@ -461,13 +476,14 @@ export function runWebSyncEffect(
|
|
|
461
476
|
kind: WebSyncKind,
|
|
462
477
|
accountId?: string,
|
|
463
478
|
options: WebSyncOptions = {},
|
|
479
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
464
480
|
): Effect.Effect<WebSyncResponse, Error> {
|
|
465
481
|
return Effect.gen(function* () {
|
|
466
482
|
const effectiveAccountId = getEffectiveAccountId(kind, accountId);
|
|
467
483
|
const current = runningSyncs.get(
|
|
468
|
-
getRunningSyncKey(kind, effectiveAccountId, options),
|
|
484
|
+
getRunningSyncKey(kind, effectiveAccountId, options, runtime),
|
|
469
485
|
);
|
|
470
|
-
const startedAt =
|
|
486
|
+
const startedAt = runtime.now().toISOString();
|
|
471
487
|
if (current) {
|
|
472
488
|
return {
|
|
473
489
|
ok: false,
|
|
@@ -480,7 +496,7 @@ export function runWebSyncEffect(
|
|
|
480
496
|
} satisfies WebSyncResponse;
|
|
481
497
|
}
|
|
482
498
|
|
|
483
|
-
const job = startWebSync(kind, effectiveAccountId, options);
|
|
499
|
+
const job = startWebSync(kind, effectiveAccountId, options, runtime);
|
|
484
500
|
while (job.inProgress) {
|
|
485
501
|
yield* Effect.sleep(25);
|
|
486
502
|
const latest = getWebSyncJob(job.id);
|
|
@@ -501,8 +517,9 @@ export function runWebSync(
|
|
|
501
517
|
kind: WebSyncKind,
|
|
502
518
|
accountId?: string,
|
|
503
519
|
options: WebSyncOptions = {},
|
|
520
|
+
runtime: ServerRuntimeServices = defaultServerRuntimeServices,
|
|
504
521
|
): Promise<WebSyncResponse> {
|
|
505
|
-
return runEffectPromise(runWebSyncEffect(kind, accountId, options));
|
|
522
|
+
return runEffectPromise(runWebSyncEffect(kind, accountId, options, runtime));
|
|
506
523
|
}
|
|
507
524
|
|
|
508
525
|
export function clearWebSyncLocksForTests() {
|