birdclaw 0.8.2 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/package.json +2 -1
  3. package/src/cli/command-context.ts +17 -0
  4. package/src/cli/register-analysis.ts +500 -0
  5. package/src/cli/register-compose.ts +40 -0
  6. package/src/cli/register-graph.ts +132 -0
  7. package/src/cli/register-inbox.ts +41 -0
  8. package/src/cli/register-storage.ts +106 -0
  9. package/src/cli.ts +30 -750
  10. package/src/components/AccountSwitcher.tsx +7 -15
  11. package/src/components/AvatarChip.tsx +1 -1
  12. package/src/components/AvatarPreload.ts +149 -0
  13. package/src/components/MarkdownCitations.tsx +680 -0
  14. package/src/components/MarkdownViewer.tsx +8 -674
  15. package/src/components/ProfileAnalysisClient.ts +191 -0
  16. package/src/components/ProfileAnalysisStream.tsx +16 -185
  17. package/src/components/ProfilePreview.tsx +2 -0
  18. package/src/components/links-controller.ts +162 -0
  19. package/src/components/links-model.ts +198 -0
  20. package/src/components/network-map-controller.ts +84 -0
  21. package/src/components/network-map-model.ts +255 -0
  22. package/src/components/useTimelineRouteData.ts +105 -235
  23. package/src/lib/analysis-runtime.ts +238 -0
  24. package/src/lib/api-client.ts +16 -215
  25. package/src/lib/api-contracts.ts +328 -0
  26. package/src/lib/archive-import-plan.ts +102 -0
  27. package/src/lib/archive-import.ts +170 -239
  28. package/src/lib/authored-live.ts +75 -120
  29. package/src/lib/backup.ts +335 -424
  30. package/src/lib/blocks-write.ts +30 -26
  31. package/src/lib/blocks.ts +18 -20
  32. package/src/lib/database-metrics.ts +88 -0
  33. package/src/lib/database-migrations.ts +34 -0
  34. package/src/lib/database-schema.ts +312 -0
  35. package/src/lib/database-writer.ts +69 -0
  36. package/src/lib/db.ts +84 -330
  37. package/src/lib/dm-read-model.ts +533 -0
  38. package/src/lib/dms-live.ts +34 -97
  39. package/src/lib/follow-graph.ts +17 -27
  40. package/src/lib/import-repository.ts +138 -0
  41. package/src/lib/inbox.ts +2 -1
  42. package/src/lib/live-sync-engine.ts +209 -0
  43. package/src/lib/live-transport-gateway.ts +128 -0
  44. package/src/lib/mention-threads-live.ts +90 -177
  45. package/src/lib/mentions-export.ts +1 -1
  46. package/src/lib/mentions-live.ts +57 -181
  47. package/src/lib/moderation-target.ts +15 -4
  48. package/src/lib/moderation-write.ts +1 -1
  49. package/src/lib/mutes-write.ts +30 -26
  50. package/src/lib/openai-response-runtime.ts +251 -0
  51. package/src/lib/paginated-sync.ts +93 -0
  52. package/src/lib/period-digest.ts +116 -304
  53. package/src/lib/profile-analysis.ts +36 -110
  54. package/src/lib/queries.ts +6 -2381
  55. package/src/lib/query-actions.ts +437 -0
  56. package/src/lib/query-client.tsx +47 -0
  57. package/src/lib/query-read-model-shared.ts +52 -0
  58. package/src/lib/query-read-models.ts +5 -0
  59. package/src/lib/query-resource.ts +41 -0
  60. package/src/lib/query-status.ts +164 -0
  61. package/src/lib/research.ts +1 -1
  62. package/src/lib/runtime-services.ts +20 -0
  63. package/src/lib/search-discussion.ts +75 -279
  64. package/src/lib/server-runtime-services.ts +30 -0
  65. package/src/lib/sqlite.ts +48 -12
  66. package/src/lib/streaming-ingestion.ts +240 -0
  67. package/src/lib/sync-cache.ts +6 -1
  68. package/src/lib/sync-plan.ts +175 -0
  69. package/src/lib/timeline-collections-live.ts +83 -257
  70. package/src/lib/timeline-live.ts +86 -236
  71. package/src/lib/timeline-read-model.ts +1191 -0
  72. package/src/lib/tweet-repository.ts +156 -0
  73. package/src/lib/tweet-search-live.ts +63 -167
  74. package/src/lib/web-sync.ts +67 -50
  75. package/src/lib/whois.ts +2 -1
  76. package/src/routes/__root.tsx +11 -8
  77. package/src/routes/api/action.tsx +1 -1
  78. package/src/routes/api/conversation.tsx +1 -1
  79. package/src/routes/api/query.tsx +32 -26
  80. package/src/routes/api/status.tsx +6 -4
  81. package/src/routes/api/sync.tsx +5 -2
  82. package/src/routes/blocks.tsx +97 -131
  83. package/src/routes/data-sources.tsx +17 -25
  84. package/src/routes/dms.tsx +167 -184
  85. package/src/routes/inbox.tsx +63 -57
  86. package/src/routes/links.tsx +31 -394
  87. package/src/routes/network-map.tsx +41 -344
  88. package/src/routes/rate-limits.tsx +17 -21
  89. package/src/lib/client-cache.ts +0 -109
@@ -0,0 +1,156 @@
1
+ import type { Database } from "./sqlite";
2
+ import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
3
+ import { tweetEntitiesFromXurl } from "./tweet-render";
4
+ import type { XurlMentionData, XurlMentionsResponse } from "./types";
5
+ import {
6
+ type TweetAccountEdgeKind,
7
+ upsertTweetAccountEdge,
8
+ } from "./tweet-account-edges";
9
+ import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
10
+
11
+ export type CanonicalTweetKind =
12
+ | "authored"
13
+ | "bookmark"
14
+ | "home"
15
+ | "like"
16
+ | "mention"
17
+ | "search"
18
+ | "thread"
19
+ | "thread_context";
20
+
21
+ export interface IngestTweetPayloadOptions {
22
+ accountId: string;
23
+ payload: XurlMentionsResponse;
24
+ kind: CanonicalTweetKind;
25
+ source: string;
26
+ edgeKind?: TweetAccountEdgeKind;
27
+ collectionKind?: "likes" | "bookmarks";
28
+ markRepliesAsReplied?: boolean;
29
+ replaceSecondaryKind?: boolean;
30
+ }
31
+
32
+ function getReferencedTweetId(tweet: XurlMentionData, type: string) {
33
+ return (
34
+ tweet.referenced_tweets?.find((item) => item.type === type)?.id ?? null
35
+ );
36
+ }
37
+
38
+ export function replaceTweetFts(db: Database, tweetId: string, text: string) {
39
+ db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
40
+ db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
41
+ tweetId,
42
+ text,
43
+ );
44
+ }
45
+
46
+ export function ingestTweetPayload(
47
+ db: Database,
48
+ {
49
+ accountId,
50
+ payload,
51
+ kind,
52
+ source,
53
+ edgeKind,
54
+ collectionKind,
55
+ markRepliesAsReplied = false,
56
+ replaceSecondaryKind = false,
57
+ }: IngestTweetPayloadOptions,
58
+ ) {
59
+ const usersById = new Map(
60
+ (payload.includes?.users ?? []).map((user) => [user.id, user]),
61
+ );
62
+ const upsertTweet = db.prepare(`
63
+ insert into tweets (
64
+ id, account_id, author_profile_id, kind, text, created_at,
65
+ is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
66
+ entities_json, media_json, quoted_tweet_id
67
+ ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
68
+ on conflict(id) do update set
69
+ account_id = tweets.account_id,
70
+ author_profile_id = excluded.author_profile_id,
71
+ kind = case
72
+ when ? = 1
73
+ and tweets.kind not in ('authored', 'home', 'mention')
74
+ then excluded.kind
75
+ else tweets.kind
76
+ end,
77
+ text = excluded.text,
78
+ created_at = excluded.created_at,
79
+ is_replied = max(tweets.is_replied, excluded.is_replied),
80
+ reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
81
+ like_count = excluded.like_count,
82
+ media_count = max(tweets.media_count, excluded.media_count),
83
+ entities_json = excluded.entities_json,
84
+ media_json = case
85
+ when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
86
+ else tweets.media_json
87
+ end,
88
+ bookmarked = tweets.bookmarked,
89
+ liked = tweets.liked,
90
+ quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id)
91
+ `);
92
+ const upsertCollection = collectionKind
93
+ ? db.prepare(`
94
+ insert into tweet_collections (
95
+ account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
96
+ ) values (?, ?, ?, null, ?, ?, ?)
97
+ on conflict(account_id, tweet_id, kind) do update set
98
+ source = excluded.source,
99
+ raw_json = excluded.raw_json,
100
+ updated_at = excluded.updated_at
101
+ `)
102
+ : undefined;
103
+ const tweetIds: string[] = [];
104
+
105
+ db.transaction(() => {
106
+ const observedAt = new Date().toISOString();
107
+ for (const tweet of payload.data) {
108
+ const author = usersById.get(tweet.author_id);
109
+ const profile = author
110
+ ? upsertProfileFromXUser(db, author)
111
+ : ensureStubProfileForXUser(db, tweet.author_id);
112
+ const replyToId = getReferencedTweetId(tweet, "replied_to");
113
+ const quotedTweetId = getReferencedTweetId(tweet, "quoted");
114
+ upsertTweet.run(
115
+ tweet.id,
116
+ accountId,
117
+ profile.profile.id,
118
+ kind,
119
+ tweet.text,
120
+ tweet.created_at,
121
+ markRepliesAsReplied && replyToId ? 1 : 0,
122
+ replyToId,
123
+ Number(tweet.public_metrics?.like_count ?? 0),
124
+ countTweetMedia(tweet),
125
+ collectionKind === "bookmarks" ? 1 : 0,
126
+ collectionKind === "likes" ? 1 : 0,
127
+ JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
128
+ buildMediaJsonFromIncludes(tweet, payload.includes?.media),
129
+ quotedTweetId,
130
+ replaceSecondaryKind ? 1 : 0,
131
+ );
132
+ if (edgeKind) {
133
+ upsertTweetAccountEdge(db, {
134
+ accountId,
135
+ tweetId: tweet.id,
136
+ kind: edgeKind,
137
+ source,
138
+ seenAt: observedAt,
139
+ rawJson: JSON.stringify(tweet),
140
+ });
141
+ }
142
+ upsertCollection?.run(
143
+ accountId,
144
+ tweet.id,
145
+ collectionKind,
146
+ source,
147
+ JSON.stringify(tweet),
148
+ observedAt,
149
+ );
150
+ replaceTweetFts(db, tweet.id, tweet.text);
151
+ tweetIds.push(tweet.id);
152
+ }
153
+ })();
154
+
155
+ return tweetIds;
156
+ }
@@ -1,15 +1,15 @@
1
1
  import { Effect } from "effect";
2
- import { searchTweetsViaBirdEffect } from "./bird";
3
2
  import type { Database } from "./sqlite";
4
3
  import { getNativeDb } from "./db";
5
4
  import { runEffectPromise } from "./effect-runtime";
6
- import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
7
- import { readSyncCache, writeSyncCache } from "./sync-cache";
8
- import { tweetEntitiesFromXurl } from "./tweet-render";
5
+ import { liveTransportGateway } from "./live-transport-gateway";
6
+ import {
7
+ normalizeCacheTtlMs,
8
+ resolveLiveSyncAccount,
9
+ runCachedLiveSyncEffect,
10
+ } from "./live-sync-engine";
9
11
  import type { XurlMentionsResponse, XurlTweetsResponse } from "./types";
10
- import { upsertTweetAccountEdge } from "./tweet-account-edges";
11
- import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
12
- import { searchRecentTweetsEffect } from "./xurl";
12
+ import { ingestTweetPayload } from "./tweet-repository";
13
13
 
14
14
  export type TweetSearchMode = "auto" | "bird" | "xurl" | "local";
15
15
 
@@ -85,130 +85,19 @@ function normalizeTime(value: string | undefined, optionName: string) {
85
85
  return date.toISOString();
86
86
  }
87
87
 
88
- function normalizeCacheTtlMs(value: number | undefined) {
89
- if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
90
- return DEFAULT_CACHE_TTL_MS;
91
- }
92
- return Math.floor(value);
93
- }
94
-
95
- function resolveAccount(db: Database, accountId?: string) {
96
- const row = accountId
97
- ? (db
98
- .prepare("select id, handle from accounts where id = ?")
99
- .get(accountId) as { id: string; handle: string } | undefined)
100
- : (db
101
- .prepare(
102
- `
103
- select id, handle
104
- from accounts
105
- order by is_default desc, created_at asc
106
- limit 1
107
- `,
108
- )
109
- .get() as { id: string; handle: string } | undefined);
110
-
111
- if (!row) {
112
- throw new Error(`Unknown account: ${accountId ?? "default"}`);
113
- }
114
-
115
- return {
116
- accountId: row.id,
117
- username: row.handle.replace(/^@/, ""),
118
- };
119
- }
120
-
121
- function replaceTweetFts(db: Database, tweetId: string, text: string) {
122
- db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
123
- db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
124
- tweetId,
125
- text,
126
- );
127
- }
128
-
129
88
  function mergeTweetSearchIntoLocalStore(
130
89
  db: Database,
131
90
  accountId: string,
132
91
  payload: XurlMentionsResponse,
133
92
  source: "bird" | "xurl" | "cache",
134
93
  ) {
135
- const usersById = new Map(
136
- (payload.includes?.users ?? []).map((user) => [user.id, user]),
137
- );
138
- const upsertTweet = db.prepare(
139
- `
140
- insert into tweets (
141
- id, account_id, author_profile_id, kind, text, created_at,
142
- is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
143
- entities_json, media_json, quoted_tweet_id
144
- ) values (?, ?, ?, 'search', ?, ?, 0, ?, ?, ?, 0, 0, ?, ?, ?)
145
- on conflict(id) do update set
146
- account_id = tweets.account_id,
147
- author_profile_id = excluded.author_profile_id,
148
- kind = tweets.kind,
149
- text = excluded.text,
150
- created_at = excluded.created_at,
151
- reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
152
- like_count = excluded.like_count,
153
- media_count = max(tweets.media_count, excluded.media_count),
154
- entities_json = excluded.entities_json,
155
- media_json = case
156
- when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
157
- else tweets.media_json
158
- end,
159
- quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id),
160
- bookmarked = tweets.bookmarked,
161
- liked = tweets.liked
162
- `,
163
- );
164
- const tweetIds: string[] = [];
165
-
166
- db.transaction(() => {
167
- const seenAt = new Date().toISOString();
168
- for (const tweet of payload.data) {
169
- const author =
170
- usersById.get(tweet.author_id) ??
171
- ({
172
- id: tweet.author_id,
173
- username: `user_${tweet.author_id}`,
174
- name: `user_${tweet.author_id}`,
175
- } as const);
176
- const profile = usersById.has(tweet.author_id)
177
- ? upsertProfileFromXUser(db, author)
178
- : ensureStubProfileForXUser(db, tweet.author_id);
179
- const replyToId =
180
- tweet.referenced_tweets?.find((item) => item.type === "replied_to")
181
- ?.id ?? null;
182
- const quotedTweetId =
183
- tweet.referenced_tweets?.find((item) => item.type === "quoted")?.id ??
184
- null;
185
- upsertTweet.run(
186
- tweet.id,
187
- accountId,
188
- profile.profile.id,
189
- tweet.text,
190
- tweet.created_at,
191
- replyToId,
192
- Number(tweet.public_metrics?.like_count ?? 0),
193
- countTweetMedia(tweet),
194
- JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
195
- buildMediaJsonFromIncludes(tweet, payload.includes?.media),
196
- quotedTweetId,
197
- );
198
- upsertTweetAccountEdge(db, {
199
- accountId,
200
- tweetId: tweet.id,
201
- kind: "search",
202
- source,
203
- seenAt,
204
- rawJson: JSON.stringify(tweet),
205
- });
206
- replaceTweetFts(db, tweet.id, tweet.text);
207
- tweetIds.push(tweet.id);
208
- }
209
- })();
210
-
211
- return tweetIds;
94
+ return ingestTweetPayload(db, {
95
+ accountId,
96
+ payload,
97
+ kind: "search",
98
+ edgeKind: "search",
99
+ source,
100
+ });
212
101
  }
213
102
 
214
103
  function toMentionsResponse(payload: XurlTweetsResponse): XurlMentionsResponse {
@@ -302,7 +191,7 @@ function fetchBirdSearchEffect({
302
191
  limit: number;
303
192
  maxPages: number;
304
193
  }) {
305
- return searchTweetsViaBirdEffect(query, {
194
+ return liveTransportGateway.bird.searchTweets(query, {
306
195
  maxResults: Math.min(limit, XURL_PAGE_SIZE),
307
196
  all: maxPages > 1 || limit > XURL_PAGE_SIZE,
308
197
  maxPages,
@@ -332,13 +221,16 @@ function fetchXurlSearchEffect({
332
221
  limit -
333
222
  responses.reduce((total, response) => total + response.data.length, 0);
334
223
  if (remaining <= 0) break;
335
- const response = yield* searchRecentTweetsEffect(query, {
336
- maxResults: Math.max(10, Math.min(XURL_PAGE_SIZE, remaining)),
337
- paginationToken: nextToken,
338
- startTime: since,
339
- endTime: until,
340
- timeoutMs,
341
- });
224
+ const response = yield* liveTransportGateway.xurl.searchRecentTweets(
225
+ query,
226
+ {
227
+ maxResults: Math.max(10, Math.min(XURL_PAGE_SIZE, remaining)),
228
+ paginationToken: nextToken,
229
+ startTime: since,
230
+ endTime: until,
231
+ timeoutMs,
232
+ },
233
+ );
342
234
  responses.push(toMentionsResponse(response));
343
235
  nextToken =
344
236
  typeof response.meta?.next_token === "string"
@@ -368,42 +260,44 @@ function runModeEffect(
368
260
  return Effect.gen(function* () {
369
261
  const db = getNativeDb();
370
262
  const key = cacheKey({ ...options, mode });
371
- const cached = yield* trySync(() =>
372
- readSyncCache<XurlMentionsResponse>(key, db),
373
- );
374
- const ageMs = cached
375
- ? Date.now() - new Date(cached.updatedAt).getTime()
376
- : Number.POSITIVE_INFINITY;
377
- const payload =
378
- !options.refresh && cached && ageMs <= options.cacheTtlMs
379
- ? cached.value
380
- : yield* (
381
- mode === "bird"
382
- ? fetchBirdSearchEffect(options).pipe(Effect.mapError(toError))
383
- : fetchXurlSearchEffect(options)
384
- ).pipe(
263
+ const fetch =
264
+ mode === "bird"
265
+ ? fetchBirdSearchEffect(options).pipe(Effect.mapError(toError))
266
+ : fetchXurlSearchEffect(options);
267
+ const syncResult = yield* runCachedLiveSyncEffect({
268
+ db,
269
+ cacheKey: key,
270
+ refresh: options.refresh,
271
+ cacheTtlMs: options.cacheTtlMs,
272
+ transports: [
273
+ {
274
+ source: mode,
275
+ fetch: fetch.pipe(
385
276
  Effect.map((response) => limitResponse(response, options.limit)),
386
- );
387
- if (!cached || options.refresh || ageMs > options.cacheTtlMs) {
388
- yield* trySync(() => writeSyncCache(key, payload, db));
389
- }
390
- const tweetIds = yield* trySync(() =>
391
- mergeTweetSearchIntoLocalStore(
392
- db,
393
- options.accountId,
394
- payload,
395
- !options.refresh && cached && ageMs <= options.cacheTtlMs
396
- ? "cache"
397
- : mode,
398
- ),
399
- );
277
+ ),
278
+ },
279
+ ],
280
+ persistLive: (writeDb, payload, source) =>
281
+ mergeTweetSearchIntoLocalStore(
282
+ writeDb,
283
+ options.accountId,
284
+ payload,
285
+ source,
286
+ ),
287
+ persistCached: (writeDb, payload) =>
288
+ mergeTweetSearchIntoLocalStore(
289
+ writeDb,
290
+ options.accountId,
291
+ payload,
292
+ "cache",
293
+ ),
294
+ });
295
+ const { payload, source } = syncResult;
296
+ const tweetIds = syncResult.persisted ?? [];
400
297
 
401
298
  return {
402
299
  ok: true,
403
- source:
404
- !options.refresh && cached && ageMs <= options.cacheTtlMs
405
- ? "cache"
406
- : mode,
300
+ source,
407
301
  accountId: options.accountId,
408
302
  query: options.query,
409
303
  count: tweetIds.length,
@@ -479,9 +373,11 @@ export function syncTweetSearchEffect({
479
373
  const normalizedUntil = yield* trySync(() =>
480
374
  normalizeTime(until, "--until"),
481
375
  );
482
- const ttlMs = normalizeCacheTtlMs(cacheTtlMs);
376
+ const ttlMs = normalizeCacheTtlMs(cacheTtlMs, DEFAULT_CACHE_TTL_MS);
483
377
  const db = getNativeDb();
484
- const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
378
+ const resolvedAccount = yield* trySync(() =>
379
+ resolveLiveSyncAccount(db, account),
380
+ );
485
381
  const accountId = resolvedAccount.accountId;
486
382
  if (mode === "local") {
487
383
  return {
@@ -1,11 +1,14 @@
1
1
  import { existsSync } from "node:fs";
2
2
  import { Effect } from "effect";
3
3
  import { maybeAutoSyncBackupEffect } from "./backup";
4
- import { getBirdclawPaths } from "./config";
5
4
  import { syncDirectMessagesViaCachedBirdEffect } from "./dms-live";
6
5
  import { runEffectBackground, runEffectPromise } from "./effect-runtime";
7
6
  import { syncMentionThreadsEffect } from "./mention-threads-live";
8
7
  import { syncMentionsEffect } from "./mentions-live";
8
+ import {
9
+ defaultServerRuntimeServices,
10
+ type ServerRuntimeServices,
11
+ } from "./server-runtime-services";
9
12
  import NativeSqliteDatabase from "./sqlite";
10
13
  import { syncTimelineCollectionEffect } from "./timeline-collections-live";
11
14
  import { syncHomeTimelineEffect } from "./timeline-live";
@@ -69,6 +72,7 @@ interface WebSyncPlan {
69
72
  run: (
70
73
  accountId: string | undefined,
71
74
  options: WebSyncOptions,
75
+ runtime: ServerRuntimeServices,
72
76
  ) => Effect.Effect<WebSyncStep[], unknown>;
73
77
  }
74
78
 
@@ -132,12 +136,12 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
132
136
  timeline: {
133
137
  label: "Home timeline",
134
138
  accountAware: true,
135
- run: (account) =>
139
+ run: (account, _options, runtime) =>
136
140
  Effect.gen(function* () {
137
141
  const result = yield* syncHomeTimelineEffect({
138
142
  account,
139
143
  mode:
140
- !account || account === resolveDefaultSyncAccountId()
144
+ !account || account === resolveDefaultSyncAccountId(runtime)
141
145
  ? "auto"
142
146
  : "xurl",
143
147
  limit: 100,
@@ -201,12 +205,14 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
201
205
  likes: {
202
206
  label: "Likes",
203
207
  accountAware: true,
204
- run: (account) => syncSavedCollection("likes", account),
208
+ run: (account, _options, runtime) =>
209
+ syncSavedCollection("likes", account, runtime),
205
210
  },
206
211
  bookmarks: {
207
212
  label: "Bookmarks",
208
213
  accountAware: true,
209
- run: (account) => syncSavedCollection("bookmarks", account),
214
+ run: (account, _options, runtime) =>
215
+ syncSavedCollection("bookmarks", account, runtime),
210
216
  },
211
217
  dms: {
212
218
  label: "Direct messages",
@@ -242,10 +248,11 @@ const WEB_SYNC_PLANS: Record<WebSyncKind, WebSyncPlan> = {
242
248
  function syncSavedCollection(
243
249
  kind: "likes" | "bookmarks",
244
250
  account: string | undefined,
251
+ runtime: ServerRuntimeServices,
245
252
  ): Effect.Effect<WebSyncStep[], unknown> {
246
253
  return Effect.gen(function* () {
247
254
  const isNonDefaultAccount =
248
- account !== undefined && account !== resolveDefaultSyncAccountId();
255
+ account !== undefined && account !== resolveDefaultSyncAccountId(runtime);
249
256
  const result = yield* syncTimelineCollectionEffect({
250
257
  kind,
251
258
  account,
@@ -270,13 +277,14 @@ export function performWebSyncEffect(
270
277
  kind: WebSyncKind,
271
278
  accountId?: string,
272
279
  options: WebSyncOptions = {},
280
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
273
281
  ) {
274
282
  return Effect.gen(function* () {
275
- const startedAt = new Date().toISOString();
276
- const steps = yield* WEB_SYNC_PLANS[kind].run(accountId, options);
283
+ const startedAt = runtime.now().toISOString();
284
+ const steps = yield* WEB_SYNC_PLANS[kind].run(accountId, options, runtime);
277
285
 
278
286
  const backup = yield* maybeAutoSyncBackupEffect();
279
- const finishedAt = new Date().toISOString();
287
+ const finishedAt = runtime.now().toISOString();
280
288
  return {
281
289
  ok: true,
282
290
  kind,
@@ -290,12 +298,12 @@ export function performWebSyncEffect(
290
298
  });
291
299
  }
292
300
 
293
- function createWebSyncJobId(kind: WebSyncKind) {
294
- return `sync_${kind}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
301
+ function createWebSyncJobId(kind: WebSyncKind, runtime: ServerRuntimeServices) {
302
+ return `sync_${kind}_${runtime.now().getTime().toString(36)}_${runtime.random().toString(36).slice(2, 8)}`;
295
303
  }
296
304
 
297
- function resolveDefaultSyncAccountId() {
298
- const dbPath = getBirdclawPaths().dbPath;
305
+ function resolveDefaultSyncAccountId(runtime: ServerRuntimeServices) {
306
+ const dbPath = runtime.getPaths().dbPath;
299
307
  if (!existsSync(dbPath)) {
300
308
  return "acct_primary";
301
309
  }
@@ -336,12 +344,13 @@ function getRunningSyncKey(
336
344
  kind: WebSyncKind,
337
345
  accountId: string | undefined,
338
346
  options: WebSyncOptions = {},
347
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
339
348
  ) {
340
349
  if (!WEB_SYNC_PLANS[kind].accountAware) {
341
350
  const optionKey = serializeSyncOptions(kind, options);
342
351
  return optionKey ? `${kind}:${optionKey}` : kind;
343
352
  }
344
- return `${kind}:${accountId ?? resolveDefaultSyncAccountId()}`;
353
+ return `${kind}:${accountId ?? resolveDefaultSyncAccountId(runtime)}`;
345
354
  }
346
355
 
347
356
  function getEffectiveAccountId(
@@ -380,8 +389,9 @@ function toFailedResponse(
380
389
  startedAt: string,
381
390
  error: unknown,
382
391
  accountId?: string,
392
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
383
393
  ): WebSyncResponse {
384
- const finishedAt = new Date().toISOString();
394
+ const finishedAt = runtime.now().toISOString();
385
395
  const message = messageFromError(error);
386
396
  return {
387
397
  ok: false,
@@ -399,17 +409,18 @@ export function startWebSync(
399
409
  kind: WebSyncKind,
400
410
  accountId?: string,
401
411
  options: WebSyncOptions = {},
412
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
402
413
  ): WebSyncJobSnapshot {
403
414
  const effectiveAccountId = getEffectiveAccountId(kind, accountId);
404
- const syncKey = getRunningSyncKey(kind, effectiveAccountId, options);
415
+ const syncKey = getRunningSyncKey(kind, effectiveAccountId, options, runtime);
405
416
  const current = runningSyncs.get(syncKey);
406
417
  if (current) {
407
418
  return current;
408
419
  }
409
420
 
410
- const startedAt = new Date().toISOString();
421
+ const startedAt = runtime.now().toISOString();
411
422
  const job: WebSyncJobSnapshot = {
412
- id: createWebSyncJobId(kind),
423
+ id: createWebSyncJobId(kind, runtime),
413
424
  kind,
414
425
  ...(effectiveAccountId ? { accountId: effectiveAccountId } : {}),
415
426
  status: "running",
@@ -420,35 +431,39 @@ export function startWebSync(
420
431
  webSyncJobKeys.set(job.id, syncKey);
421
432
  setJobSnapshot(job);
422
433
 
423
- runEffectBackground(performWebSyncEffect(kind, effectiveAccountId, options), {
424
- onSuccess: (result) => {
425
- setJobSnapshot({
426
- ...job,
427
- status: "succeeded",
428
- finishedAt: result.finishedAt,
429
- summary: result.summary,
430
- inProgress: false,
431
- result,
432
- });
433
- },
434
- onFailure: (error) => {
435
- const result = toFailedResponse(
436
- kind,
437
- startedAt,
438
- error,
439
- effectiveAccountId,
440
- );
441
- setJobSnapshot({
442
- ...job,
443
- status: "failed",
444
- finishedAt: result.finishedAt,
445
- summary: result.summary,
446
- inProgress: false,
447
- result,
448
- error: result.error,
449
- });
434
+ runEffectBackground(
435
+ performWebSyncEffect(kind, effectiveAccountId, options, runtime),
436
+ {
437
+ onSuccess: (result) => {
438
+ setJobSnapshot({
439
+ ...job,
440
+ status: "succeeded",
441
+ finishedAt: result.finishedAt,
442
+ summary: result.summary,
443
+ inProgress: false,
444
+ result,
445
+ });
446
+ },
447
+ onFailure: (error) => {
448
+ const result = toFailedResponse(
449
+ kind,
450
+ startedAt,
451
+ error,
452
+ effectiveAccountId,
453
+ runtime,
454
+ );
455
+ setJobSnapshot({
456
+ ...job,
457
+ status: "failed",
458
+ finishedAt: result.finishedAt,
459
+ summary: result.summary,
460
+ inProgress: false,
461
+ result,
462
+ error: result.error,
463
+ });
464
+ },
450
465
  },
451
- });
466
+ );
452
467
 
453
468
  return job;
454
469
  }
@@ -461,13 +476,14 @@ export function runWebSyncEffect(
461
476
  kind: WebSyncKind,
462
477
  accountId?: string,
463
478
  options: WebSyncOptions = {},
479
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
464
480
  ): Effect.Effect<WebSyncResponse, Error> {
465
481
  return Effect.gen(function* () {
466
482
  const effectiveAccountId = getEffectiveAccountId(kind, accountId);
467
483
  const current = runningSyncs.get(
468
- getRunningSyncKey(kind, effectiveAccountId, options),
484
+ getRunningSyncKey(kind, effectiveAccountId, options, runtime),
469
485
  );
470
- const startedAt = new Date().toISOString();
486
+ const startedAt = runtime.now().toISOString();
471
487
  if (current) {
472
488
  return {
473
489
  ok: false,
@@ -480,7 +496,7 @@ export function runWebSyncEffect(
480
496
  } satisfies WebSyncResponse;
481
497
  }
482
498
 
483
- const job = startWebSync(kind, effectiveAccountId, options);
499
+ const job = startWebSync(kind, effectiveAccountId, options, runtime);
484
500
  while (job.inProgress) {
485
501
  yield* Effect.sleep(25);
486
502
  const latest = getWebSyncJob(job.id);
@@ -501,8 +517,9 @@ export function runWebSync(
501
517
  kind: WebSyncKind,
502
518
  accountId?: string,
503
519
  options: WebSyncOptions = {},
520
+ runtime: ServerRuntimeServices = defaultServerRuntimeServices,
504
521
  ): Promise<WebSyncResponse> {
505
- return runEffectPromise(runWebSyncEffect(kind, accountId, options));
522
+ return runEffectPromise(runWebSyncEffect(kind, accountId, options, runtime));
506
523
  }
507
524
 
508
525
  export function clearWebSyncLocksForTests() {