birdclaw 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +113 -7
- package/bin/birdclaw.mjs +50 -11
- package/package.json +30 -28
- package/playwright.config.ts +1 -0
- package/public/birdclaw-mark.png +0 -0
- package/public/favicon.ico +0 -0
- package/public/logo192.png +0 -0
- package/public/logo512.png +0 -0
- package/public/manifest.json +2 -2
- package/scripts/browser-perf.mjs +399 -0
- package/scripts/build-docs-site.mjs +940 -0
- package/scripts/docs-site-assets.mjs +311 -0
- package/scripts/run-vitest.mjs +21 -0
- package/scripts/sanitize-node-options.mjs +23 -0
- package/scripts/start-test-server.mjs +29 -0
- package/src/cli.ts +496 -19
- package/src/components/AppNav.tsx +66 -29
- package/src/components/AvatarChip.tsx +10 -5
- package/src/components/BrandMark.tsx +67 -0
- package/src/components/ConversationThread.tsx +126 -0
- package/src/components/DmWorkspace.tsx +118 -105
- package/src/components/EmbeddedTweetCard.tsx +20 -14
- package/src/components/FeedState.tsx +147 -0
- package/src/components/InboxCard.tsx +104 -90
- package/src/components/LinkPreviewCard.tsx +270 -0
- package/src/components/ProfilePreview.tsx +8 -3
- package/src/components/SavedTimelineView.tsx +89 -71
- package/src/components/SyncNowButton.tsx +105 -0
- package/src/components/ThemeSlider.tsx +10 -59
- package/src/components/TimelineCard.tsx +326 -86
- package/src/components/TimelineRouteFrame.tsx +156 -0
- package/src/components/TweetMediaGrid.tsx +120 -23
- package/src/components/TweetRichText.tsx +19 -4
- package/src/components/useTimelineRouteData.ts +137 -0
- package/src/lib/api-client.ts +229 -0
- package/src/lib/archive-finder.ts +24 -20
- package/src/lib/archive-import.ts +1582 -67
- package/src/lib/authored-live.ts +1074 -0
- package/src/lib/backup.ts +316 -14
- package/src/lib/bird-actions.ts +1 -10
- package/src/lib/bird-command.ts +57 -0
- package/src/lib/bird.ts +89 -5
- package/src/lib/config.ts +1 -1
- package/src/lib/conversation-surface.ts +174 -0
- package/src/lib/db.ts +193 -4
- package/src/lib/follow-graph.ts +1053 -0
- package/src/lib/link-index.ts +11 -98
- package/src/lib/link-insights.ts +834 -0
- package/src/lib/link-preview-metadata.ts +334 -0
- package/src/lib/media-fetch.ts +787 -0
- package/src/lib/media-includes.ts +165 -0
- package/src/lib/mention-threads-live.ts +535 -43
- package/src/lib/mentions-live.ts +623 -19
- package/src/lib/moderation-target.ts +6 -0
- package/src/lib/profile-hydration.ts +1 -1
- package/src/lib/profile-resolver.ts +115 -1
- package/src/lib/queries.ts +326 -35
- package/src/lib/seed.ts +127 -8
- package/src/lib/timeline-collections-live.ts +145 -22
- package/src/lib/timeline-live.ts +10 -15
- package/src/lib/tweet-account-edges.ts +9 -2
- package/src/lib/tweet-render.ts +97 -0
- package/src/lib/types.ts +185 -2
- package/src/lib/ui.ts +383 -147
- package/src/lib/url-expansion-store.ts +110 -0
- package/src/lib/url-expansion.ts +20 -3
- package/src/lib/web-sync.ts +443 -0
- package/src/lib/x-profile.ts +7 -2
- package/src/lib/xurl.ts +296 -12
- package/src/routeTree.gen.ts +126 -0
- package/src/routes/__root.tsx +7 -3
- package/src/routes/api/conversation.tsx +34 -0
- package/src/routes/api/link-insights.tsx +79 -0
- package/src/routes/api/link-preview.tsx +43 -0
- package/src/routes/api/profile-hydrate.tsx +51 -0
- package/src/routes/api/sync.tsx +59 -0
- package/src/routes/blocks.tsx +111 -86
- package/src/routes/dms.tsx +172 -87
- package/src/routes/inbox.tsx +98 -86
- package/src/routes/index.tsx +22 -115
- package/src/routes/links.tsx +928 -0
- package/src/routes/mentions.tsx +22 -115
- package/src/styles.css +169 -43
- package/vite.config.ts +8 -0
package/src/lib/backup.ts
CHANGED
|
@@ -184,9 +184,9 @@ function getExportRowSets(db: Database) {
|
|
|
184
184
|
rows: rowsForQuery(
|
|
185
185
|
db,
|
|
186
186
|
`
|
|
187
|
-
select id, handle, display_name, bio, followers_count,
|
|
188
|
-
|
|
189
|
-
raw_json, created_at
|
|
187
|
+
select id, handle, display_name, bio, followers_count,
|
|
188
|
+
following_count, public_metrics_json, avatar_hue, avatar_url,
|
|
189
|
+
location, url, verified_type, entities_json, raw_json, created_at
|
|
190
190
|
from profiles
|
|
191
191
|
order by id
|
|
192
192
|
`,
|
|
@@ -287,6 +287,31 @@ function getExportRowSets(db: Database) {
|
|
|
287
287
|
is_replied, media_count
|
|
288
288
|
from dm_messages
|
|
289
289
|
order by conversation_id, created_at, id
|
|
290
|
+
`,
|
|
291
|
+
),
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
logicalName: "url_expansions",
|
|
295
|
+
rows: rowsForQuery(
|
|
296
|
+
db,
|
|
297
|
+
`
|
|
298
|
+
select short_url, expanded_url, final_url, status, expanded_tweet_id,
|
|
299
|
+
expanded_handle, title, description, image_url, site_name, error,
|
|
300
|
+
source, updated_at
|
|
301
|
+
from url_expansions
|
|
302
|
+
order by short_url
|
|
303
|
+
`,
|
|
304
|
+
),
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
logicalName: "link_occurrences",
|
|
308
|
+
rows: rowsForQuery(
|
|
309
|
+
db,
|
|
310
|
+
`
|
|
311
|
+
select source_kind, source_id, source_position, short_url, account_id,
|
|
312
|
+
conversation_id, direction, created_at
|
|
313
|
+
from link_occurrences
|
|
314
|
+
order by source_kind, source_id, source_position, short_url
|
|
290
315
|
`,
|
|
291
316
|
),
|
|
292
317
|
},
|
|
@@ -331,6 +356,53 @@ function getExportRowSets(db: Database) {
|
|
|
331
356
|
select entity_kind, entity_id, model, score, summary, reasoning, updated_at
|
|
332
357
|
from ai_scores
|
|
333
358
|
order by entity_kind, entity_id, model
|
|
359
|
+
`,
|
|
360
|
+
),
|
|
361
|
+
},
|
|
362
|
+
{
|
|
363
|
+
logicalName: "follow_snapshots",
|
|
364
|
+
rows: rowsForQuery(
|
|
365
|
+
db,
|
|
366
|
+
`
|
|
367
|
+
select id, account_id, direction, source, status, page_count,
|
|
368
|
+
result_count, started_at, completed_at, raw_meta_json
|
|
369
|
+
from follow_snapshots
|
|
370
|
+
order by account_id, direction, completed_at, id
|
|
371
|
+
`,
|
|
372
|
+
),
|
|
373
|
+
},
|
|
374
|
+
{
|
|
375
|
+
logicalName: "follow_snapshot_members",
|
|
376
|
+
rows: rowsForQuery(
|
|
377
|
+
db,
|
|
378
|
+
`
|
|
379
|
+
select snapshot_id, profile_id, external_user_id, position
|
|
380
|
+
from follow_snapshot_members
|
|
381
|
+
order by snapshot_id, position, profile_id
|
|
382
|
+
`,
|
|
383
|
+
),
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
logicalName: "follow_edges",
|
|
387
|
+
rows: rowsForQuery(
|
|
388
|
+
db,
|
|
389
|
+
`
|
|
390
|
+
select account_id, direction, profile_id, external_user_id, source,
|
|
391
|
+
current, first_seen_at, last_seen_at, ended_at, updated_at
|
|
392
|
+
from follow_edges
|
|
393
|
+
order by account_id, direction, profile_id
|
|
394
|
+
`,
|
|
395
|
+
),
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
logicalName: "follow_events",
|
|
399
|
+
rows: rowsForQuery(
|
|
400
|
+
db,
|
|
401
|
+
`
|
|
402
|
+
select id, account_id, direction, profile_id, external_user_id, kind,
|
|
403
|
+
event_at, snapshot_id
|
|
404
|
+
from follow_events
|
|
405
|
+
order by account_id, direction, event_at, kind, profile_id, id
|
|
334
406
|
`,
|
|
335
407
|
),
|
|
336
408
|
},
|
|
@@ -393,7 +465,9 @@ function buildShards(db: Database) {
|
|
|
393
465
|
case "tweet_account_edges":
|
|
394
466
|
for (const row of rowSet.rows) {
|
|
395
467
|
const kind =
|
|
396
|
-
row.kind === "home" ||
|
|
468
|
+
row.kind === "home" ||
|
|
469
|
+
row.kind === "mention" ||
|
|
470
|
+
row.kind === "authored"
|
|
397
471
|
? row.kind
|
|
398
472
|
: "unknown";
|
|
399
473
|
addRows(shards, `data/timeline_edges/${kind}.jsonl`, [row]);
|
|
@@ -411,6 +485,12 @@ function buildShards(db: Database) {
|
|
|
411
485
|
);
|
|
412
486
|
}
|
|
413
487
|
break;
|
|
488
|
+
case "url_expansions":
|
|
489
|
+
addRows(shards, "data/links/url_expansions.jsonl", rowSet.rows);
|
|
490
|
+
break;
|
|
491
|
+
case "link_occurrences":
|
|
492
|
+
addRows(shards, "data/links/occurrences.jsonl", rowSet.rows);
|
|
493
|
+
break;
|
|
414
494
|
case "blocks":
|
|
415
495
|
case "mutes":
|
|
416
496
|
addRows(
|
|
@@ -425,6 +505,18 @@ function buildShards(db: Database) {
|
|
|
425
505
|
case "ai_scores":
|
|
426
506
|
addRows(shards, "data/ai_scores.jsonl", rowSet.rows);
|
|
427
507
|
break;
|
|
508
|
+
case "follow_snapshots":
|
|
509
|
+
addRows(shards, "data/follow_snapshots.jsonl", rowSet.rows);
|
|
510
|
+
break;
|
|
511
|
+
case "follow_snapshot_members":
|
|
512
|
+
addRows(shards, "data/follow_snapshot_members.jsonl", rowSet.rows);
|
|
513
|
+
break;
|
|
514
|
+
case "follow_edges":
|
|
515
|
+
addRows(shards, "data/follow_edges.jsonl", rowSet.rows);
|
|
516
|
+
break;
|
|
517
|
+
case "follow_events":
|
|
518
|
+
addRows(shards, "data/follow_events.jsonl", rowSet.rows);
|
|
519
|
+
break;
|
|
428
520
|
}
|
|
429
521
|
}
|
|
430
522
|
|
|
@@ -514,11 +606,17 @@ function computeCounts(files: BackupFileManifest[]) {
|
|
|
514
606
|
? "dm_conversations"
|
|
515
607
|
: second === "dms"
|
|
516
608
|
? "dm_messages"
|
|
517
|
-
: second === "
|
|
518
|
-
? third
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
609
|
+
: second === "links"
|
|
610
|
+
? third === "url_expansions.jsonl"
|
|
611
|
+
? "url_expansions"
|
|
612
|
+
: third === "occurrences.jsonl"
|
|
613
|
+
? "link_occurrences"
|
|
614
|
+
: "links"
|
|
615
|
+
: second === "moderation"
|
|
616
|
+
? third?.replace(/\.jsonl$/, "") || "moderation"
|
|
617
|
+
: second === "actions"
|
|
618
|
+
? third?.replace(/\.jsonl$/, "") || "actions"
|
|
619
|
+
: second?.replace(/\.jsonl$/, "") || "unknown";
|
|
522
620
|
counts[key] = (counts[key] ?? 0) + file.rows;
|
|
523
621
|
}
|
|
524
622
|
return counts;
|
|
@@ -552,11 +650,18 @@ data/collections/likes.jsonl
|
|
|
552
650
|
data/collections/bookmarks.jsonl
|
|
553
651
|
data/dms/conversations.jsonl
|
|
554
652
|
data/dms/YYYY.jsonl
|
|
653
|
+
data/links/url_expansions.jsonl
|
|
654
|
+
data/links/occurrences.jsonl
|
|
555
655
|
data/moderation/blocks.jsonl
|
|
556
656
|
data/moderation/mutes.jsonl
|
|
657
|
+
data/follow_snapshots.jsonl
|
|
658
|
+
data/follow_snapshot_members.jsonl
|
|
659
|
+
data/follow_edges.jsonl
|
|
660
|
+
data/follow_events.jsonl
|
|
557
661
|
\`\`\`
|
|
558
662
|
|
|
559
663
|
Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. Timeline edges keep account-scoped home/mention membership separate from canonical tweet content. DMs are sharded by year and keep \`conversation_id\` in each row.
|
|
664
|
+
The links shard stores expanded short URLs and their source tweet/DM occurrences so linked-tweet search can be rebuilt without re-expanding every \`t.co\` URL.
|
|
560
665
|
|
|
561
666
|
Never commit live tokens, browser cookies, raw SQLite WAL/SHM sidecars, or temporary cache files here.
|
|
562
667
|
`,
|
|
@@ -967,6 +1072,10 @@ function insertFtsRows(
|
|
|
967
1072
|
|
|
968
1073
|
function clearBackupImportData(db: Database) {
|
|
969
1074
|
db.exec(`
|
|
1075
|
+
delete from follow_events;
|
|
1076
|
+
delete from follow_edges;
|
|
1077
|
+
delete from follow_snapshot_members;
|
|
1078
|
+
delete from follow_snapshots;
|
|
970
1079
|
delete from ai_scores;
|
|
971
1080
|
delete from tweet_actions;
|
|
972
1081
|
delete from tweet_account_edges;
|
|
@@ -1029,6 +1138,12 @@ export async function importBackup({
|
|
|
1029
1138
|
mutes,
|
|
1030
1139
|
actions,
|
|
1031
1140
|
scores,
|
|
1141
|
+
urlExpansions,
|
|
1142
|
+
linkOccurrences,
|
|
1143
|
+
followSnapshots,
|
|
1144
|
+
followSnapshotMembers,
|
|
1145
|
+
followEdges,
|
|
1146
|
+
followEvents,
|
|
1032
1147
|
] = await Promise.all([
|
|
1033
1148
|
readRows((file) => file === "data/accounts.jsonl"),
|
|
1034
1149
|
readRows((file) => file === "data/profiles.jsonl"),
|
|
@@ -1047,6 +1162,12 @@ export async function importBackup({
|
|
|
1047
1162
|
readRows((file) => file === "data/moderation/mutes.jsonl"),
|
|
1048
1163
|
readRows((file) => file === "data/actions/tweet_actions.jsonl"),
|
|
1049
1164
|
readRows((file) => file === "data/ai_scores.jsonl"),
|
|
1165
|
+
readRows((file) => file === "data/links/url_expansions.jsonl"),
|
|
1166
|
+
readRows((file) => file === "data/links/occurrences.jsonl"),
|
|
1167
|
+
readRows((file) => file === "data/follow_snapshots.jsonl"),
|
|
1168
|
+
readRows((file) => file === "data/follow_snapshot_members.jsonl"),
|
|
1169
|
+
readRows((file) => file === "data/follow_edges.jsonl"),
|
|
1170
|
+
readRows((file) => file === "data/follow_events.jsonl"),
|
|
1050
1171
|
]);
|
|
1051
1172
|
|
|
1052
1173
|
db.transaction(() => {
|
|
@@ -1152,15 +1273,19 @@ export async function importBackup({
|
|
|
1152
1273
|
`
|
|
1153
1274
|
insert into profiles (
|
|
1154
1275
|
id, handle, display_name, bio, followers_count, following_count,
|
|
1155
|
-
avatar_hue, avatar_url, location, url,
|
|
1156
|
-
raw_json, created_at
|
|
1157
|
-
) values (?, ?, ?, ?, ?, coalesce(?, 0), ?, ?, ?, ?, ?, coalesce(?, '{}'), coalesce(?, '{}'), ?)
|
|
1276
|
+
public_metrics_json, avatar_hue, avatar_url, location, url,
|
|
1277
|
+
verified_type, entities_json, raw_json, created_at
|
|
1278
|
+
) values (?, ?, ?, ?, ?, coalesce(?, 0), coalesce(?, '{}'), ?, ?, ?, ?, ?, coalesce(?, '{}'), coalesce(?, '{}'), ?)
|
|
1158
1279
|
on conflict(id) do update set
|
|
1159
1280
|
handle = coalesce(nullif(excluded.handle, ''), profiles.handle),
|
|
1160
1281
|
display_name = coalesce(nullif(excluded.display_name, ''), profiles.display_name),
|
|
1161
1282
|
bio = coalesce(nullif(excluded.bio, ''), profiles.bio),
|
|
1162
1283
|
followers_count = max(profiles.followers_count, excluded.followers_count),
|
|
1163
1284
|
following_count = max(profiles.following_count, excluded.following_count),
|
|
1285
|
+
public_metrics_json = case
|
|
1286
|
+
when excluded.public_metrics_json not in ('', '{}', 'null') then excluded.public_metrics_json
|
|
1287
|
+
else profiles.public_metrics_json
|
|
1288
|
+
end,
|
|
1164
1289
|
avatar_hue = case when profiles.avatar_hue = 0 then excluded.avatar_hue else profiles.avatar_hue end,
|
|
1165
1290
|
avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url),
|
|
1166
1291
|
location = coalesce(excluded.location, profiles.location),
|
|
@@ -1184,6 +1309,7 @@ export async function importBackup({
|
|
|
1184
1309
|
"bio",
|
|
1185
1310
|
"followers_count",
|
|
1186
1311
|
"following_count",
|
|
1312
|
+
"public_metrics_json",
|
|
1187
1313
|
"avatar_hue",
|
|
1188
1314
|
"avatar_url",
|
|
1189
1315
|
"location",
|
|
@@ -1237,6 +1363,118 @@ export async function importBackup({
|
|
|
1237
1363
|
insertRows(
|
|
1238
1364
|
db,
|
|
1239
1365
|
`
|
|
1366
|
+
insert into follow_snapshots (
|
|
1367
|
+
id, account_id, direction, source, status, page_count, result_count,
|
|
1368
|
+
started_at, completed_at, raw_meta_json
|
|
1369
|
+
) values (?, ?, ?, coalesce(?, 'backup'), ?, coalesce(?, 0), coalesce(?, 0), ?, ?, coalesce(?, '{}'))
|
|
1370
|
+
on conflict(id) do update set
|
|
1371
|
+
account_id = coalesce(nullif(excluded.account_id, ''), follow_snapshots.account_id),
|
|
1372
|
+
direction = coalesce(nullif(excluded.direction, ''), follow_snapshots.direction),
|
|
1373
|
+
source = coalesce(nullif(excluded.source, ''), follow_snapshots.source),
|
|
1374
|
+
status = coalesce(nullif(excluded.status, ''), follow_snapshots.status),
|
|
1375
|
+
page_count = max(follow_snapshots.page_count, excluded.page_count),
|
|
1376
|
+
result_count = max(follow_snapshots.result_count, excluded.result_count),
|
|
1377
|
+
started_at = min(follow_snapshots.started_at, excluded.started_at),
|
|
1378
|
+
completed_at = max(follow_snapshots.completed_at, excluded.completed_at),
|
|
1379
|
+
raw_meta_json = case
|
|
1380
|
+
when excluded.raw_meta_json not in ('', '{}', 'null') then excluded.raw_meta_json
|
|
1381
|
+
else follow_snapshots.raw_meta_json
|
|
1382
|
+
end
|
|
1383
|
+
`,
|
|
1384
|
+
followSnapshots,
|
|
1385
|
+
[
|
|
1386
|
+
"id",
|
|
1387
|
+
"account_id",
|
|
1388
|
+
"direction",
|
|
1389
|
+
"source",
|
|
1390
|
+
"status",
|
|
1391
|
+
"page_count",
|
|
1392
|
+
"result_count",
|
|
1393
|
+
"started_at",
|
|
1394
|
+
"completed_at",
|
|
1395
|
+
"raw_meta_json",
|
|
1396
|
+
],
|
|
1397
|
+
);
|
|
1398
|
+
insertRows(
|
|
1399
|
+
db,
|
|
1400
|
+
`
|
|
1401
|
+
insert into follow_snapshot_members (
|
|
1402
|
+
snapshot_id, profile_id, external_user_id, position
|
|
1403
|
+
) values (?, ?, ?, coalesce(?, 0))
|
|
1404
|
+
on conflict(snapshot_id, profile_id) do update set
|
|
1405
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_snapshot_members.external_user_id),
|
|
1406
|
+
position = excluded.position
|
|
1407
|
+
`,
|
|
1408
|
+
followSnapshotMembers,
|
|
1409
|
+
["snapshot_id", "profile_id", "external_user_id", "position"],
|
|
1410
|
+
);
|
|
1411
|
+
insertRows(
|
|
1412
|
+
db,
|
|
1413
|
+
`
|
|
1414
|
+
insert into follow_edges (
|
|
1415
|
+
account_id, direction, profile_id, external_user_id, source, current,
|
|
1416
|
+
first_seen_at, last_seen_at, ended_at, updated_at
|
|
1417
|
+
) values (?, ?, ?, ?, coalesce(?, 'backup'), coalesce(?, 1), ?, ?, ?, ?)
|
|
1418
|
+
on conflict(account_id, direction, profile_id) do update set
|
|
1419
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_edges.external_user_id),
|
|
1420
|
+
source = coalesce(nullif(excluded.source, ''), follow_edges.source),
|
|
1421
|
+
current = case
|
|
1422
|
+
when excluded.updated_at >= follow_edges.updated_at then excluded.current
|
|
1423
|
+
else follow_edges.current
|
|
1424
|
+
end,
|
|
1425
|
+
first_seen_at = min(follow_edges.first_seen_at, excluded.first_seen_at),
|
|
1426
|
+
last_seen_at = max(follow_edges.last_seen_at, excluded.last_seen_at),
|
|
1427
|
+
ended_at = case
|
|
1428
|
+
when excluded.updated_at >= follow_edges.updated_at then excluded.ended_at
|
|
1429
|
+
else follow_edges.ended_at
|
|
1430
|
+
end,
|
|
1431
|
+
updated_at = max(follow_edges.updated_at, excluded.updated_at)
|
|
1432
|
+
`,
|
|
1433
|
+
followEdges,
|
|
1434
|
+
[
|
|
1435
|
+
"account_id",
|
|
1436
|
+
"direction",
|
|
1437
|
+
"profile_id",
|
|
1438
|
+
"external_user_id",
|
|
1439
|
+
"source",
|
|
1440
|
+
"current",
|
|
1441
|
+
"first_seen_at",
|
|
1442
|
+
"last_seen_at",
|
|
1443
|
+
"ended_at",
|
|
1444
|
+
"updated_at",
|
|
1445
|
+
],
|
|
1446
|
+
);
|
|
1447
|
+
insertRows(
|
|
1448
|
+
db,
|
|
1449
|
+
`
|
|
1450
|
+
insert into follow_events (
|
|
1451
|
+
id, account_id, direction, profile_id, external_user_id, kind, event_at,
|
|
1452
|
+
snapshot_id
|
|
1453
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1454
|
+
on conflict(id) do update set
|
|
1455
|
+
account_id = coalesce(nullif(excluded.account_id, ''), follow_events.account_id),
|
|
1456
|
+
direction = coalesce(nullif(excluded.direction, ''), follow_events.direction),
|
|
1457
|
+
profile_id = coalesce(nullif(excluded.profile_id, ''), follow_events.profile_id),
|
|
1458
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_events.external_user_id),
|
|
1459
|
+
kind = coalesce(nullif(excluded.kind, ''), follow_events.kind),
|
|
1460
|
+
event_at = coalesce(nullif(excluded.event_at, ''), follow_events.event_at),
|
|
1461
|
+
snapshot_id = coalesce(nullif(excluded.snapshot_id, ''), follow_events.snapshot_id)
|
|
1462
|
+
`,
|
|
1463
|
+
followEvents,
|
|
1464
|
+
[
|
|
1465
|
+
"id",
|
|
1466
|
+
"account_id",
|
|
1467
|
+
"direction",
|
|
1468
|
+
"profile_id",
|
|
1469
|
+
"external_user_id",
|
|
1470
|
+
"kind",
|
|
1471
|
+
"event_at",
|
|
1472
|
+
"snapshot_id",
|
|
1473
|
+
],
|
|
1474
|
+
);
|
|
1475
|
+
insertRows(
|
|
1476
|
+
db,
|
|
1477
|
+
`
|
|
1240
1478
|
insert into tweets (
|
|
1241
1479
|
id, account_id, author_profile_id, kind, text, created_at, is_replied,
|
|
1242
1480
|
reply_to_id, like_count, media_count, bookmarked, liked, entities_json,
|
|
@@ -1246,8 +1484,8 @@ export async function importBackup({
|
|
|
1246
1484
|
account_id = coalesce(nullif(excluded.account_id, ''), tweets.account_id),
|
|
1247
1485
|
author_profile_id = coalesce(nullif(excluded.author_profile_id, ''), tweets.author_profile_id),
|
|
1248
1486
|
kind = case
|
|
1249
|
-
when tweets.kind in ('home', 'mention') then tweets.kind
|
|
1250
|
-
when excluded.kind in ('home', 'mention') then excluded.kind
|
|
1487
|
+
when tweets.kind in ('home', 'mention', 'authored') then tweets.kind
|
|
1488
|
+
when excluded.kind in ('home', 'mention', 'authored') then excluded.kind
|
|
1251
1489
|
else coalesce(nullif(excluded.kind, ''), tweets.kind)
|
|
1252
1490
|
end,
|
|
1253
1491
|
text = coalesce(nullif(excluded.text, ''), tweets.text),
|
|
@@ -1409,6 +1647,70 @@ export async function importBackup({
|
|
|
1409
1647
|
insertRows(
|
|
1410
1648
|
db,
|
|
1411
1649
|
`
|
|
1650
|
+
insert into url_expansions (
|
|
1651
|
+
short_url, expanded_url, final_url, status, expanded_tweet_id,
|
|
1652
|
+
expanded_handle, title, description, image_url, site_name, error, source,
|
|
1653
|
+
updated_at
|
|
1654
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1655
|
+
on conflict(short_url) do update set
|
|
1656
|
+
expanded_url = excluded.expanded_url,
|
|
1657
|
+
final_url = excluded.final_url,
|
|
1658
|
+
status = excluded.status,
|
|
1659
|
+
expanded_tweet_id = excluded.expanded_tweet_id,
|
|
1660
|
+
expanded_handle = excluded.expanded_handle,
|
|
1661
|
+
title = excluded.title,
|
|
1662
|
+
description = excluded.description,
|
|
1663
|
+
image_url = excluded.image_url,
|
|
1664
|
+
site_name = excluded.site_name,
|
|
1665
|
+
error = excluded.error,
|
|
1666
|
+
source = excluded.source,
|
|
1667
|
+
updated_at = excluded.updated_at
|
|
1668
|
+
`,
|
|
1669
|
+
urlExpansions,
|
|
1670
|
+
[
|
|
1671
|
+
"short_url",
|
|
1672
|
+
"expanded_url",
|
|
1673
|
+
"final_url",
|
|
1674
|
+
"status",
|
|
1675
|
+
"expanded_tweet_id",
|
|
1676
|
+
"expanded_handle",
|
|
1677
|
+
"title",
|
|
1678
|
+
"description",
|
|
1679
|
+
"image_url",
|
|
1680
|
+
"site_name",
|
|
1681
|
+
"error",
|
|
1682
|
+
"source",
|
|
1683
|
+
"updated_at",
|
|
1684
|
+
],
|
|
1685
|
+
);
|
|
1686
|
+
insertRows(
|
|
1687
|
+
db,
|
|
1688
|
+
`
|
|
1689
|
+
insert into link_occurrences (
|
|
1690
|
+
source_kind, source_id, source_position, short_url, account_id,
|
|
1691
|
+
conversation_id, direction, created_at
|
|
1692
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1693
|
+
on conflict(source_kind, source_id, source_position, short_url) do update set
|
|
1694
|
+
account_id = excluded.account_id,
|
|
1695
|
+
conversation_id = excluded.conversation_id,
|
|
1696
|
+
direction = excluded.direction,
|
|
1697
|
+
created_at = excluded.created_at
|
|
1698
|
+
`,
|
|
1699
|
+
linkOccurrences,
|
|
1700
|
+
[
|
|
1701
|
+
"source_kind",
|
|
1702
|
+
"source_id",
|
|
1703
|
+
"source_position",
|
|
1704
|
+
"short_url",
|
|
1705
|
+
"account_id",
|
|
1706
|
+
"conversation_id",
|
|
1707
|
+
"direction",
|
|
1708
|
+
"created_at",
|
|
1709
|
+
],
|
|
1710
|
+
);
|
|
1711
|
+
insertRows(
|
|
1712
|
+
db,
|
|
1713
|
+
`
|
|
1412
1714
|
insert into blocks (account_id, profile_id, source, created_at)
|
|
1413
1715
|
values (?, ?, ?, ?)
|
|
1414
1716
|
on conflict(account_id, profile_id) do update set
|
package/src/lib/bird-actions.ts
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { promisify } from "node:util";
|
|
3
|
-
import { getBirdCommand } from "./config";
|
|
1
|
+
import { runBirdCommand } from "./bird-command";
|
|
4
2
|
import type { XurlMentionUser } from "./types";
|
|
5
3
|
|
|
6
|
-
const execFileAsync = promisify(execFile);
|
|
7
|
-
|
|
8
4
|
function liveWritesDisabled() {
|
|
9
5
|
return process.env.BIRDCLAW_DISABLE_LIVE_WRITES === "1";
|
|
10
6
|
}
|
|
@@ -42,11 +38,6 @@ function normalizeOutput(stdout?: string, stderr?: string) {
|
|
|
42
38
|
return stripAnsi(stdout || stderr || "ok").trim();
|
|
43
39
|
}
|
|
44
40
|
|
|
45
|
-
async function runBirdCommand(args: string[]) {
|
|
46
|
-
const birdCommand = getBirdCommand();
|
|
47
|
-
return execFileAsync(birdCommand, args);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
41
|
async function runBirdJsonCommand(args: string[]) {
|
|
51
42
|
const { stdout } = await runBirdCommand(args);
|
|
52
43
|
return JSON.parse(stripAnsi(stdout)) as Record<string, unknown>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import type { ExecFileOptions } from "node:child_process";
|
|
3
|
+
import { access } from "node:fs/promises";
|
|
4
|
+
import { constants } from "node:fs";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { getBirdCommand } from "./config";
|
|
7
|
+
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
|
|
10
|
+
function isPathCommand(command: string) {
|
|
11
|
+
return command.includes("/") || command.startsWith(".");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function formatBirdInstallHint(command: string) {
|
|
15
|
+
return [
|
|
16
|
+
`bird command unavailable: ${command}`,
|
|
17
|
+
"Install bird on PATH, set BIRDCLAW_BIRD_COMMAND, or update ~/.birdclaw/config.json mentions.birdCommand.",
|
|
18
|
+
].join("\n");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function assertBirdCommandAvailable(command: string) {
|
|
22
|
+
if (!isPathCommand(command)) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
await access(command, constants.X_OK);
|
|
28
|
+
} catch {
|
|
29
|
+
throw new Error(formatBirdInstallHint(command));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export async function runBirdCommand(
|
|
34
|
+
args: string[],
|
|
35
|
+
options?: ExecFileOptions,
|
|
36
|
+
): Promise<{ stdout: string; stderr: string }> {
|
|
37
|
+
const birdCommand = getBirdCommand();
|
|
38
|
+
await assertBirdCommandAvailable(birdCommand);
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
const result =
|
|
42
|
+
options === undefined
|
|
43
|
+
? await execFileAsync(birdCommand, args)
|
|
44
|
+
: await execFileAsync(birdCommand, args, options);
|
|
45
|
+
return result as { stdout: string; stderr: string };
|
|
46
|
+
} catch (error) {
|
|
47
|
+
if (
|
|
48
|
+
error &&
|
|
49
|
+
typeof error === "object" &&
|
|
50
|
+
"code" in error &&
|
|
51
|
+
(error.code === "ENOENT" || error.code === "EACCES")
|
|
52
|
+
) {
|
|
53
|
+
throw new Error(formatBirdInstallHint(birdCommand));
|
|
54
|
+
}
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
package/src/lib/bird.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { promisify } from "node:util";
|
|
|
6
6
|
import { getBirdCommand } from "./config";
|
|
7
7
|
import type {
|
|
8
8
|
XurlMentionData,
|
|
9
|
+
XurlFollowUsersResponse,
|
|
9
10
|
XurlMentionsResponse,
|
|
10
11
|
XurlMentionUser,
|
|
11
12
|
XurlReferencedTweet,
|
|
@@ -98,6 +99,13 @@ interface BirdProfilesPayload {
|
|
|
98
99
|
errors?: Array<{ target?: string; error?: string }>;
|
|
99
100
|
}
|
|
100
101
|
|
|
102
|
+
type BirdFollowUsersPayload =
|
|
103
|
+
| NonNullable<BirdUserOverviewPayload["user"]>[]
|
|
104
|
+
| {
|
|
105
|
+
users?: NonNullable<BirdUserOverviewPayload["user"]>[];
|
|
106
|
+
nextCursor?: string | null;
|
|
107
|
+
};
|
|
108
|
+
|
|
101
109
|
function toIsoTimestamp(value: string) {
|
|
102
110
|
const parsed = new Date(value);
|
|
103
111
|
if (Number.isNaN(parsed.getTime())) {
|
|
@@ -173,13 +181,30 @@ function parseBirdJson(stdout: string) {
|
|
|
173
181
|
}
|
|
174
182
|
|
|
175
183
|
function formatBirdCommandError(error: unknown, birdCommand: string) {
|
|
184
|
+
const text = [
|
|
185
|
+
error instanceof Error ? error.message : "",
|
|
186
|
+
error &&
|
|
187
|
+
typeof error === "object" &&
|
|
188
|
+
"stderr" in error &&
|
|
189
|
+
typeof error.stderr === "string"
|
|
190
|
+
? error.stderr
|
|
191
|
+
: "",
|
|
192
|
+
error &&
|
|
193
|
+
typeof error === "object" &&
|
|
194
|
+
"stdout" in error &&
|
|
195
|
+
typeof error.stdout === "string"
|
|
196
|
+
? error.stdout
|
|
197
|
+
: "",
|
|
198
|
+
].join("\n");
|
|
176
199
|
if (
|
|
177
|
-
error instanceof Error &&
|
|
178
|
-
|
|
179
|
-
|
|
200
|
+
(error instanceof Error &&
|
|
201
|
+
"code" in error &&
|
|
202
|
+
(error as { code?: unknown }).code === "ENOENT") ||
|
|
203
|
+
(/No such file or directory|command not found|cannot execute/i.test(text) &&
|
|
204
|
+
text.includes(birdCommand))
|
|
180
205
|
) {
|
|
181
206
|
return new Error(
|
|
182
|
-
`bird
|
|
207
|
+
`bird command unavailable: ${birdCommand}\nInstall bird on PATH, set BIRDCLAW_BIRD_COMMAND, or update ~/.birdclaw/config.json mentions.birdCommand.`,
|
|
183
208
|
);
|
|
184
209
|
}
|
|
185
210
|
|
|
@@ -206,7 +231,7 @@ async function runBirdJsonCommand(args: string[], timeoutMs?: number) {
|
|
|
206
231
|
const shellScript = 'out="$1"; shift; exec "$@" > "$out"';
|
|
207
232
|
await execFileAsync(
|
|
208
233
|
"/bin/bash",
|
|
209
|
-
["-
|
|
234
|
+
["-c", shellScript, "birdclaw-bird", stdoutPath, birdCommand, ...args],
|
|
210
235
|
{ maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES, timeout: timeoutMs },
|
|
211
236
|
);
|
|
212
237
|
return readFileSync(stdoutPath, "utf8");
|
|
@@ -435,6 +460,65 @@ export async function listHomeTimelineViaBird({
|
|
|
435
460
|
return normalizeBirdTweets(getBirdTweetItems(payload, "home"));
|
|
436
461
|
}
|
|
437
462
|
|
|
463
|
+
function normalizeBirdFollowUsers(
|
|
464
|
+
payload: unknown,
|
|
465
|
+
command: "followers" | "following",
|
|
466
|
+
maxResults: number,
|
|
467
|
+
): XurlFollowUsersResponse {
|
|
468
|
+
const rawPayload = payload as BirdFollowUsersPayload;
|
|
469
|
+
const users = Array.isArray(rawPayload) ? rawPayload : rawPayload.users;
|
|
470
|
+
if (!Array.isArray(users)) {
|
|
471
|
+
throw new Error(`bird ${command} returned unexpected JSON`);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
const data = users
|
|
475
|
+
.map(toXurlMentionUser)
|
|
476
|
+
.filter((user): user is XurlMentionUser => Boolean(user));
|
|
477
|
+
const nextToken =
|
|
478
|
+
!Array.isArray(rawPayload) && typeof rawPayload.nextCursor === "string"
|
|
479
|
+
? rawPayload.nextCursor
|
|
480
|
+
: null;
|
|
481
|
+
|
|
482
|
+
return {
|
|
483
|
+
data,
|
|
484
|
+
meta: {
|
|
485
|
+
result_count: data.length,
|
|
486
|
+
page_count:
|
|
487
|
+
data.length > 0 ? Math.max(1, Math.ceil(data.length / maxResults)) : 1,
|
|
488
|
+
next_token: nextToken,
|
|
489
|
+
},
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
export async function listFollowUsersViaBird({
|
|
494
|
+
direction,
|
|
495
|
+
userId,
|
|
496
|
+
maxResults,
|
|
497
|
+
all,
|
|
498
|
+
maxPages,
|
|
499
|
+
}: {
|
|
500
|
+
direction: "followers" | "following";
|
|
501
|
+
userId?: string;
|
|
502
|
+
maxResults: number;
|
|
503
|
+
all?: boolean;
|
|
504
|
+
maxPages?: number;
|
|
505
|
+
}): Promise<XurlFollowUsersResponse> {
|
|
506
|
+
const args = [direction, "-n", String(maxResults), "--json"];
|
|
507
|
+
if (userId) {
|
|
508
|
+
args.push("--user", userId);
|
|
509
|
+
}
|
|
510
|
+
if (all) {
|
|
511
|
+
args.push("--all");
|
|
512
|
+
}
|
|
513
|
+
if (maxPages !== undefined) {
|
|
514
|
+
args.push("--max-pages", String(maxPages));
|
|
515
|
+
}
|
|
516
|
+
const stdout = await runBirdJsonCommand(args);
|
|
517
|
+
const payload = parseBirdJson(stdout);
|
|
518
|
+
|
|
519
|
+
return normalizeBirdFollowUsers(payload, direction, maxResults);
|
|
520
|
+
}
|
|
521
|
+
|
|
438
522
|
export async function listThreadViaBird({
|
|
439
523
|
tweetId,
|
|
440
524
|
all,
|
package/src/lib/config.ts
CHANGED