birdclaw 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -1
- package/README.md +108 -7
- package/package.json +24 -23
- package/playwright.config.ts +1 -0
- package/public/favicon.ico +0 -0
- package/public/logo192.png +0 -0
- package/public/logo512.png +0 -0
- package/public/manifest.json +2 -2
- package/src/cli.ts +563 -18
- package/src/components/AppNav.tsx +66 -29
- package/src/components/AvatarChip.tsx +10 -5
- package/src/components/ConversationThread.tsx +125 -0
- package/src/components/DmWorkspace.tsx +96 -98
- package/src/components/EmbeddedTweetCard.tsx +20 -14
- package/src/components/InboxCard.tsx +92 -90
- package/src/components/LinkPreviewCard.tsx +270 -0
- package/src/components/ProfilePreview.tsx +8 -3
- package/src/components/SavedTimelineView.tsx +48 -38
- package/src/components/TimelineCard.tsx +228 -84
- package/src/components/TweetRichText.tsx +6 -2
- package/src/lib/archive-import.ts +1567 -65
- package/src/lib/authored-live.ts +1074 -0
- package/src/lib/backup.ts +318 -14
- package/src/lib/bird-actions.ts +1 -10
- package/src/lib/bird-command.ts +57 -0
- package/src/lib/bird.ts +89 -5
- package/src/lib/config.ts +1 -1
- package/src/lib/db.ts +282 -4
- package/src/lib/follow-graph.ts +1053 -0
- package/src/lib/link-index.ts +629 -0
- package/src/lib/link-insights.ts +834 -0
- package/src/lib/link-preview-metadata.ts +334 -0
- package/src/lib/media-fetch.ts +787 -0
- package/src/lib/media-includes.ts +165 -0
- package/src/lib/mention-threads-live.ts +535 -43
- package/src/lib/mentions-live.ts +623 -19
- package/src/lib/moderation-target.ts +6 -0
- package/src/lib/profile-hydration.ts +1 -1
- package/src/lib/profile-resolver.ts +115 -1
- package/src/lib/queries.ts +233 -7
- package/src/lib/seed.ts +127 -8
- package/src/lib/timeline-collections-live.ts +145 -22
- package/src/lib/timeline-live.ts +10 -15
- package/src/lib/tweet-account-edges.ts +9 -2
- package/src/lib/tweet-render.ts +97 -0
- package/src/lib/types.ts +219 -2
- package/src/lib/ui.ts +375 -147
- package/src/lib/url-expansion-store.ts +110 -0
- package/src/lib/url-expansion.ts +33 -8
- package/src/lib/x-profile.ts +7 -2
- package/src/lib/xurl.ts +296 -12
- package/src/routeTree.gen.ts +105 -0
- package/src/routes/__root.tsx +7 -3
- package/src/routes/api/conversation.tsx +34 -0
- package/src/routes/api/link-insights.tsx +79 -0
- package/src/routes/api/link-preview.tsx +43 -0
- package/src/routes/api/profile-hydrate.tsx +51 -0
- package/src/routes/blocks.tsx +111 -86
- package/src/routes/dms.tsx +90 -78
- package/src/routes/inbox.tsx +98 -86
- package/src/routes/index.tsx +63 -50
- package/src/routes/links.tsx +883 -0
- package/src/routes/mentions.tsx +63 -50
- package/src/styles.css +106 -43
package/src/lib/backup.ts
CHANGED
|
@@ -184,9 +184,9 @@ function getExportRowSets(db: Database) {
|
|
|
184
184
|
rows: rowsForQuery(
|
|
185
185
|
db,
|
|
186
186
|
`
|
|
187
|
-
select id, handle, display_name, bio, followers_count,
|
|
188
|
-
|
|
189
|
-
raw_json, created_at
|
|
187
|
+
select id, handle, display_name, bio, followers_count,
|
|
188
|
+
following_count, public_metrics_json, avatar_hue, avatar_url,
|
|
189
|
+
location, url, verified_type, entities_json, raw_json, created_at
|
|
190
190
|
from profiles
|
|
191
191
|
order by id
|
|
192
192
|
`,
|
|
@@ -287,6 +287,31 @@ function getExportRowSets(db: Database) {
|
|
|
287
287
|
is_replied, media_count
|
|
288
288
|
from dm_messages
|
|
289
289
|
order by conversation_id, created_at, id
|
|
290
|
+
`,
|
|
291
|
+
),
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
logicalName: "url_expansions",
|
|
295
|
+
rows: rowsForQuery(
|
|
296
|
+
db,
|
|
297
|
+
`
|
|
298
|
+
select short_url, expanded_url, final_url, status, expanded_tweet_id,
|
|
299
|
+
expanded_handle, title, description, image_url, site_name, error,
|
|
300
|
+
source, updated_at
|
|
301
|
+
from url_expansions
|
|
302
|
+
order by short_url
|
|
303
|
+
`,
|
|
304
|
+
),
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
logicalName: "link_occurrences",
|
|
308
|
+
rows: rowsForQuery(
|
|
309
|
+
db,
|
|
310
|
+
`
|
|
311
|
+
select source_kind, source_id, source_position, short_url, account_id,
|
|
312
|
+
conversation_id, direction, created_at
|
|
313
|
+
from link_occurrences
|
|
314
|
+
order by source_kind, source_id, source_position, short_url
|
|
290
315
|
`,
|
|
291
316
|
),
|
|
292
317
|
},
|
|
@@ -331,6 +356,53 @@ function getExportRowSets(db: Database) {
|
|
|
331
356
|
select entity_kind, entity_id, model, score, summary, reasoning, updated_at
|
|
332
357
|
from ai_scores
|
|
333
358
|
order by entity_kind, entity_id, model
|
|
359
|
+
`,
|
|
360
|
+
),
|
|
361
|
+
},
|
|
362
|
+
{
|
|
363
|
+
logicalName: "follow_snapshots",
|
|
364
|
+
rows: rowsForQuery(
|
|
365
|
+
db,
|
|
366
|
+
`
|
|
367
|
+
select id, account_id, direction, source, status, page_count,
|
|
368
|
+
result_count, started_at, completed_at, raw_meta_json
|
|
369
|
+
from follow_snapshots
|
|
370
|
+
order by account_id, direction, completed_at, id
|
|
371
|
+
`,
|
|
372
|
+
),
|
|
373
|
+
},
|
|
374
|
+
{
|
|
375
|
+
logicalName: "follow_snapshot_members",
|
|
376
|
+
rows: rowsForQuery(
|
|
377
|
+
db,
|
|
378
|
+
`
|
|
379
|
+
select snapshot_id, profile_id, external_user_id, position
|
|
380
|
+
from follow_snapshot_members
|
|
381
|
+
order by snapshot_id, position, profile_id
|
|
382
|
+
`,
|
|
383
|
+
),
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
logicalName: "follow_edges",
|
|
387
|
+
rows: rowsForQuery(
|
|
388
|
+
db,
|
|
389
|
+
`
|
|
390
|
+
select account_id, direction, profile_id, external_user_id, source,
|
|
391
|
+
current, first_seen_at, last_seen_at, ended_at, updated_at
|
|
392
|
+
from follow_edges
|
|
393
|
+
order by account_id, direction, profile_id
|
|
394
|
+
`,
|
|
395
|
+
),
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
logicalName: "follow_events",
|
|
399
|
+
rows: rowsForQuery(
|
|
400
|
+
db,
|
|
401
|
+
`
|
|
402
|
+
select id, account_id, direction, profile_id, external_user_id, kind,
|
|
403
|
+
event_at, snapshot_id
|
|
404
|
+
from follow_events
|
|
405
|
+
order by account_id, direction, event_at, kind, profile_id, id
|
|
334
406
|
`,
|
|
335
407
|
),
|
|
336
408
|
},
|
|
@@ -393,7 +465,9 @@ function buildShards(db: Database) {
|
|
|
393
465
|
case "tweet_account_edges":
|
|
394
466
|
for (const row of rowSet.rows) {
|
|
395
467
|
const kind =
|
|
396
|
-
row.kind === "home" ||
|
|
468
|
+
row.kind === "home" ||
|
|
469
|
+
row.kind === "mention" ||
|
|
470
|
+
row.kind === "authored"
|
|
397
471
|
? row.kind
|
|
398
472
|
: "unknown";
|
|
399
473
|
addRows(shards, `data/timeline_edges/${kind}.jsonl`, [row]);
|
|
@@ -411,6 +485,12 @@ function buildShards(db: Database) {
|
|
|
411
485
|
);
|
|
412
486
|
}
|
|
413
487
|
break;
|
|
488
|
+
case "url_expansions":
|
|
489
|
+
addRows(shards, "data/links/url_expansions.jsonl", rowSet.rows);
|
|
490
|
+
break;
|
|
491
|
+
case "link_occurrences":
|
|
492
|
+
addRows(shards, "data/links/occurrences.jsonl", rowSet.rows);
|
|
493
|
+
break;
|
|
414
494
|
case "blocks":
|
|
415
495
|
case "mutes":
|
|
416
496
|
addRows(
|
|
@@ -425,6 +505,18 @@ function buildShards(db: Database) {
|
|
|
425
505
|
case "ai_scores":
|
|
426
506
|
addRows(shards, "data/ai_scores.jsonl", rowSet.rows);
|
|
427
507
|
break;
|
|
508
|
+
case "follow_snapshots":
|
|
509
|
+
addRows(shards, "data/follow_snapshots.jsonl", rowSet.rows);
|
|
510
|
+
break;
|
|
511
|
+
case "follow_snapshot_members":
|
|
512
|
+
addRows(shards, "data/follow_snapshot_members.jsonl", rowSet.rows);
|
|
513
|
+
break;
|
|
514
|
+
case "follow_edges":
|
|
515
|
+
addRows(shards, "data/follow_edges.jsonl", rowSet.rows);
|
|
516
|
+
break;
|
|
517
|
+
case "follow_events":
|
|
518
|
+
addRows(shards, "data/follow_events.jsonl", rowSet.rows);
|
|
519
|
+
break;
|
|
428
520
|
}
|
|
429
521
|
}
|
|
430
522
|
|
|
@@ -514,11 +606,17 @@ function computeCounts(files: BackupFileManifest[]) {
|
|
|
514
606
|
? "dm_conversations"
|
|
515
607
|
: second === "dms"
|
|
516
608
|
? "dm_messages"
|
|
517
|
-
: second === "
|
|
518
|
-
? third
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
609
|
+
: second === "links"
|
|
610
|
+
? third === "url_expansions.jsonl"
|
|
611
|
+
? "url_expansions"
|
|
612
|
+
: third === "occurrences.jsonl"
|
|
613
|
+
? "link_occurrences"
|
|
614
|
+
: "links"
|
|
615
|
+
: second === "moderation"
|
|
616
|
+
? third?.replace(/\.jsonl$/, "") || "moderation"
|
|
617
|
+
: second === "actions"
|
|
618
|
+
? third?.replace(/\.jsonl$/, "") || "actions"
|
|
619
|
+
: second?.replace(/\.jsonl$/, "") || "unknown";
|
|
522
620
|
counts[key] = (counts[key] ?? 0) + file.rows;
|
|
523
621
|
}
|
|
524
622
|
return counts;
|
|
@@ -552,11 +650,18 @@ data/collections/likes.jsonl
|
|
|
552
650
|
data/collections/bookmarks.jsonl
|
|
553
651
|
data/dms/conversations.jsonl
|
|
554
652
|
data/dms/YYYY.jsonl
|
|
653
|
+
data/links/url_expansions.jsonl
|
|
654
|
+
data/links/occurrences.jsonl
|
|
555
655
|
data/moderation/blocks.jsonl
|
|
556
656
|
data/moderation/mutes.jsonl
|
|
657
|
+
data/follow_snapshots.jsonl
|
|
658
|
+
data/follow_snapshot_members.jsonl
|
|
659
|
+
data/follow_edges.jsonl
|
|
660
|
+
data/follow_events.jsonl
|
|
557
661
|
\`\`\`
|
|
558
662
|
|
|
559
663
|
Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. Timeline edges keep account-scoped home/mention membership separate from canonical tweet content. DMs are sharded by year and keep \`conversation_id\` in each row.
|
|
664
|
+
The links shard stores expanded short URLs and their source tweet/DM occurrences so linked-tweet search can be rebuilt without re-expanding every \`t.co\` URL.
|
|
560
665
|
|
|
561
666
|
Never commit live tokens, browser cookies, raw SQLite WAL/SHM sidecars, or temporary cache files here.
|
|
562
667
|
`,
|
|
@@ -967,10 +1072,16 @@ function insertFtsRows(
|
|
|
967
1072
|
|
|
968
1073
|
function clearBackupImportData(db: Database) {
|
|
969
1074
|
db.exec(`
|
|
1075
|
+
delete from follow_events;
|
|
1076
|
+
delete from follow_edges;
|
|
1077
|
+
delete from follow_snapshot_members;
|
|
1078
|
+
delete from follow_snapshots;
|
|
970
1079
|
delete from ai_scores;
|
|
971
1080
|
delete from tweet_actions;
|
|
972
1081
|
delete from tweet_account_edges;
|
|
973
1082
|
delete from tweet_collections;
|
|
1083
|
+
delete from link_occurrences;
|
|
1084
|
+
delete from url_expansions;
|
|
974
1085
|
delete from blocks;
|
|
975
1086
|
delete from mutes;
|
|
976
1087
|
delete from dm_fts;
|
|
@@ -1027,6 +1138,12 @@ export async function importBackup({
|
|
|
1027
1138
|
mutes,
|
|
1028
1139
|
actions,
|
|
1029
1140
|
scores,
|
|
1141
|
+
urlExpansions,
|
|
1142
|
+
linkOccurrences,
|
|
1143
|
+
followSnapshots,
|
|
1144
|
+
followSnapshotMembers,
|
|
1145
|
+
followEdges,
|
|
1146
|
+
followEvents,
|
|
1030
1147
|
] = await Promise.all([
|
|
1031
1148
|
readRows((file) => file === "data/accounts.jsonl"),
|
|
1032
1149
|
readRows((file) => file === "data/profiles.jsonl"),
|
|
@@ -1045,6 +1162,12 @@ export async function importBackup({
|
|
|
1045
1162
|
readRows((file) => file === "data/moderation/mutes.jsonl"),
|
|
1046
1163
|
readRows((file) => file === "data/actions/tweet_actions.jsonl"),
|
|
1047
1164
|
readRows((file) => file === "data/ai_scores.jsonl"),
|
|
1165
|
+
readRows((file) => file === "data/links/url_expansions.jsonl"),
|
|
1166
|
+
readRows((file) => file === "data/links/occurrences.jsonl"),
|
|
1167
|
+
readRows((file) => file === "data/follow_snapshots.jsonl"),
|
|
1168
|
+
readRows((file) => file === "data/follow_snapshot_members.jsonl"),
|
|
1169
|
+
readRows((file) => file === "data/follow_edges.jsonl"),
|
|
1170
|
+
readRows((file) => file === "data/follow_events.jsonl"),
|
|
1048
1171
|
]);
|
|
1049
1172
|
|
|
1050
1173
|
db.transaction(() => {
|
|
@@ -1150,15 +1273,19 @@ export async function importBackup({
|
|
|
1150
1273
|
`
|
|
1151
1274
|
insert into profiles (
|
|
1152
1275
|
id, handle, display_name, bio, followers_count, following_count,
|
|
1153
|
-
avatar_hue, avatar_url, location, url,
|
|
1154
|
-
raw_json, created_at
|
|
1155
|
-
) values (?, ?, ?, ?, ?, coalesce(?, 0), ?, ?, ?, ?, ?, coalesce(?, '{}'), coalesce(?, '{}'), ?)
|
|
1276
|
+
public_metrics_json, avatar_hue, avatar_url, location, url,
|
|
1277
|
+
verified_type, entities_json, raw_json, created_at
|
|
1278
|
+
) values (?, ?, ?, ?, ?, coalesce(?, 0), coalesce(?, '{}'), ?, ?, ?, ?, ?, coalesce(?, '{}'), coalesce(?, '{}'), ?)
|
|
1156
1279
|
on conflict(id) do update set
|
|
1157
1280
|
handle = coalesce(nullif(excluded.handle, ''), profiles.handle),
|
|
1158
1281
|
display_name = coalesce(nullif(excluded.display_name, ''), profiles.display_name),
|
|
1159
1282
|
bio = coalesce(nullif(excluded.bio, ''), profiles.bio),
|
|
1160
1283
|
followers_count = max(profiles.followers_count, excluded.followers_count),
|
|
1161
1284
|
following_count = max(profiles.following_count, excluded.following_count),
|
|
1285
|
+
public_metrics_json = case
|
|
1286
|
+
when excluded.public_metrics_json not in ('', '{}', 'null') then excluded.public_metrics_json
|
|
1287
|
+
else profiles.public_metrics_json
|
|
1288
|
+
end,
|
|
1162
1289
|
avatar_hue = case when profiles.avatar_hue = 0 then excluded.avatar_hue else profiles.avatar_hue end,
|
|
1163
1290
|
avatar_url = coalesce(excluded.avatar_url, profiles.avatar_url),
|
|
1164
1291
|
location = coalesce(excluded.location, profiles.location),
|
|
@@ -1182,6 +1309,7 @@ export async function importBackup({
|
|
|
1182
1309
|
"bio",
|
|
1183
1310
|
"followers_count",
|
|
1184
1311
|
"following_count",
|
|
1312
|
+
"public_metrics_json",
|
|
1185
1313
|
"avatar_hue",
|
|
1186
1314
|
"avatar_url",
|
|
1187
1315
|
"location",
|
|
@@ -1235,6 +1363,118 @@ export async function importBackup({
|
|
|
1235
1363
|
insertRows(
|
|
1236
1364
|
db,
|
|
1237
1365
|
`
|
|
1366
|
+
insert into follow_snapshots (
|
|
1367
|
+
id, account_id, direction, source, status, page_count, result_count,
|
|
1368
|
+
started_at, completed_at, raw_meta_json
|
|
1369
|
+
) values (?, ?, ?, coalesce(?, 'backup'), ?, coalesce(?, 0), coalesce(?, 0), ?, ?, coalesce(?, '{}'))
|
|
1370
|
+
on conflict(id) do update set
|
|
1371
|
+
account_id = coalesce(nullif(excluded.account_id, ''), follow_snapshots.account_id),
|
|
1372
|
+
direction = coalesce(nullif(excluded.direction, ''), follow_snapshots.direction),
|
|
1373
|
+
source = coalesce(nullif(excluded.source, ''), follow_snapshots.source),
|
|
1374
|
+
status = coalesce(nullif(excluded.status, ''), follow_snapshots.status),
|
|
1375
|
+
page_count = max(follow_snapshots.page_count, excluded.page_count),
|
|
1376
|
+
result_count = max(follow_snapshots.result_count, excluded.result_count),
|
|
1377
|
+
started_at = min(follow_snapshots.started_at, excluded.started_at),
|
|
1378
|
+
completed_at = max(follow_snapshots.completed_at, excluded.completed_at),
|
|
1379
|
+
raw_meta_json = case
|
|
1380
|
+
when excluded.raw_meta_json not in ('', '{}', 'null') then excluded.raw_meta_json
|
|
1381
|
+
else follow_snapshots.raw_meta_json
|
|
1382
|
+
end
|
|
1383
|
+
`,
|
|
1384
|
+
followSnapshots,
|
|
1385
|
+
[
|
|
1386
|
+
"id",
|
|
1387
|
+
"account_id",
|
|
1388
|
+
"direction",
|
|
1389
|
+
"source",
|
|
1390
|
+
"status",
|
|
1391
|
+
"page_count",
|
|
1392
|
+
"result_count",
|
|
1393
|
+
"started_at",
|
|
1394
|
+
"completed_at",
|
|
1395
|
+
"raw_meta_json",
|
|
1396
|
+
],
|
|
1397
|
+
);
|
|
1398
|
+
insertRows(
|
|
1399
|
+
db,
|
|
1400
|
+
`
|
|
1401
|
+
insert into follow_snapshot_members (
|
|
1402
|
+
snapshot_id, profile_id, external_user_id, position
|
|
1403
|
+
) values (?, ?, ?, coalesce(?, 0))
|
|
1404
|
+
on conflict(snapshot_id, profile_id) do update set
|
|
1405
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_snapshot_members.external_user_id),
|
|
1406
|
+
position = excluded.position
|
|
1407
|
+
`,
|
|
1408
|
+
followSnapshotMembers,
|
|
1409
|
+
["snapshot_id", "profile_id", "external_user_id", "position"],
|
|
1410
|
+
);
|
|
1411
|
+
insertRows(
|
|
1412
|
+
db,
|
|
1413
|
+
`
|
|
1414
|
+
insert into follow_edges (
|
|
1415
|
+
account_id, direction, profile_id, external_user_id, source, current,
|
|
1416
|
+
first_seen_at, last_seen_at, ended_at, updated_at
|
|
1417
|
+
) values (?, ?, ?, ?, coalesce(?, 'backup'), coalesce(?, 1), ?, ?, ?, ?)
|
|
1418
|
+
on conflict(account_id, direction, profile_id) do update set
|
|
1419
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_edges.external_user_id),
|
|
1420
|
+
source = coalesce(nullif(excluded.source, ''), follow_edges.source),
|
|
1421
|
+
current = case
|
|
1422
|
+
when excluded.updated_at >= follow_edges.updated_at then excluded.current
|
|
1423
|
+
else follow_edges.current
|
|
1424
|
+
end,
|
|
1425
|
+
first_seen_at = min(follow_edges.first_seen_at, excluded.first_seen_at),
|
|
1426
|
+
last_seen_at = max(follow_edges.last_seen_at, excluded.last_seen_at),
|
|
1427
|
+
ended_at = case
|
|
1428
|
+
when excluded.updated_at >= follow_edges.updated_at then excluded.ended_at
|
|
1429
|
+
else follow_edges.ended_at
|
|
1430
|
+
end,
|
|
1431
|
+
updated_at = max(follow_edges.updated_at, excluded.updated_at)
|
|
1432
|
+
`,
|
|
1433
|
+
followEdges,
|
|
1434
|
+
[
|
|
1435
|
+
"account_id",
|
|
1436
|
+
"direction",
|
|
1437
|
+
"profile_id",
|
|
1438
|
+
"external_user_id",
|
|
1439
|
+
"source",
|
|
1440
|
+
"current",
|
|
1441
|
+
"first_seen_at",
|
|
1442
|
+
"last_seen_at",
|
|
1443
|
+
"ended_at",
|
|
1444
|
+
"updated_at",
|
|
1445
|
+
],
|
|
1446
|
+
);
|
|
1447
|
+
insertRows(
|
|
1448
|
+
db,
|
|
1449
|
+
`
|
|
1450
|
+
insert into follow_events (
|
|
1451
|
+
id, account_id, direction, profile_id, external_user_id, kind, event_at,
|
|
1452
|
+
snapshot_id
|
|
1453
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1454
|
+
on conflict(id) do update set
|
|
1455
|
+
account_id = coalesce(nullif(excluded.account_id, ''), follow_events.account_id),
|
|
1456
|
+
direction = coalesce(nullif(excluded.direction, ''), follow_events.direction),
|
|
1457
|
+
profile_id = coalesce(nullif(excluded.profile_id, ''), follow_events.profile_id),
|
|
1458
|
+
external_user_id = coalesce(nullif(excluded.external_user_id, ''), follow_events.external_user_id),
|
|
1459
|
+
kind = coalesce(nullif(excluded.kind, ''), follow_events.kind),
|
|
1460
|
+
event_at = coalesce(nullif(excluded.event_at, ''), follow_events.event_at),
|
|
1461
|
+
snapshot_id = coalesce(nullif(excluded.snapshot_id, ''), follow_events.snapshot_id)
|
|
1462
|
+
`,
|
|
1463
|
+
followEvents,
|
|
1464
|
+
[
|
|
1465
|
+
"id",
|
|
1466
|
+
"account_id",
|
|
1467
|
+
"direction",
|
|
1468
|
+
"profile_id",
|
|
1469
|
+
"external_user_id",
|
|
1470
|
+
"kind",
|
|
1471
|
+
"event_at",
|
|
1472
|
+
"snapshot_id",
|
|
1473
|
+
],
|
|
1474
|
+
);
|
|
1475
|
+
insertRows(
|
|
1476
|
+
db,
|
|
1477
|
+
`
|
|
1238
1478
|
insert into tweets (
|
|
1239
1479
|
id, account_id, author_profile_id, kind, text, created_at, is_replied,
|
|
1240
1480
|
reply_to_id, like_count, media_count, bookmarked, liked, entities_json,
|
|
@@ -1244,8 +1484,8 @@ export async function importBackup({
|
|
|
1244
1484
|
account_id = coalesce(nullif(excluded.account_id, ''), tweets.account_id),
|
|
1245
1485
|
author_profile_id = coalesce(nullif(excluded.author_profile_id, ''), tweets.author_profile_id),
|
|
1246
1486
|
kind = case
|
|
1247
|
-
when tweets.kind in ('home', 'mention') then tweets.kind
|
|
1248
|
-
when excluded.kind in ('home', 'mention') then excluded.kind
|
|
1487
|
+
when tweets.kind in ('home', 'mention', 'authored') then tweets.kind
|
|
1488
|
+
when excluded.kind in ('home', 'mention', 'authored') then excluded.kind
|
|
1249
1489
|
else coalesce(nullif(excluded.kind, ''), tweets.kind)
|
|
1250
1490
|
end,
|
|
1251
1491
|
text = coalesce(nullif(excluded.text, ''), tweets.text),
|
|
@@ -1407,6 +1647,70 @@ export async function importBackup({
|
|
|
1407
1647
|
insertRows(
|
|
1408
1648
|
db,
|
|
1409
1649
|
`
|
|
1650
|
+
insert into url_expansions (
|
|
1651
|
+
short_url, expanded_url, final_url, status, expanded_tweet_id,
|
|
1652
|
+
expanded_handle, title, description, image_url, site_name, error, source,
|
|
1653
|
+
updated_at
|
|
1654
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1655
|
+
on conflict(short_url) do update set
|
|
1656
|
+
expanded_url = excluded.expanded_url,
|
|
1657
|
+
final_url = excluded.final_url,
|
|
1658
|
+
status = excluded.status,
|
|
1659
|
+
expanded_tweet_id = excluded.expanded_tweet_id,
|
|
1660
|
+
expanded_handle = excluded.expanded_handle,
|
|
1661
|
+
title = excluded.title,
|
|
1662
|
+
description = excluded.description,
|
|
1663
|
+
image_url = excluded.image_url,
|
|
1664
|
+
site_name = excluded.site_name,
|
|
1665
|
+
error = excluded.error,
|
|
1666
|
+
source = excluded.source,
|
|
1667
|
+
updated_at = excluded.updated_at
|
|
1668
|
+
`,
|
|
1669
|
+
urlExpansions,
|
|
1670
|
+
[
|
|
1671
|
+
"short_url",
|
|
1672
|
+
"expanded_url",
|
|
1673
|
+
"final_url",
|
|
1674
|
+
"status",
|
|
1675
|
+
"expanded_tweet_id",
|
|
1676
|
+
"expanded_handle",
|
|
1677
|
+
"title",
|
|
1678
|
+
"description",
|
|
1679
|
+
"image_url",
|
|
1680
|
+
"site_name",
|
|
1681
|
+
"error",
|
|
1682
|
+
"source",
|
|
1683
|
+
"updated_at",
|
|
1684
|
+
],
|
|
1685
|
+
);
|
|
1686
|
+
insertRows(
|
|
1687
|
+
db,
|
|
1688
|
+
`
|
|
1689
|
+
insert into link_occurrences (
|
|
1690
|
+
source_kind, source_id, source_position, short_url, account_id,
|
|
1691
|
+
conversation_id, direction, created_at
|
|
1692
|
+
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1693
|
+
on conflict(source_kind, source_id, source_position, short_url) do update set
|
|
1694
|
+
account_id = excluded.account_id,
|
|
1695
|
+
conversation_id = excluded.conversation_id,
|
|
1696
|
+
direction = excluded.direction,
|
|
1697
|
+
created_at = excluded.created_at
|
|
1698
|
+
`,
|
|
1699
|
+
linkOccurrences,
|
|
1700
|
+
[
|
|
1701
|
+
"source_kind",
|
|
1702
|
+
"source_id",
|
|
1703
|
+
"source_position",
|
|
1704
|
+
"short_url",
|
|
1705
|
+
"account_id",
|
|
1706
|
+
"conversation_id",
|
|
1707
|
+
"direction",
|
|
1708
|
+
"created_at",
|
|
1709
|
+
],
|
|
1710
|
+
);
|
|
1711
|
+
insertRows(
|
|
1712
|
+
db,
|
|
1713
|
+
`
|
|
1410
1714
|
insert into blocks (account_id, profile_id, source, created_at)
|
|
1411
1715
|
values (?, ?, ?, ?)
|
|
1412
1716
|
on conflict(account_id, profile_id) do update set
|
package/src/lib/bird-actions.ts
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { promisify } from "node:util";
|
|
3
|
-
import { getBirdCommand } from "./config";
|
|
1
|
+
import { runBirdCommand } from "./bird-command";
|
|
4
2
|
import type { XurlMentionUser } from "./types";
|
|
5
3
|
|
|
6
|
-
const execFileAsync = promisify(execFile);
|
|
7
|
-
|
|
8
4
|
function liveWritesDisabled() {
|
|
9
5
|
return process.env.BIRDCLAW_DISABLE_LIVE_WRITES === "1";
|
|
10
6
|
}
|
|
@@ -42,11 +38,6 @@ function normalizeOutput(stdout?: string, stderr?: string) {
|
|
|
42
38
|
return stripAnsi(stdout || stderr || "ok").trim();
|
|
43
39
|
}
|
|
44
40
|
|
|
45
|
-
async function runBirdCommand(args: string[]) {
|
|
46
|
-
const birdCommand = getBirdCommand();
|
|
47
|
-
return execFileAsync(birdCommand, args);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
41
|
async function runBirdJsonCommand(args: string[]) {
|
|
51
42
|
const { stdout } = await runBirdCommand(args);
|
|
52
43
|
return JSON.parse(stripAnsi(stdout)) as Record<string, unknown>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import type { ExecFileOptions } from "node:child_process";
|
|
3
|
+
import { access } from "node:fs/promises";
|
|
4
|
+
import { constants } from "node:fs";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { getBirdCommand } from "./config";
|
|
7
|
+
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
|
|
10
|
+
function isPathCommand(command: string) {
|
|
11
|
+
return command.includes("/") || command.startsWith(".");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function formatBirdInstallHint(command: string) {
|
|
15
|
+
return [
|
|
16
|
+
`bird command unavailable: ${command}`,
|
|
17
|
+
"Install bird on PATH, set BIRDCLAW_BIRD_COMMAND, or update ~/.birdclaw/config.json mentions.birdCommand.",
|
|
18
|
+
].join("\n");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function assertBirdCommandAvailable(command: string) {
|
|
22
|
+
if (!isPathCommand(command)) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
await access(command, constants.X_OK);
|
|
28
|
+
} catch {
|
|
29
|
+
throw new Error(formatBirdInstallHint(command));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export async function runBirdCommand(
|
|
34
|
+
args: string[],
|
|
35
|
+
options?: ExecFileOptions,
|
|
36
|
+
): Promise<{ stdout: string; stderr: string }> {
|
|
37
|
+
const birdCommand = getBirdCommand();
|
|
38
|
+
await assertBirdCommandAvailable(birdCommand);
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
const result =
|
|
42
|
+
options === undefined
|
|
43
|
+
? await execFileAsync(birdCommand, args)
|
|
44
|
+
: await execFileAsync(birdCommand, args, options);
|
|
45
|
+
return result as { stdout: string; stderr: string };
|
|
46
|
+
} catch (error) {
|
|
47
|
+
if (
|
|
48
|
+
error &&
|
|
49
|
+
typeof error === "object" &&
|
|
50
|
+
"code" in error &&
|
|
51
|
+
(error.code === "ENOENT" || error.code === "EACCES")
|
|
52
|
+
) {
|
|
53
|
+
throw new Error(formatBirdInstallHint(birdCommand));
|
|
54
|
+
}
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
package/src/lib/bird.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { promisify } from "node:util";
|
|
|
6
6
|
import { getBirdCommand } from "./config";
|
|
7
7
|
import type {
|
|
8
8
|
XurlMentionData,
|
|
9
|
+
XurlFollowUsersResponse,
|
|
9
10
|
XurlMentionsResponse,
|
|
10
11
|
XurlMentionUser,
|
|
11
12
|
XurlReferencedTweet,
|
|
@@ -98,6 +99,13 @@ interface BirdProfilesPayload {
|
|
|
98
99
|
errors?: Array<{ target?: string; error?: string }>;
|
|
99
100
|
}
|
|
100
101
|
|
|
102
|
+
type BirdFollowUsersPayload =
|
|
103
|
+
| NonNullable<BirdUserOverviewPayload["user"]>[]
|
|
104
|
+
| {
|
|
105
|
+
users?: NonNullable<BirdUserOverviewPayload["user"]>[];
|
|
106
|
+
nextCursor?: string | null;
|
|
107
|
+
};
|
|
108
|
+
|
|
101
109
|
function toIsoTimestamp(value: string) {
|
|
102
110
|
const parsed = new Date(value);
|
|
103
111
|
if (Number.isNaN(parsed.getTime())) {
|
|
@@ -173,13 +181,30 @@ function parseBirdJson(stdout: string) {
|
|
|
173
181
|
}
|
|
174
182
|
|
|
175
183
|
function formatBirdCommandError(error: unknown, birdCommand: string) {
|
|
184
|
+
const text = [
|
|
185
|
+
error instanceof Error ? error.message : "",
|
|
186
|
+
error &&
|
|
187
|
+
typeof error === "object" &&
|
|
188
|
+
"stderr" in error &&
|
|
189
|
+
typeof error.stderr === "string"
|
|
190
|
+
? error.stderr
|
|
191
|
+
: "",
|
|
192
|
+
error &&
|
|
193
|
+
typeof error === "object" &&
|
|
194
|
+
"stdout" in error &&
|
|
195
|
+
typeof error.stdout === "string"
|
|
196
|
+
? error.stdout
|
|
197
|
+
: "",
|
|
198
|
+
].join("\n");
|
|
176
199
|
if (
|
|
177
|
-
error instanceof Error &&
|
|
178
|
-
|
|
179
|
-
|
|
200
|
+
(error instanceof Error &&
|
|
201
|
+
"code" in error &&
|
|
202
|
+
(error as { code?: unknown }).code === "ENOENT") ||
|
|
203
|
+
(/No such file or directory|command not found|cannot execute/i.test(text) &&
|
|
204
|
+
text.includes(birdCommand))
|
|
180
205
|
) {
|
|
181
206
|
return new Error(
|
|
182
|
-
`bird
|
|
207
|
+
`bird command unavailable: ${birdCommand}\nInstall bird on PATH, set BIRDCLAW_BIRD_COMMAND, or update ~/.birdclaw/config.json mentions.birdCommand.`,
|
|
183
208
|
);
|
|
184
209
|
}
|
|
185
210
|
|
|
@@ -206,7 +231,7 @@ async function runBirdJsonCommand(args: string[], timeoutMs?: number) {
|
|
|
206
231
|
const shellScript = 'out="$1"; shift; exec "$@" > "$out"';
|
|
207
232
|
await execFileAsync(
|
|
208
233
|
"/bin/bash",
|
|
209
|
-
["-
|
|
234
|
+
["-c", shellScript, "birdclaw-bird", stdoutPath, birdCommand, ...args],
|
|
210
235
|
{ maxBuffer: BIRD_JSON_MAX_BUFFER_BYTES, timeout: timeoutMs },
|
|
211
236
|
);
|
|
212
237
|
return readFileSync(stdoutPath, "utf8");
|
|
@@ -435,6 +460,65 @@ export async function listHomeTimelineViaBird({
|
|
|
435
460
|
return normalizeBirdTweets(getBirdTweetItems(payload, "home"));
|
|
436
461
|
}
|
|
437
462
|
|
|
463
|
+
function normalizeBirdFollowUsers(
|
|
464
|
+
payload: unknown,
|
|
465
|
+
command: "followers" | "following",
|
|
466
|
+
maxResults: number,
|
|
467
|
+
): XurlFollowUsersResponse {
|
|
468
|
+
const rawPayload = payload as BirdFollowUsersPayload;
|
|
469
|
+
const users = Array.isArray(rawPayload) ? rawPayload : rawPayload.users;
|
|
470
|
+
if (!Array.isArray(users)) {
|
|
471
|
+
throw new Error(`bird ${command} returned unexpected JSON`);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
const data = users
|
|
475
|
+
.map(toXurlMentionUser)
|
|
476
|
+
.filter((user): user is XurlMentionUser => Boolean(user));
|
|
477
|
+
const nextToken =
|
|
478
|
+
!Array.isArray(rawPayload) && typeof rawPayload.nextCursor === "string"
|
|
479
|
+
? rawPayload.nextCursor
|
|
480
|
+
: null;
|
|
481
|
+
|
|
482
|
+
return {
|
|
483
|
+
data,
|
|
484
|
+
meta: {
|
|
485
|
+
result_count: data.length,
|
|
486
|
+
page_count:
|
|
487
|
+
data.length > 0 ? Math.max(1, Math.ceil(data.length / maxResults)) : 1,
|
|
488
|
+
next_token: nextToken,
|
|
489
|
+
},
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
export async function listFollowUsersViaBird({
|
|
494
|
+
direction,
|
|
495
|
+
userId,
|
|
496
|
+
maxResults,
|
|
497
|
+
all,
|
|
498
|
+
maxPages,
|
|
499
|
+
}: {
|
|
500
|
+
direction: "followers" | "following";
|
|
501
|
+
userId?: string;
|
|
502
|
+
maxResults: number;
|
|
503
|
+
all?: boolean;
|
|
504
|
+
maxPages?: number;
|
|
505
|
+
}): Promise<XurlFollowUsersResponse> {
|
|
506
|
+
const args = [direction, "-n", String(maxResults), "--json"];
|
|
507
|
+
if (userId) {
|
|
508
|
+
args.push("--user", userId);
|
|
509
|
+
}
|
|
510
|
+
if (all) {
|
|
511
|
+
args.push("--all");
|
|
512
|
+
}
|
|
513
|
+
if (maxPages !== undefined) {
|
|
514
|
+
args.push("--max-pages", String(maxPages));
|
|
515
|
+
}
|
|
516
|
+
const stdout = await runBirdJsonCommand(args);
|
|
517
|
+
const payload = parseBirdJson(stdout);
|
|
518
|
+
|
|
519
|
+
return normalizeBirdFollowUsers(payload, direction, maxResults);
|
|
520
|
+
}
|
|
521
|
+
|
|
438
522
|
export async function listThreadViaBird({
|
|
439
523
|
tweetId,
|
|
440
524
|
all,
|
package/src/lib/config.ts
CHANGED