birdclaw 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +37 -9
- package/package.json +4 -6
- package/src/cli.ts +289 -20
- package/src/lib/archive-import.ts +27 -0
- package/src/lib/backup.ts +271 -33
- package/src/lib/bird-actions.ts +17 -0
- package/src/lib/bird.ts +189 -0
- package/src/lib/blocks.ts +3 -3
- package/src/lib/bookmark-sync-job.ts +3 -3
- package/src/lib/db.ts +404 -14
- package/src/lib/dms-live.ts +3 -3
- package/src/lib/identity-search-index.ts +369 -0
- package/src/lib/link-index.ts +716 -0
- package/src/lib/mention-threads-live.ts +5 -9
- package/src/lib/mentions-live.ts +18 -7
- package/src/lib/moderation-target.ts +4 -4
- package/src/lib/moderation-write.ts +3 -3
- package/src/lib/profile-affiliation-hydration.ts +189 -0
- package/src/lib/profile-affiliations.ts +262 -0
- package/src/lib/profile-bio-entities.ts +318 -0
- package/src/lib/profile-history.ts +164 -0
- package/src/lib/profile-hydration.ts +4 -28
- package/src/lib/profile-resolver.ts +428 -0
- package/src/lib/queries.ts +391 -57
- package/src/lib/research.ts +11 -0
- package/src/lib/seed.ts +2 -2
- package/src/lib/sqlite.ts +143 -0
- package/src/lib/timeline-collections-live.ts +7 -7
- package/src/lib/timeline-live.ts +16 -9
- package/src/lib/tweet-account-edges.ts +39 -0
- package/src/lib/types.ts +108 -0
- package/src/lib/url-expansion.ts +155 -0
- package/src/lib/whois.ts +1060 -0
- package/src/lib/x-profile.ts +140 -12
- package/src/lib/xurl.ts +7 -6
package/src/lib/bird.ts
CHANGED
|
@@ -73,6 +73,31 @@ export interface BirdDmsResponse {
|
|
|
73
73
|
events: BirdDmEvent[];
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
interface BirdUserOverviewPayload {
|
|
77
|
+
user?: {
|
|
78
|
+
id?: string;
|
|
79
|
+
username?: string;
|
|
80
|
+
name?: string;
|
|
81
|
+
description?: string;
|
|
82
|
+
location?: string;
|
|
83
|
+
url?: string;
|
|
84
|
+
verified?: boolean;
|
|
85
|
+
verifiedType?: string;
|
|
86
|
+
verified_type?: string;
|
|
87
|
+
followersCount?: number;
|
|
88
|
+
followingCount?: number;
|
|
89
|
+
profileImageUrl?: string;
|
|
90
|
+
createdAt?: string;
|
|
91
|
+
entities?: Record<string, unknown>;
|
|
92
|
+
affiliation?: Record<string, unknown>;
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
interface BirdProfilesPayload {
|
|
97
|
+
users?: NonNullable<BirdUserOverviewPayload["user"]>[];
|
|
98
|
+
errors?: Array<{ target?: string; error?: string }>;
|
|
99
|
+
}
|
|
100
|
+
|
|
76
101
|
function toIsoTimestamp(value: string) {
|
|
77
102
|
const parsed = new Date(value);
|
|
78
103
|
if (Number.isNaN(parsed.getTime())) {
|
|
@@ -161,6 +186,18 @@ function formatBirdCommandError(error: unknown, birdCommand: string) {
|
|
|
161
186
|
return error;
|
|
162
187
|
}
|
|
163
188
|
|
|
189
|
+
function isUnsupportedBirdOptionError(error: unknown, option: string) {
|
|
190
|
+
if (!error || typeof error !== "object") {
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
const text = [
|
|
194
|
+
error instanceof Error ? error.message : "",
|
|
195
|
+
"stderr" in error && typeof error.stderr === "string" ? error.stderr : "",
|
|
196
|
+
"stdout" in error && typeof error.stdout === "string" ? error.stdout : "",
|
|
197
|
+
].join("\n");
|
|
198
|
+
return text.includes(option) && /unknown option|error:/i.test(text);
|
|
199
|
+
}
|
|
200
|
+
|
|
164
201
|
async function runBirdJsonCommand(args: string[], timeoutMs?: number) {
|
|
165
202
|
const tempDir = mkdtempSync(join(tmpdir(), "birdclaw-bird-"));
|
|
166
203
|
const stdoutPath = join(tempDir, "stdout.json");
|
|
@@ -446,3 +483,155 @@ export async function listDirectMessagesViaBird({
|
|
|
446
483
|
|
|
447
484
|
return payload as BirdDmsResponse;
|
|
448
485
|
}
|
|
486
|
+
|
|
487
|
+
export async function lookupProfileViaBird(
|
|
488
|
+
usernameOrId: string,
|
|
489
|
+
): Promise<XurlMentionUser | null> {
|
|
490
|
+
const target = usernameOrId.trim().replace(/^@/, "");
|
|
491
|
+
if (!target) {
|
|
492
|
+
return null;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
let stdout: string;
|
|
496
|
+
try {
|
|
497
|
+
stdout = await runBirdJsonCommand([
|
|
498
|
+
"user",
|
|
499
|
+
target,
|
|
500
|
+
"--json",
|
|
501
|
+
"--profile-only",
|
|
502
|
+
]);
|
|
503
|
+
} catch (error) {
|
|
504
|
+
if (!isUnsupportedBirdOptionError(error, "--profile-only")) {
|
|
505
|
+
throw error;
|
|
506
|
+
}
|
|
507
|
+
stdout = await runBirdJsonCommand([
|
|
508
|
+
"user",
|
|
509
|
+
target,
|
|
510
|
+
"--json",
|
|
511
|
+
"--count",
|
|
512
|
+
"1",
|
|
513
|
+
]);
|
|
514
|
+
}
|
|
515
|
+
const payload = parseBirdJson(stdout) as BirdUserOverviewPayload;
|
|
516
|
+
return toXurlMentionUser(payload.user);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function toXurlMentionUser(
|
|
520
|
+
user: BirdUserOverviewPayload["user"],
|
|
521
|
+
): XurlMentionUser | null {
|
|
522
|
+
if (!user?.id || !user.username) {
|
|
523
|
+
return null;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
return {
|
|
527
|
+
id: String(user.id),
|
|
528
|
+
username: String(user.username).replace(/^@/, ""),
|
|
529
|
+
name: String(user.name ?? user.username),
|
|
530
|
+
description:
|
|
531
|
+
typeof user.description === "string" ? user.description : undefined,
|
|
532
|
+
location: typeof user.location === "string" ? user.location : undefined,
|
|
533
|
+
url: typeof user.url === "string" ? user.url : undefined,
|
|
534
|
+
verified: typeof user.verified === "boolean" ? user.verified : undefined,
|
|
535
|
+
verified_type:
|
|
536
|
+
typeof user.verifiedType === "string"
|
|
537
|
+
? user.verifiedType
|
|
538
|
+
: typeof user.verified_type === "string"
|
|
539
|
+
? user.verified_type
|
|
540
|
+
: undefined,
|
|
541
|
+
profile_image_url:
|
|
542
|
+
typeof user.profileImageUrl === "string"
|
|
543
|
+
? user.profileImageUrl
|
|
544
|
+
: undefined,
|
|
545
|
+
entities:
|
|
546
|
+
user.entities && typeof user.entities === "object"
|
|
547
|
+
? user.entities
|
|
548
|
+
: undefined,
|
|
549
|
+
affiliation:
|
|
550
|
+
user.affiliation && typeof user.affiliation === "object"
|
|
551
|
+
? user.affiliation
|
|
552
|
+
: undefined,
|
|
553
|
+
created_at: typeof user.createdAt === "string" ? user.createdAt : undefined,
|
|
554
|
+
public_metrics: {
|
|
555
|
+
followers_count: Number(user.followersCount ?? 0),
|
|
556
|
+
following_count: Number(user.followingCount ?? 0),
|
|
557
|
+
},
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
export async function lookupProfilesViaBird(
|
|
562
|
+
usernameOrIds: string[],
|
|
563
|
+
): Promise<
|
|
564
|
+
Array<{ target: string; user: XurlMentionUser | null; error?: string }>
|
|
565
|
+
> {
|
|
566
|
+
const targets = Array.from(
|
|
567
|
+
new Set(
|
|
568
|
+
usernameOrIds
|
|
569
|
+
.map((target) => target.trim().replace(/^@/, ""))
|
|
570
|
+
.filter((target) => target.length > 0),
|
|
571
|
+
),
|
|
572
|
+
);
|
|
573
|
+
if (targets.length === 0) {
|
|
574
|
+
return [];
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
try {
|
|
578
|
+
const stdout = await runBirdJsonCommand(["profiles", ...targets, "--json"]);
|
|
579
|
+
const payload = parseBirdJson(stdout) as BirdProfilesPayload;
|
|
580
|
+
const users = (payload.users ?? [])
|
|
581
|
+
.map(toXurlMentionUser)
|
|
582
|
+
.filter((user): user is XurlMentionUser => Boolean(user));
|
|
583
|
+
const byTarget = new Map<string, XurlMentionUser>();
|
|
584
|
+
for (const user of users) {
|
|
585
|
+
byTarget.set(String(user.id), user);
|
|
586
|
+
byTarget.set(user.username.toLowerCase(), user);
|
|
587
|
+
}
|
|
588
|
+
const errors = new Map(
|
|
589
|
+
(payload.errors ?? []).map((item) => [
|
|
590
|
+
String(item.target ?? "")
|
|
591
|
+
.replace(/^@/, "")
|
|
592
|
+
.toLowerCase(),
|
|
593
|
+
item.error ?? "Unknown error",
|
|
594
|
+
]),
|
|
595
|
+
);
|
|
596
|
+
return targets.map((target) => ({
|
|
597
|
+
target,
|
|
598
|
+
user: byTarget.get(target.toLowerCase()) ?? null,
|
|
599
|
+
...(errors.has(target.toLowerCase())
|
|
600
|
+
? { error: errors.get(target.toLowerCase()) }
|
|
601
|
+
: {}),
|
|
602
|
+
}));
|
|
603
|
+
} catch (error) {
|
|
604
|
+
if (!isUnsupportedBirdOptionError(error, "profiles")) {
|
|
605
|
+
throw error;
|
|
606
|
+
}
|
|
607
|
+
return Promise.all(
|
|
608
|
+
targets.map(async (target) => {
|
|
609
|
+
try {
|
|
610
|
+
return { target, user: await lookupProfileViaBird(target) };
|
|
611
|
+
} catch (lookupError) {
|
|
612
|
+
return {
|
|
613
|
+
target,
|
|
614
|
+
user: null,
|
|
615
|
+
error:
|
|
616
|
+
lookupError instanceof Error
|
|
617
|
+
? lookupError.message
|
|
618
|
+
: String(lookupError),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
}),
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
export const __test__ = {
|
|
627
|
+
toIsoTimestamp,
|
|
628
|
+
escapeJsonStringControlChars,
|
|
629
|
+
parseBirdJson,
|
|
630
|
+
formatBirdCommandError,
|
|
631
|
+
isUnsupportedBirdOptionError,
|
|
632
|
+
getBirdTweetItems,
|
|
633
|
+
getBirdTweetItem,
|
|
634
|
+
toMediaEntities,
|
|
635
|
+
toReferencedTweets,
|
|
636
|
+
normalizeBirdTweets,
|
|
637
|
+
};
|
package/src/lib/blocks.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type Database from "
|
|
1
|
+
import type { Database } from "./sqlite";
|
|
2
2
|
import { getNativeDb } from "./db";
|
|
3
3
|
import {
|
|
4
4
|
getAccountHandle,
|
|
@@ -16,7 +16,7 @@ function remoteBlockSyncDisabled() {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
function upsertRemoteBlock(
|
|
19
|
-
db: Database
|
|
19
|
+
db: Database,
|
|
20
20
|
accountId: string,
|
|
21
21
|
profileId: string,
|
|
22
22
|
blockedAt: string,
|
|
@@ -33,7 +33,7 @@ function upsertRemoteBlock(
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
function pruneRemoteBlocks(
|
|
36
|
-
db: Database
|
|
36
|
+
db: Database,
|
|
37
37
|
accountId: string,
|
|
38
38
|
profileIds: string[],
|
|
39
39
|
) {
|
|
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { promisify } from "node:util";
|
|
6
|
-
import type Database from "
|
|
6
|
+
import type { Database } from "./sqlite";
|
|
7
7
|
import { maybeAutoSyncBackup, type BackupAutoUpdateResult } from "./backup";
|
|
8
8
|
import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
|
|
9
9
|
import { getNativeDb } from "./db";
|
|
@@ -31,7 +31,7 @@ export interface BookmarkSyncJobOptions {
|
|
|
31
31
|
cacheTtlMs?: number;
|
|
32
32
|
logPath?: string;
|
|
33
33
|
lockPath?: string;
|
|
34
|
-
db?: Database
|
|
34
|
+
db?: Database;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
export interface BookmarkSyncAuditEntry {
|
|
@@ -117,7 +117,7 @@ export function getDefaultBookmarkSyncLockPath() {
|
|
|
117
117
|
return path.join(getBirdclawPaths().rootDir, "locks", "bookmarks-sync.lock");
|
|
118
118
|
}
|
|
119
119
|
|
|
120
|
-
function countBookmarks(db: Database
|
|
120
|
+
function countBookmarks(db: Database) {
|
|
121
121
|
const row = db
|
|
122
122
|
.prepare("select count(*) as count from tweet_collections where kind = ?")
|
|
123
123
|
.get("bookmarks") as { count: number };
|