birdclaw 0.1.0 → 0.2.0

package/src/lib/bookmark-sync-job.ts

@@ -0,0 +1,446 @@
+import { execFile } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import type Database from "better-sqlite3";
+import { maybeAutoSyncBackup, type BackupAutoUpdateResult } from "./backup";
+import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
+import { getNativeDb } from "./db";
+import {
+	syncTimelineCollection,
+	type TimelineCollectionMode,
+} from "./timeline-collections-live";
+
+const execFileAsync = promisify(execFile);
+const DEFAULT_BOOKMARK_SYNC_INTERVAL_SECONDS = 3 * 60 * 60;
+const DEFAULT_BOOKMARK_SYNC_LIMIT = 100;
+const DEFAULT_BOOKMARK_SYNC_MAX_PAGES = 5;
+const DEFAULT_LAUNCHD_LABEL = "com.steipete.birdclaw.bookmarks-sync";
+const DEFAULT_LAUNCHD_PATH =
+	"/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin";
+const DEFAULT_LOCK_STALE_MS = 6 * 60 * 60 * 1000;
+
+export interface BookmarkSyncJobOptions {
+	account?: string;
+	mode?: TimelineCollectionMode;
+	limit?: number;
+	all?: boolean;
+	maxPages?: number;
+	refresh?: boolean;
+	cacheTtlMs?: number;
+	logPath?: string;
+	lockPath?: string;
+	db?: Database.Database;
+}
+
+export interface BookmarkSyncAuditEntry {
+	job: "bookmarks-sync";
+	ok: boolean;
+	startedAt: string;
+	finishedAt: string;
+	durationMs: number;
+	host: string;
+	pid: number;
+	options: {
+		account?: string;
+		mode: TimelineCollectionMode;
+		limit: number;
+		all: boolean;
+		maxPages?: number;
+		refresh: boolean;
+		cacheTtlMs?: number;
+	};
+	before: {
+		bookmarks: number;
+	};
+	after: {
+		bookmarks: number;
+	};
+	added: number;
+	skipped?: "already-running";
+	sync?: {
+		source: string;
+		count: number;
+		accountId: string;
+	};
+	backup?: BackupAutoUpdateResult;
+	error?: string;
+}
+
+export interface BookmarkSyncLaunchAgentOptions {
+	label?: string;
+	intervalSeconds?: number;
+	program?: string;
+	mode?: TimelineCollectionMode;
+	limit?: number;
+	all?: boolean;
+	maxPages?: number;
+	refresh?: boolean;
+	cacheTtlSeconds?: number;
+	logPath?: string;
+	envFile?: string;
+	stdoutPath?: string;
+	stderrPath?: string;
+	launchAgentsDir?: string;
+	load?: boolean;
+}
+
+export interface BookmarkSyncLaunchAgentInstallResult {
+	ok: true;
+	label: string;
+	plistPath: string;
+	loaded: boolean;
+	programArguments: string[];
+	logPath: string;
+	stdoutPath: string;
+	stderrPath: string;
+	intervalSeconds: number;
+	envFile?: string;
+}
+
+function expandHome(input: string) {
+	return input === "~" || input.startsWith("~/")
+		? path.join(os.homedir(), input.slice(2))
+		: input;
+}
+
+function resolvePath(input: string) {
+	return path.resolve(expandHome(input));
+}
+
+export function getDefaultBookmarkSyncAuditLogPath() {
+	return path.join(getBirdclawPaths().rootDir, "audit", "bookmarks-sync.jsonl");
+}
+
+export function getDefaultBookmarkSyncLockPath() {
+	return path.join(getBirdclawPaths().rootDir, "locks", "bookmarks-sync.lock");
+}
+
+function countBookmarks(db: Database.Database) {
+	const row = db
+		.prepare("select count(*) as count from tweet_collections where kind = ?")
+		.get("bookmarks") as { count: number };
+	return row.count;
+}
+
+async function appendAuditEntry(
+	logPath: string,
+	entry: BookmarkSyncAuditEntry,
+) {
+	await fs.mkdir(path.dirname(logPath), { recursive: true });
+	await fs.appendFile(logPath, `${JSON.stringify(entry)}\n`, "utf8");
+}
+
+function messageFromError(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+async function acquireLock(lockPath: string) {
+	await fs.mkdir(path.dirname(lockPath), { recursive: true });
+	try {
+		const handle = await fs.open(lockPath, "wx");
+		await handle.writeFile(
+			`${JSON.stringify({
+				pid: process.pid,
+				host: os.hostname(),
+				startedAt: new Date().toISOString(),
+			})}\n`,
+			"utf8",
+		);
+		await handle.close();
+		return async () => {
+			await fs.rm(lockPath, { force: true });
+		};
+	} catch (error) {
+		if (
+			typeof error === "object" &&
+			error !== null &&
+			"code" in error &&
+			error.code === "EEXIST"
+		) {
+			const stats = await fs.stat(lockPath).catch(() => undefined);
+			if (stats && Date.now() - stats.mtimeMs > DEFAULT_LOCK_STALE_MS) {
+				await fs.rm(lockPath, { force: true });
+				return acquireLock(lockPath);
+			}
+			return undefined;
+		}
+		throw error;
+	}
+}
+
+export async function runBookmarkSyncJob({
+	account,
+	mode = "auto",
+	limit = DEFAULT_BOOKMARK_SYNC_LIMIT,
+	all,
+	maxPages = DEFAULT_BOOKMARK_SYNC_MAX_PAGES,
+	refresh = true,
+	cacheTtlMs,
+	logPath,
+	lockPath,
+	db,
+}: BookmarkSyncJobOptions = {}): Promise<BookmarkSyncAuditEntry> {
+	ensureBirdclawDirs();
+	const database = db ?? getNativeDb({ seedDemoData: false });
+	const resolvedLogPath = resolvePath(
+		logPath ?? getDefaultBookmarkSyncAuditLogPath(),
+	);
+	const resolvedLockPath = resolvePath(
+		lockPath ?? getDefaultBookmarkSyncLockPath(),
+	);
+	const effectiveAll = all ?? maxPages !== undefined;
+	const started = Date.now();
+	const startedAt = new Date(started).toISOString();
+	const before = { bookmarks: countBookmarks(database) };
+	const options = {
+		...(account ? { account } : {}),
+		mode,
+		limit,
+		all: effectiveAll,
+		...(maxPages === undefined ? {} : { maxPages }),
+		refresh,
+		...(cacheTtlMs === undefined ? {} : { cacheTtlMs }),
+	};
+
+	const releaseLock = await acquireLock(resolvedLockPath);
+	if (!releaseLock) {
+		const finished = Date.now();
+		const entry: BookmarkSyncAuditEntry = {
+			job: "bookmarks-sync",
+			ok: true,
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			before,
+			after: before,
+			added: 0,
+			skipped: "already-running",
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	}
+
+	try {
+		const sync = await syncTimelineCollection({
+			kind: "bookmarks",
+			account,
+			mode,
+			limit,
+			all: effectiveAll,
+			maxPages,
+			refresh,
+			cacheTtlMs,
+		});
+		const backup = await maybeAutoSyncBackup(database);
+		const finished = Date.now();
+		const after = { bookmarks: countBookmarks(database) };
+		const entry: BookmarkSyncAuditEntry = {
+			job: "bookmarks-sync",
+			ok: true,
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			before,
+			after,
+			added: Math.max(0, after.bookmarks - before.bookmarks),
+			sync: {
+				source: sync.source,
+				count: sync.count,
+				accountId: sync.accountId,
+			},
+			backup,
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	} catch (error) {
+		const finished = Date.now();
+		const after = { bookmarks: countBookmarks(database) };
+		const entry: BookmarkSyncAuditEntry = {
+			job: "bookmarks-sync",
+			ok: false,
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			before,
+			after,
+			added: Math.max(0, after.bookmarks - before.bookmarks),
+			error: messageFromError(error),
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	} finally {
+		await releaseLock();
+	}
+}
+
+function xmlEscape(value: string) {
+	return value
+		.replaceAll("&", "&amp;")
+		.replaceAll("<", "&lt;")
+		.replaceAll(">", "&gt;");
+}
+
+function stringEntry(value: string) {
+	return `<string>${xmlEscape(value)}</string>`;
+}
+
+function shellQuote(value: string) {
+	return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+function buildProgramArguments({
+	program = "birdclaw",
+	mode = "auto",
+	limit = DEFAULT_BOOKMARK_SYNC_LIMIT,
+	all = false,
+	maxPages = DEFAULT_BOOKMARK_SYNC_MAX_PAGES,
+	refresh = true,
+	cacheTtlSeconds,
+	logPath,
+	envFile,
+}: BookmarkSyncLaunchAgentOptions) {
+	const args =
+		path.isAbsolute(program) || program.includes("/")
+			? [program]
+			: ["/usr/bin/env", program];
+	args.push(
+		"--json",
+		"jobs",
+		"sync-bookmarks",
+		"--mode",
+		mode,
+		"--limit",
+		String(limit),
+		"--log",
+		resolvePath(logPath ?? getDefaultBookmarkSyncAuditLogPath()),
+	);
+	if (all || maxPages !== undefined) {
+		args.push("--all");
+	}
+	if (maxPages !== undefined) {
+		args.push("--max-pages", String(maxPages));
+	}
+	if (refresh) {
+		args.push("--refresh");
+	}
+	if (cacheTtlSeconds !== undefined) {
+		args.push("--cache-ttl", String(cacheTtlSeconds));
+	}
+	if (envFile) {
+		const resolvedEnvFile = resolvePath(envFile);
+		return [
+			"/bin/bash",
+			"-lc",
+			[
+				"set -a",
+				`[ ! -f ${shellQuote(resolvedEnvFile)} ] || . ${shellQuote(resolvedEnvFile)}`,
+				"set +a",
+				`exec ${args.map(shellQuote).join(" ")}`,
+			].join("; "),
+		];
+	}
+	return args;
+}
+
+export function buildBookmarkSyncLaunchAgentPlist(
+	options: BookmarkSyncLaunchAgentOptions = {},
+) {
+	const label = options.label ?? DEFAULT_LAUNCHD_LABEL;
+	const intervalSeconds =
+		options.intervalSeconds ?? DEFAULT_BOOKMARK_SYNC_INTERVAL_SECONDS;
+	const logPath = resolvePath(
+		options.logPath ?? getDefaultBookmarkSyncAuditLogPath(),
+	);
+	const stdoutPath = resolvePath(
+		options.stdoutPath ??
+			path.join(getBirdclawPaths().rootDir, "logs", "bookmarks-sync.out.log"),
+	);
+	const stderrPath = resolvePath(
+		options.stderrPath ??
+			path.join(getBirdclawPaths().rootDir, "logs", "bookmarks-sync.err.log"),
+	);
+	const programArguments = buildProgramArguments({ ...options, logPath });
+	const plist = `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  ${stringEntry(label)}
+  <key>ProgramArguments</key>
+  <array>
+    ${programArguments.map(stringEntry).join("\n    ")}
+  </array>
+  <key>StartInterval</key>
+  <integer>${String(intervalSeconds)}</integer>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>StandardOutPath</key>
+  ${stringEntry(stdoutPath)}
+  <key>StandardErrorPath</key>
+  ${stringEntry(stderrPath)}
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    ${stringEntry(DEFAULT_LAUNCHD_PATH)}
+  </dict>
+</dict>
+</plist>
+`;
+	return {
+		label,
+		intervalSeconds,
+		logPath,
+		...(options.envFile ? { envFile: resolvePath(options.envFile) } : {}),
+		stdoutPath,
+		stderrPath,
+		programArguments,
+		plist,
+	};
+}
+
+export async function installBookmarkSyncLaunchAgent(
+	options: BookmarkSyncLaunchAgentOptions = {},
+): Promise<BookmarkSyncLaunchAgentInstallResult> {
+	ensureBirdclawDirs();
+	const agent = buildBookmarkSyncLaunchAgentPlist(options);
+	const launchAgentsDir = resolvePath(
+		options.launchAgentsDir ?? "~/Library/LaunchAgents",
+	);
+	const plistPath = path.join(launchAgentsDir, `${agent.label}.plist`);
+	await fs.mkdir(launchAgentsDir, { recursive: true });
+	await fs.mkdir(path.dirname(agent.logPath), { recursive: true });
+	await fs.mkdir(path.dirname(agent.stdoutPath), { recursive: true });
+	await fs.mkdir(path.dirname(agent.stderrPath), { recursive: true });
+	await fs.writeFile(plistPath, agent.plist, "utf8");
+
+	let loaded = false;
+	if (options.load !== false) {
+		await execFileAsync("launchctl", ["unload", plistPath]).catch(() => {});
+		await execFileAsync("launchctl", ["load", "-w", plistPath]);
+		loaded = true;
+	}
+
+	return {
+		ok: true,
+		label: agent.label,
+		plistPath,
+		loaded,
+		programArguments: agent.programArguments,
+		logPath: agent.logPath,
+		stdoutPath: agent.stdoutPath,
+		stderrPath: agent.stderrPath,
+		intervalSeconds: agent.intervalSeconds,
+		...(agent.envFile ? { envFile: agent.envFile } : {}),
+	};
+}

package/src/lib/config.ts

@@ -21,6 +21,12 @@ export interface BirdclawConfig {
 	actions?: {
 		transport?: ActionsTransport;
 	};
+	backup?: {
+		repoPath?: string;
+		remote?: string;
+		autoSync?: boolean;
+		staleAfterSeconds?: number;
+	};
 }
 
 let cachedPaths: BirdclawPaths | undefined;

package/src/lib/db.ts

@@ -7,6 +7,7 @@ export interface AccountsTable {
 	id: string;
 	name: string;
 	handle: string;
+	external_user_id: string | null;
 	transport: string;
 	is_default: number;
 	created_at: string;
@@ -41,6 +42,16 @@ export interface TweetsTable {
 	quoted_tweet_id: string | null;
 }
 
+export interface TweetCollectionsTable {
+	account_id: string;
+	tweet_id: string;
+	kind: "likes" | "bookmarks";
+	collected_at: string | null;
+	source: string;
+	raw_json: string;
+	updated_at: string;
+}
+
 export interface DmConversationsTable {
 	id: string;
 	account_id: string;
@@ -105,6 +116,7 @@ export interface BirdclawDatabase {
 	accounts: AccountsTable;
 	profiles: ProfilesTable;
 	tweets: TweetsTable;
+	tweet_collections: TweetCollectionsTable;
 	dm_conversations: DmConversationsTable;
 	dm_messages: DmMessagesTable;
 	tweet_actions: TweetActionsTable;
@@ -117,6 +129,10 @@ export interface BirdclawDatabase {
 let nativeDb: BetterSqlite3.Database | undefined;
 let kyselyDb: Kysely<BirdclawDatabase> | undefined;
 
+export interface InitDatabaseOptions {
+	seedDemoData?: boolean;
+}
+
 const BASE_SCHEMA_SQL = `
   pragma journal_mode = wal;
   pragma busy_timeout = 5000;
@@ -126,6 +142,7 @@ const BASE_SCHEMA_SQL = `
     id text primary key,
     name text not null,
     handle text not null unique,
+    external_user_id text,
     transport text not null,
     is_default integer not null default 0,
     created_at text not null
@@ -160,6 +177,17 @@ const BASE_SCHEMA_SQL = `
     quoted_tweet_id text
   );
 
+  create table if not exists tweet_collections (
+    account_id text not null,
+    tweet_id text not null,
+    kind text not null,
+    collected_at text,
+    source text not null,
+    raw_json text not null default '{}',
+    updated_at text not null,
+    primary key (account_id, tweet_id, kind)
+  );
+
   create table if not exists dm_conversations (
     id text primary key,
     account_id text not null,
@@ -238,6 +266,8 @@ const INDEX_SQL = `
   create index if not exists idx_tweets_kind_created on tweets(kind, created_at desc);
   create index if not exists idx_tweets_account_created on tweets(account_id, created_at desc);
   create index if not exists idx_tweets_quoted on tweets(quoted_tweet_id);
+  create index if not exists idx_tweet_collections_kind_account on tweet_collections(kind, account_id, collected_at desc, tweet_id);
+  create index if not exists idx_tweet_collections_tweet on tweet_collections(tweet_id);
   create index if not exists idx_dm_conversations_account on dm_conversations(account_id, last_message_at desc);
   create index if not exists idx_dm_messages_conversation on dm_messages(conversation_id, created_at asc);
   create index if not exists idx_profiles_followers on profiles(followers_count desc);
@@ -281,21 +311,69 @@ function ensureProfileAvatarColumns(db: BetterSqlite3.Database) {
 	}
 }
 
+function ensureAccountExternalUserIdColumn(db: BetterSqlite3.Database) {
+	const columnNames = getColumnNames(db, "accounts");
+	if (!columnNames.has("external_user_id")) {
+		db.exec("alter table accounts add column external_user_id text");
+	}
+}
+
+function ensureTweetCollectionsTable(db: BetterSqlite3.Database) {
+	db.exec(`
+    create table if not exists tweet_collections (
+      account_id text not null,
+      tweet_id text not null,
+      kind text not null,
+      collected_at text,
+      source text not null,
+      raw_json text not null default '{}',
+      updated_at text not null,
+      primary key (account_id, tweet_id, kind)
+    );
+  `);
+}
+
+function backfillTweetCollections(db: BetterSqlite3.Database) {
+	const now = new Date().toISOString();
+	const insert = db.prepare(`
+    insert or ignore into tweet_collections (
+      account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
+    )
+    select account_id, id, ?, null, 'legacy', '{}', ?
+    from tweets
+    where
+      case
+        when ? = 'likes' then liked
+        else bookmarked
+      end = 1
+  `);
+
+	db.transaction(() => {
+		insert.run("likes", now, "likes");
+		insert.run("bookmarks", now, "bookmarks");
+	})();
+}
+
 function ensureSchemaIndexes(db: BetterSqlite3.Database) {
 	db.exec(INDEX_SQL);
 }
 
-function initDatabase() {
+function initDatabase(options: InitDatabaseOptions = {}) {
 	ensureBirdclawDirs();
 
 	if (!nativeDb) {
 		const { dbPath } = getBirdclawPaths();
 		nativeDb = new BetterSqlite3(dbPath);
 		nativeDb.exec(BASE_SCHEMA_SQL);
+		ensureAccountExternalUserIdColumn(nativeDb);
 		ensureTweetMetadataColumns(nativeDb);
 		ensureProfileAvatarColumns(nativeDb);
+		ensureTweetCollectionsTable(nativeDb);
 		ensureSchemaIndexes(nativeDb);
-		seedDemoData(nativeDb);
+		if (options.seedDemoData !== false) {
+			seedDemoData(nativeDb);
+		}
+		backfillTweetCollections(nativeDb);
 	}
 
 	if (!kyselyDb) {
@@ -307,8 +385,8 @@ function initDatabase() {
 	}
 }
 
-export function getNativeDb() {
-	initDatabase();
+export function getNativeDb(options: InitDatabaseOptions = {}) {
+	initDatabase(options);
 	return nativeDb as BetterSqlite3.Database;
 }
 

package/src/lib/mentions-live.ts

@@ -10,6 +10,7 @@ import type {
 	TweetEntities,
 	XurlMentionData,
 	XurlMentionsResponse,
+	XurlMentionUser,
 } from "./types";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 import { listMentionsViaXurl, lookupUsersByHandles } from "./xurl";
@@ -257,11 +258,7 @@ function mergeMentionPayloads(
 ): XurlMentionsResponse {
 	const tweets: XurlMentionData[] = [];
 	const seenTweetIds = new Set<string>();
-	const users: XurlMentionsResponse["includes"] extends { users?: infer T }
-		? T extends Array<infer U>
-			? U[]
-			: never
-		: never = [];
+	const users: XurlMentionUser[] = [];
 	const seenUserIds = new Set<string>();
 
 	for (const page of pages) {

package/src/lib/moderation-target.ts

@@ -106,7 +106,7 @@ export async function resolveProfile(
 		return local;
 	}
 
-	let user: Record<string, unknown> | undefined;
+	let user: XurlMentionUser | undefined;
 	let lastError: unknown;
 
 	try {
@@ -146,18 +146,18 @@ export async function resolveProfile(
 	}
 
 	if (local) {
-		return upsertProfileFromXUser(db, user as XurlMentionUser);
+		return upsertProfileFromXUser(db, user);
 	}
 
 	const username = String(user.username ?? "").replace(/^@/, "");
 	if (username) {
 		const localByHandle = resolveLocalProfile(db, username);
 		if (localByHandle) {
-			return upsertProfileFromXUser(db, user as XurlMentionUser);
+			return upsertProfileFromXUser(db, user);
 		}
 	}
 
-	return upsertProfileFromXUser(db, user as XurlMentionUser);
+	return upsertProfileFromXUser(db, user);
 }
 
 export async function getAuthenticatedUserId() {

package/src/lib/mutes-write.ts

@@ -20,7 +20,7 @@ export async function addMute(
 	const transport = await runModerationAction({
 		action: "mute",
 		query: actionQuery,
-		targetUserId: resolved.externalUserId,
+		targetUserId: resolved.externalUserId ?? undefined,
 		transport: options.transport,
 	});
 
@@ -92,7 +92,7 @@ export async function removeMute(
 	const transport = await runModerationAction({
 		action: "unmute",
 		query: actionQuery,
-		targetUserId: resolved.externalUserId,
+		targetUserId: resolved.externalUserId ?? undefined,
 		transport: options.transport,
 	});