birdclaw 0.5.0 → 0.6.0

package/src/lib/account-sync-job.ts

@@ -0,0 +1,654 @@
+import { execFile } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { maybeAutoSyncBackup, type BackupAutoUpdateResult } from "./backup";
+import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
+import { getNativeDb } from "./db";
+import { syncDirectMessagesViaCachedBird } from "./dms-live";
+import { syncMentionThreads } from "./mention-threads-live";
+import { syncMentions } from "./mentions-live";
+import type { Database } from "./sqlite";
+import {
+	syncTimelineCollection,
+	type TimelineCollectionKind,
+	type TimelineCollectionMode,
+} from "./timeline-collections-live";
+import { syncHomeTimeline } from "./timeline-live";
+
+const execFileAsync = promisify(execFile);
+const DEFAULT_ACCOUNT_SYNC_INTERVAL_SECONDS = 30 * 60;
+const DEFAULT_ACCOUNT_SYNC_LIMIT = 100;
+const DEFAULT_ACCOUNT_SYNC_MAX_PAGES = 3;
+const DEFAULT_ACCOUNT_SYNC_LABEL = "com.steipete.birdclaw.account-sync";
+const DEFAULT_LAUNCHD_PATH =
+	"/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin";
+const DEFAULT_LOCK_STALE_MS = 60 * 60 * 1000;
+
+export type AccountSyncStepKind =
+	| "timeline"
+	| "mentions"
+	| "mention-threads"
+	| "likes"
+	| "bookmarks"
+	| "dms";
+
+export interface AccountSyncJobOptions {
+	account?: string;
+	steps?: AccountSyncStepKind[];
+	mode?: TimelineCollectionMode;
+	limit?: number;
+	maxPages?: number;
+	refresh?: boolean;
+	cacheTtlMs?: number;
+	allowBirdAccount?: boolean;
+	logPath?: string;
+	lockPath?: string;
+	db?: Database;
+}
+
+export interface AccountSyncAuditStep {
+	kind: AccountSyncStepKind;
+	ok: boolean;
+	count: number;
+	source?: string;
+	error?: string;
+}
+
+export interface AccountSyncAuditEntry {
+	job: "account-sync";
+	ok: boolean;
+	startedAt: string;
+	finishedAt: string;
+	durationMs: number;
+	host: string;
+	pid: number;
+	options: {
+		account?: string;
+		steps: AccountSyncStepKind[];
+		mode: TimelineCollectionMode;
+		limit: number;
+		maxPages: number;
+		refresh: boolean;
+		cacheTtlMs?: number;
+		allowBirdAccount?: boolean;
+	};
+	steps: AccountSyncAuditStep[];
+	skipped?: "already-running";
+	backup?: BackupAutoUpdateResult;
+	error?: string;
+}
+
+export interface AccountSyncLaunchAgentOptions {
+	label?: string;
+	intervalSeconds?: number;
+	program?: string;
+	account?: string;
+	steps?: AccountSyncStepKind[];
+	mode?: TimelineCollectionMode;
+	limit?: number;
+	maxPages?: number;
+	refresh?: boolean;
+	allowBirdAccount?: boolean;
+	cacheTtlSeconds?: number;
+	logPath?: string;
+	envFile?: string;
+	stdoutPath?: string;
+	stderrPath?: string;
+	launchAgentsDir?: string;
+	load?: boolean;
+}
+
+export interface AccountSyncLaunchAgentInstallResult {
+	ok: true;
+	label: string;
+	plistPath: string;
+	loaded: boolean;
+	programArguments: string[];
+	logPath: string;
+	stdoutPath: string;
+	stderrPath: string;
+	intervalSeconds: number;
+	envFile?: string;
+}
+
+const DEFAULT_STEPS: AccountSyncStepKind[] = [
+	"timeline",
+	"mentions",
+	"mention-threads",
+	"likes",
+	"bookmarks",
+	"dms",
+];
+
+function expandHome(input: string) {
+	return input === "~" || input.startsWith("~/")
+		? path.join(os.homedir(), input.slice(2))
+		: input;
+}
+
+function resolvePath(input: string) {
+	return path.resolve(expandHome(input));
+}
+
+export function getDefaultAccountSyncAuditLogPath() {
+	return path.join(getBirdclawPaths().rootDir, "audit", "account-sync.jsonl");
+}
+
+export function getDefaultAccountSyncLockPath() {
+	return path.join(getBirdclawPaths().rootDir, "locks", "account-sync.lock");
+}
+
+function messageFromError(error: unknown) {
+	return error instanceof Error ? error.message : String(error);
+}
+
+async function appendAuditEntry(logPath: string, entry: AccountSyncAuditEntry) {
+	await fs.mkdir(path.dirname(logPath), { recursive: true });
+	await fs.appendFile(logPath, `${JSON.stringify(entry)}\n`, "utf8");
+}
+
+async function acquireLock(lockPath: string) {
+	await fs.mkdir(path.dirname(lockPath), { recursive: true });
+	try {
+		const handle = await fs.open(lockPath, "wx");
+		await handle.writeFile(
+			`${JSON.stringify({
+				pid: process.pid,
+				host: os.hostname(),
+				startedAt: new Date().toISOString(),
+			})}\n`,
+			"utf8",
+		);
+		await handle.close();
+		return async () => {
+			await fs.rm(lockPath, { force: true });
+		};
+	} catch (error) {
+		if (
+			typeof error === "object" &&
+			error !== null &&
+			"code" in error &&
+			error.code === "EEXIST"
+		) {
+			const stats = await fs.stat(lockPath).catch(() => undefined);
+			if (stats && Date.now() - stats.mtimeMs > DEFAULT_LOCK_STALE_MS) {
+				await fs.rm(lockPath, { force: true });
+				return acquireLock(lockPath);
+			}
+			return undefined;
+		}
+		throw error;
+	}
+}
+
+function readNumber(value: unknown, key: string): number {
+	if (!value || typeof value !== "object") return 0;
+	const raw = (value as Record<string, unknown>)[key];
+	return typeof raw === "number" && Number.isFinite(raw) ? raw : 0;
+}
+
+function readString(value: unknown, key: string) {
+	if (!value || typeof value !== "object") return undefined;
+	const raw = (value as Record<string, unknown>)[key];
+	return typeof raw === "string" ? raw : undefined;
+}
+
+function defaultAccountId(db: Database) {
+	const row = db
+		.prepare(
+			`
+      select id
+      from accounts
+      order by is_default desc, created_at asc
+      limit 1
+      `,
+		)
+		.get() as { id: string } | undefined;
+	return row?.id;
+}
+
+function isExplicitNonDefaultAccount(
+	db: Database,
+	account: string | undefined,
+) {
+	if (!account) return false;
+	return account !== defaultAccountId(db);
+}
+
+function birdAccountError(kind: AccountSyncStepKind) {
+	return `Bird-backed ${kind} sync requires --allow-bird-account for non-default accounts; source matching cookies with --env-path first.`;
+}
+
+function resolveCollectionModeForAccount({
+	mode,
+	allowBirdAccount,
+}: {
+	mode: TimelineCollectionMode;
+	allowBirdAccount: boolean | undefined;
+}) {
+	if (allowBirdAccount || mode === "xurl") return mode;
+	return mode === "bird" ? undefined : "xurl";
+}
+
+async function runStep({
+	kind,
+	account,
+	mode,
+	limit,
+	maxPages,
+	refresh,
+	cacheTtlMs,
+	allowBirdAccount,
+}: Required<
+	Pick<AccountSyncJobOptions, "mode" | "limit" | "maxPages" | "refresh">
+> &
+	Pick<AccountSyncJobOptions, "account" | "cacheTtlMs" | "allowBirdAccount"> & {
+		kind: AccountSyncStepKind;
+	}): Promise<AccountSyncAuditStep> {
+	try {
+		if (kind === "timeline") {
+			if (!allowBirdAccount) {
+				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
+			}
+			const result = await syncHomeTimeline({
+				account,
+				limit,
+				following: true,
+				refresh,
+				cacheTtlMs,
+			});
+			return {
+				kind,
+				ok: true,
+				count: readNumber(result, "count"),
+				source: readString(result, "source"),
+			};
+		}
+		if (kind === "mentions") {
+			if (!allowBirdAccount) {
+				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
+			}
+			const result = await syncMentions({
+				account,
+				mode: "bird",
+				limit,
+				maxPages,
+				refresh,
+				cacheTtlMs,
+			});
+			return {
+				kind,
+				ok: true,
+				count: readNumber(result, "count"),
+				source: readString(result, "source"),
+			};
+		}
+		if (kind === "mention-threads") {
+			const result = await syncMentionThreads({
+				account,
+				mode: "xurl",
+				limit: Math.min(30, limit),
+				delayMs: 1500,
+				timeoutMs: 15000,
+			});
+			return {
+				kind,
+				ok: true,
+				count: readNumber(result, "mergedTweets"),
+				source: readString(result, "source"),
+			};
+		}
+		if (kind === "dms") {
+			const dmMode = allowBirdAccount
+				? mode
+				: mode === "bird"
+					? undefined
+					: "xurl";
+			if (!dmMode) {
+				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
+			}
+			const result = await syncDirectMessagesViaCachedBird({
+				account,
+				mode: dmMode,
+				limit: Math.min(50, limit),
+				refresh,
+				cacheTtlMs,
+			});
+			return {
+				kind,
+				ok: true,
+				count: readNumber(result, "messages"),
+				source: readString(result, "source"),
+			};
+		}
+
+		const collectionKind = kind as TimelineCollectionKind;
+		const collectionMode = resolveCollectionModeForAccount({
+			mode,
+			allowBirdAccount,
+		});
+		if (!collectionMode) {
+			return { kind, ok: false, count: 0, error: birdAccountError(kind) };
+		}
+
+		const result = await syncTimelineCollection({
+			kind: collectionKind,
+			account,
+			mode: collectionMode,
+			limit,
+			all: true,
+			maxPages,
+			refresh,
+			cacheTtlMs,
+			earlyStop: true,
+		});
+		return {
+			kind,
+			ok: true,
+			count: readNumber(result, "count"),
+			source: readString(result, "source"),
+		};
+	} catch (error) {
+		return {
+			kind,
+			ok: false,
+			count: 0,
+			error: messageFromError(error),
+		};
+	}
+}
+
+export async function runAccountSyncJob({
+	account,
+	steps = DEFAULT_STEPS,
+	mode = "auto",
+	limit = DEFAULT_ACCOUNT_SYNC_LIMIT,
+	maxPages = DEFAULT_ACCOUNT_SYNC_MAX_PAGES,
+	refresh = true,
+	cacheTtlMs,
+	allowBirdAccount,
+	logPath,
+	lockPath,
+	db,
+}: AccountSyncJobOptions = {}): Promise<AccountSyncAuditEntry> {
+	ensureBirdclawDirs();
+	const database = db ?? getNativeDb({ seedDemoData: false });
+	const resolvedLogPath = resolvePath(
+		logPath ?? getDefaultAccountSyncAuditLogPath(),
+	);
+	const resolvedLockPath = resolvePath(
+		lockPath ?? getDefaultAccountSyncLockPath(),
+	);
+	const started = Date.now();
+	const startedAt = new Date(started).toISOString();
+	const options = {
+		...(account ? { account } : {}),
+		steps,
+		mode,
+		limit,
+		maxPages,
+		refresh,
+		...(cacheTtlMs === undefined ? {} : { cacheTtlMs }),
+		...(allowBirdAccount ? { allowBirdAccount } : {}),
+	};
+	const birdAccountAllowed =
+		!isExplicitNonDefaultAccount(database, account) ||
+		Boolean(allowBirdAccount);
+
+	const releaseLock = await acquireLock(resolvedLockPath);
+	if (!releaseLock) {
+		const finished = Date.now();
+		const entry: AccountSyncAuditEntry = {
+			job: "account-sync",
+			ok: true,
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			steps: [],
+			skipped: "already-running",
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	}
+
+	const stepResults: AccountSyncAuditStep[] = [];
+	try {
+		for (const kind of steps) {
+			stepResults.push(
+				await runStep({
+					kind,
+					account,
+					mode,
+					limit,
+					maxPages,
+					refresh,
+					cacheTtlMs,
+					allowBirdAccount: birdAccountAllowed,
+				}),
+			);
+		}
+		const backup = await maybeAutoSyncBackup(database);
+		const finished = Date.now();
+		const entry: AccountSyncAuditEntry = {
+			job: "account-sync",
+			ok: stepResults.every((step) => step.ok),
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			steps: stepResults,
+			backup,
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	} catch (error) {
+		const finished = Date.now();
+		const entry: AccountSyncAuditEntry = {
+			job: "account-sync",
+			ok: false,
+			startedAt,
+			finishedAt: new Date(finished).toISOString(),
+			durationMs: finished - started,
+			host: os.hostname(),
+			pid: process.pid,
+			options,
+			steps: stepResults,
+			error: messageFromError(error),
+		};
+		await appendAuditEntry(resolvedLogPath, entry);
+		return entry;
+	} finally {
+		await releaseLock();
+	}
+}
+
+function xmlEscape(value: string) {
+	return value
+		.replaceAll("&", "&amp;")
+		.replaceAll("<", "&lt;")
+		.replaceAll(">", "&gt;");
+}
+
+function stringEntry(value: string) {
+	return `<string>${xmlEscape(value)}</string>`;
+}
+
+function shellQuote(value: string) {
+	return `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+function buildProgramArguments({
+	program = "birdclaw",
+	account,
+	steps,
+	mode = "auto",
+	limit = DEFAULT_ACCOUNT_SYNC_LIMIT,
+	maxPages = DEFAULT_ACCOUNT_SYNC_MAX_PAGES,
+	refresh = true,
+	allowBirdAccount,
+	cacheTtlSeconds,
+	logPath,
+	envFile,
+}: AccountSyncLaunchAgentOptions) {
+	const args =
+		path.isAbsolute(program) || program.includes("/")
+			? [program]
+			: ["/usr/bin/env", program];
+	args.push(
+		"--json",
+		"jobs",
+		"sync-account",
+		"--mode",
+		mode,
+		"--limit",
+		String(limit),
+		"--max-pages",
+		String(maxPages),
+		"--log",
+		resolvePath(logPath ?? getDefaultAccountSyncAuditLogPath()),
+	);
+	if (account) {
+		args.push("--account", account);
+	}
+	if (steps?.length) {
+		args.push("--steps", steps.join(","));
+	}
+	if (refresh) {
+		args.push("--refresh");
+	}
+	if (allowBirdAccount) {
+		args.push("--allow-bird-account");
+	}
+	if (cacheTtlSeconds !== undefined) {
+		args.push("--cache-ttl", String(cacheTtlSeconds));
+	}
+	if (envFile) {
+		const resolvedEnvFile = resolvePath(envFile);
+		return [
+			"/bin/bash",
+			"-lc",
+			[
+				"set -a",
+				`[ ! -f ${shellQuote(resolvedEnvFile)} ] || . ${shellQuote(resolvedEnvFile)}`,
+				"set +a",
+				`exec ${args.map(shellQuote).join(" ")}`,
+			].join("; "),
+		];
+	}
+	return args;
+}
+
+export function buildAccountSyncLaunchAgentPlist(
+	options: AccountSyncLaunchAgentOptions = {},
+) {
+	const label = options.label ?? DEFAULT_ACCOUNT_SYNC_LABEL;
+	const intervalSeconds =
+		options.intervalSeconds ?? DEFAULT_ACCOUNT_SYNC_INTERVAL_SECONDS;
+	const logPath = resolvePath(
+		options.logPath ?? getDefaultAccountSyncAuditLogPath(),
+	);
+	const stdoutPath = resolvePath(
+		options.stdoutPath ??
+			path.join(getBirdclawPaths().rootDir, "logs", "account-sync.out.log"),
+	);
+	const stderrPath = resolvePath(
+		options.stderrPath ??
+			path.join(getBirdclawPaths().rootDir, "logs", "account-sync.err.log"),
+	);
+	const programArguments = buildProgramArguments({ ...options, logPath });
+	const plist = `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  ${stringEntry(label)}
+  <key>ProgramArguments</key>
+  <array>
+    ${programArguments.map(stringEntry).join("\n    ")}
+  </array>
+  <key>StartInterval</key>
+  <integer>${String(intervalSeconds)}</integer>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>StandardOutPath</key>
+  ${stringEntry(stdoutPath)}
+  <key>StandardErrorPath</key>
+  ${stringEntry(stderrPath)}
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    ${stringEntry(DEFAULT_LAUNCHD_PATH)}
+  </dict>
+</dict>
+</plist>
+`;
+	return {
+		label,
+		intervalSeconds,
+		logPath,
+		...(options.envFile ? { envFile: resolvePath(options.envFile) } : {}),
+		stdoutPath,
+		stderrPath,
+		programArguments,
+		plist,
+	};
+}
+
+export async function installAccountSyncLaunchAgent(
+	options: AccountSyncLaunchAgentOptions = {},
+): Promise<AccountSyncLaunchAgentInstallResult> {
+	ensureBirdclawDirs();
+	const agent = buildAccountSyncLaunchAgentPlist(options);
+	const launchAgentsDir = resolvePath(
+		options.launchAgentsDir ?? "~/Library/LaunchAgents",
+	);
+	const plistPath = path.join(launchAgentsDir, `${agent.label}.plist`);
+	await fs.mkdir(launchAgentsDir, { recursive: true });
+	await fs.mkdir(path.dirname(agent.logPath), { recursive: true });
+	await fs.mkdir(path.dirname(agent.stdoutPath), { recursive: true });
+	await fs.mkdir(path.dirname(agent.stderrPath), { recursive: true });
+	await fs.writeFile(plistPath, agent.plist, "utf8");
+
+	let loaded = false;
+	if (options.load !== false) {
+		await execFileAsync("launchctl", ["unload", plistPath]).catch(() => {});
+		await execFileAsync("launchctl", ["load", "-w", plistPath]);
+		loaded = true;
+	}
+
+	return {
+		ok: true,
+		label: agent.label,
+		plistPath,
+		loaded,
+		programArguments: agent.programArguments,
+		logPath: agent.logPath,
+		stdoutPath: agent.stdoutPath,
+		stderrPath: agent.stderrPath,
+		intervalSeconds: agent.intervalSeconds,
+		...(agent.envFile ? { envFile: agent.envFile } : {}),
+	};
+}
+
+export function parseAccountSyncSteps(value: string | undefined) {
+	if (!value) return undefined;
+	const valid = new Set<AccountSyncStepKind>(DEFAULT_STEPS);
+	const steps = value
+		.split(",")
+		.map((step) => step.trim())
+		.filter(Boolean);
+	if (steps.length === 0) return undefined;
+	for (const step of steps) {
+		if (!valid.has(step as AccountSyncStepKind)) {
+			throw new Error(`--steps must contain ${Array.from(valid).join(", ")}`);
+		}
+	}
+	return steps as AccountSyncStepKind[];
+}