birdclaw 0.8.3 → 0.8.5

package/src/lib/account-sync-job.ts

@@ -1,666 +0,0 @@
-import { execFile } from "node:child_process";
-import fs from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-import { promisify } from "node:util";
-import { maybeAutoSyncBackup, type BackupAutoUpdateResult } from "./backup";
-import { ensureBirdclawDirs, getBirdclawPaths } from "./config";
-import { getNativeDb } from "./db";
-import { syncDirectMessagesViaCachedBird } from "./dms-live";
-import { syncMentionThreads } from "./mention-threads-live";
-import { syncMentions } from "./mentions-live";
-import type { Database } from "./sqlite";
-import {
-	syncTimelineCollection,
-	type TimelineCollectionKind,
-	type TimelineCollectionMode,
-} from "./timeline-collections-live";
-import { syncHomeTimeline } from "./timeline-live";
-
-const execFileAsync = promisify(execFile);
-const DEFAULT_ACCOUNT_SYNC_INTERVAL_SECONDS = 30 * 60;
-const DEFAULT_ACCOUNT_SYNC_LIMIT = 100;
-const DEFAULT_ACCOUNT_SYNC_MAX_PAGES = 3;
-const DEFAULT_ACCOUNT_SYNC_LABEL = "com.steipete.birdclaw.account-sync";
-const DEFAULT_LAUNCHD_PATH =
-	"/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin";
-const DEFAULT_LOCK_STALE_MS = 60 * 60 * 1000;
-
-export type AccountSyncStepKind =
-	| "timeline"
-	| "mentions"
-	| "mention-threads"
-	| "likes"
-	| "bookmarks"
-	| "dms";
-
-export interface AccountSyncJobOptions {
-	account?: string;
-	steps?: AccountSyncStepKind[];
-	mode?: TimelineCollectionMode;
-	limit?: number;
-	maxPages?: number;
-	refresh?: boolean;
-	cacheTtlMs?: number;
-	allowBirdAccount?: boolean;
-	logPath?: string;
-	lockPath?: string;
-	db?: Database;
-}
-
-export interface AccountSyncAuditStep {
-	kind: AccountSyncStepKind;
-	ok: boolean;
-	count: number;
-	source?: string;
-	error?: string;
-}
-
-export interface AccountSyncAuditEntry {
-	job: "account-sync";
-	ok: boolean;
-	startedAt: string;
-	finishedAt: string;
-	durationMs: number;
-	host: string;
-	pid: number;
-	options: {
-		account?: string;
-		steps: AccountSyncStepKind[];
-		mode: TimelineCollectionMode;
-		limit: number;
-		maxPages: number;
-		refresh: boolean;
-		cacheTtlMs?: number;
-		allowBirdAccount?: boolean;
-	};
-	steps: AccountSyncAuditStep[];
-	skipped?: "already-running";
-	backup?: BackupAutoUpdateResult;
-	error?: string;
-}
-
-export interface AccountSyncLaunchAgentOptions {
-	label?: string;
-	intervalSeconds?: number;
-	program?: string;
-	account?: string;
-	steps?: AccountSyncStepKind[];
-	mode?: TimelineCollectionMode;
-	limit?: number;
-	maxPages?: number;
-	refresh?: boolean;
-	allowBirdAccount?: boolean;
-	cacheTtlSeconds?: number;
-	logPath?: string;
-	envFile?: string;
-	stdoutPath?: string;
-	stderrPath?: string;
-	launchAgentsDir?: string;
-	load?: boolean;
-}
-
-export interface AccountSyncLaunchAgentInstallResult {
-	ok: true;
-	label: string;
-	plistPath: string;
-	loaded: boolean;
-	programArguments: string[];
-	logPath: string;
-	stdoutPath: string;
-	stderrPath: string;
-	intervalSeconds: number;
-	envFile?: string;
-}
-
-const DEFAULT_STEPS: AccountSyncStepKind[] = [
-	"timeline",
-	"mentions",
-	"mention-threads",
-	"likes",
-	"bookmarks",
-	"dms",
-];
-
-function expandHome(input: string) {
-	return input === "~" || input.startsWith("~/")
-		? path.join(os.homedir(), input.slice(2))
-		: input;
-}
-
-function resolvePath(input: string) {
-	return path.resolve(expandHome(input));
-}
-
-export function getDefaultAccountSyncAuditLogPath() {
-	return path.join(getBirdclawPaths().rootDir, "audit", "account-sync.jsonl");
-}
-
-export function getDefaultAccountSyncLockPath() {
-	return path.join(getBirdclawPaths().rootDir, "locks", "account-sync.lock");
-}
-
-function messageFromError(error: unknown) {
-	return error instanceof Error ? error.message : String(error);
-}
-
-async function appendAuditEntry(logPath: string, entry: AccountSyncAuditEntry) {
-	await fs.mkdir(path.dirname(logPath), { recursive: true });
-	await fs.appendFile(logPath, `${JSON.stringify(entry)}\n`, "utf8");
-}
-
-async function acquireLock(lockPath: string) {
-	await fs.mkdir(path.dirname(lockPath), { recursive: true });
-	try {
-		const handle = await fs.open(lockPath, "wx");
-		await handle.writeFile(
-			`${JSON.stringify({
-				pid: process.pid,
-				host: os.hostname(),
-				startedAt: new Date().toISOString(),
-			})}\n`,
-			"utf8",
-		);
-		await handle.close();
-		return async () => {
-			await fs.rm(lockPath, { force: true });
-		};
-	} catch (error) {
-		if (
-			typeof error === "object" &&
-			error !== null &&
-			"code" in error &&
-			error.code === "EEXIST"
-		) {
-			const stats = await fs.stat(lockPath).catch(() => undefined);
-			if (stats && Date.now() - stats.mtimeMs > DEFAULT_LOCK_STALE_MS) {
-				await fs.rm(lockPath, { force: true });
-				return acquireLock(lockPath);
-			}
-			return undefined;
-		}
-		throw error;
-	}
-}
-
-function readNumber(value: unknown, key: string): number {
-	if (!value || typeof value !== "object") return 0;
-	const raw = (value as Record<string, unknown>)[key];
-	return typeof raw === "number" && Number.isFinite(raw) ? raw : 0;
-}
-
-function readString(value: unknown, key: string) {
-	if (!value || typeof value !== "object") return undefined;
-	const raw = (value as Record<string, unknown>)[key];
-	return typeof raw === "string" ? raw : undefined;
-}
-
-function defaultAccountId(db: Database) {
-	const row = db
-		.prepare(
-			`
-      select id
-      from accounts
-      order by is_default desc, created_at asc
-      limit 1
-      `,
-		)
-		.get() as { id: string } | undefined;
-	return row?.id;
-}
-
-function isExplicitNonDefaultAccount(
-	db: Database,
-	account: string | undefined,
-) {
-	if (!account) return false;
-	return account !== defaultAccountId(db);
-}
-
-function birdAccountError(kind: AccountSyncStepKind) {
-	return `Bird-backed ${kind} sync requires --allow-bird-account for non-default accounts; source matching cookies with --env-path first.`;
-}
-
-function resolveCollectionModeForAccount({
-	mode,
-	allowBirdAccount,
-}: {
-	mode: TimelineCollectionMode;
-	allowBirdAccount: boolean | undefined;
-}) {
-	if (allowBirdAccount || mode === "xurl") return mode;
-	return mode === "bird" ? undefined : "xurl";
-}
-
-async function runStep({
-	kind,
-	account,
-	mode,
-	limit,
-	maxPages,
-	refresh,
-	cacheTtlMs,
-	allowBirdAccount,
-}: Required<
-	Pick<AccountSyncJobOptions, "mode" | "limit" | "maxPages" | "refresh">
-> &
-	Pick<AccountSyncJobOptions, "account" | "cacheTtlMs" | "allowBirdAccount"> & {
-		kind: AccountSyncStepKind;
-	}): Promise<AccountSyncAuditStep> {
-	try {
-		if (kind === "timeline") {
-			const timelineMode =
-				mode === "auto"
-					? account
-						? allowBirdAccount
-							? "bird"
-							: "xurl"
-						: "auto"
-					: mode;
-			if (timelineMode === "bird" && !allowBirdAccount) {
-				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
-			}
-			const result = await syncHomeTimeline({
-				account,
-				mode: timelineMode,
-				limit,
-				maxPages,
-				following: true,
-				refresh,
-				cacheTtlMs,
-			});
-			return {
-				kind,
-				ok: true,
-				count: readNumber(result, "count"),
-				source: readString(result, "source"),
-			};
-		}
-		if (kind === "mentions") {
-			if (mode === "bird" && !allowBirdAccount) {
-				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
-			}
-			const mentionMode =
-				mode === "auto" ? (allowBirdAccount ? "auto" : "xurl") : mode;
-			const result = await syncMentions({
-				account,
-				mode: mentionMode,
-				limit,
-				maxPages,
-				refresh,
-				cacheTtlMs,
-			});
-			return {
-				kind,
-				ok: true,
-				count: readNumber(result, "count"),
-				source: readString(result, "source"),
-			};
-		}
-		if (kind === "mention-threads") {
-			const result = await syncMentionThreads({
-				account,
-				mode: "xurl",
-				limit: Math.min(30, limit),
-				delayMs: 1500,
-				timeoutMs: 15000,
-			});
-			return {
-				kind,
-				ok: true,
-				count: readNumber(result, "mergedTweets"),
-				source: readString(result, "source"),
-			};
-		}
-		if (kind === "dms") {
-			const dmMode = allowBirdAccount
-				? mode
-				: mode === "bird"
-					? undefined
-					: "xurl";
-			if (!dmMode) {
-				return { kind, ok: false, count: 0, error: birdAccountError(kind) };
-			}
-			const result = await syncDirectMessagesViaCachedBird({
-				account,
-				mode: dmMode,
-				limit: Math.min(50, limit),
-				refresh,
-				cacheTtlMs,
-			});
-			return {
-				kind,
-				ok: true,
-				count: readNumber(result, "messages"),
-				source: readString(result, "source"),
-			};
-		}
-
-		const collectionKind = kind as TimelineCollectionKind;
-		const collectionMode = resolveCollectionModeForAccount({
-			mode,
-			allowBirdAccount,
-		});
-		if (!collectionMode) {
-			return { kind, ok: false, count: 0, error: birdAccountError(kind) };
-		}
-
-		const result = await syncTimelineCollection({
-			kind: collectionKind,
-			account,
-			mode: collectionMode,
-			limit,
-			all: true,
-			maxPages,
-			refresh,
-			cacheTtlMs,
-			earlyStop: true,
-		});
-		return {
-			kind,
-			ok: true,
-			count: readNumber(result, "count"),
-			source: readString(result, "source"),
-		};
-	} catch (error) {
-		return {
-			kind,
-			ok: false,
-			count: 0,
-			error: messageFromError(error),
-		};
-	}
-}
-
-export async function runAccountSyncJob({
-	account,
-	steps = DEFAULT_STEPS,
-	mode = "auto",
-	limit = DEFAULT_ACCOUNT_SYNC_LIMIT,
-	maxPages = DEFAULT_ACCOUNT_SYNC_MAX_PAGES,
-	refresh = true,
-	cacheTtlMs,
-	allowBirdAccount,
-	logPath,
-	lockPath,
-	db,
-}: AccountSyncJobOptions = {}): Promise<AccountSyncAuditEntry> {
-	ensureBirdclawDirs();
-	const database = db ?? getNativeDb({ seedDemoData: false });
-	const resolvedLogPath = resolvePath(
-		logPath ?? getDefaultAccountSyncAuditLogPath(),
-	);
-	const resolvedLockPath = resolvePath(
-		lockPath ?? getDefaultAccountSyncLockPath(),
-	);
-	const started = Date.now();
-	const startedAt = new Date(started).toISOString();
-	const options = {
-		...(account ? { account } : {}),
-		steps,
-		mode,
-		limit,
-		maxPages,
-		refresh,
-		...(cacheTtlMs === undefined ? {} : { cacheTtlMs }),
-		...(allowBirdAccount ? { allowBirdAccount } : {}),
-	};
-	const birdAccountAllowed =
-		!isExplicitNonDefaultAccount(database, account) ||
-		Boolean(allowBirdAccount);
-
-	const releaseLock = await acquireLock(resolvedLockPath);
-	if (!releaseLock) {
-		const finished = Date.now();
-		const entry: AccountSyncAuditEntry = {
-			job: "account-sync",
-			ok: true,
-			startedAt,
-			finishedAt: new Date(finished).toISOString(),
-			durationMs: finished - started,
-			host: os.hostname(),
-			pid: process.pid,
-			options,
-			steps: [],
-			skipped: "already-running",
-		};
-		await appendAuditEntry(resolvedLogPath, entry);
-		return entry;
-	}
-
-	const stepResults: AccountSyncAuditStep[] = [];
-	try {
-		for (const kind of steps) {
-			stepResults.push(
-				await runStep({
-					kind,
-					account,
-					mode,
-					limit,
-					maxPages,
-					refresh,
-					cacheTtlMs,
-					allowBirdAccount: birdAccountAllowed,
-				}),
-			);
-		}
-		const backup = await maybeAutoSyncBackup(database);
-		const finished = Date.now();
-		const entry: AccountSyncAuditEntry = {
-			job: "account-sync",
-			ok: stepResults.every((step) => step.ok),
-			startedAt,
-			finishedAt: new Date(finished).toISOString(),
-			durationMs: finished - started,
-			host: os.hostname(),
-			pid: process.pid,
-			options,
-			steps: stepResults,
-			backup,
-		};
-		await appendAuditEntry(resolvedLogPath, entry);
-		return entry;
-	} catch (error) {
-		const finished = Date.now();
-		const entry: AccountSyncAuditEntry = {
-			job: "account-sync",
-			ok: false,
-			startedAt,
-			finishedAt: new Date(finished).toISOString(),
-			durationMs: finished - started,
-			host: os.hostname(),
-			pid: process.pid,
-			options,
-			steps: stepResults,
-			error: messageFromError(error),
-		};
-		await appendAuditEntry(resolvedLogPath, entry);
-		return entry;
-	} finally {
-		await releaseLock();
-	}
-}
-
-function xmlEscape(value: string) {
-	return value
-		.replaceAll("&", "&amp;")
-		.replaceAll("<", "&lt;")
-		.replaceAll(">", "&gt;");
-}
-
-function stringEntry(value: string) {
-	return `<string>${xmlEscape(value)}</string>`;
-}
-
-function shellQuote(value: string) {
-	return `'${value.replaceAll("'", "'\\''")}'`;
-}
-
-function buildProgramArguments({
-	program = "birdclaw",
-	account,
-	steps,
-	mode = "auto",
-	limit = DEFAULT_ACCOUNT_SYNC_LIMIT,
-	maxPages = DEFAULT_ACCOUNT_SYNC_MAX_PAGES,
-	refresh = true,
-	allowBirdAccount,
-	cacheTtlSeconds,
-	logPath,
-	envFile,
-}: AccountSyncLaunchAgentOptions) {
-	const args =
-		path.isAbsolute(program) || program.includes("/")
-			? [program]
-			: ["/usr/bin/env", program];
-	args.push(
-		"--json",
-		"jobs",
-		"sync-account",
-		"--mode",
-		mode,
-		"--limit",
-		String(limit),
-		"--max-pages",
-		String(maxPages),
-		"--log",
-		resolvePath(logPath ?? getDefaultAccountSyncAuditLogPath()),
-	);
-	if (account) {
-		args.push("--account", account);
-	}
-	if (steps?.length) {
-		args.push("--steps", steps.join(","));
-	}
-	if (refresh) {
-		args.push("--refresh");
-	}
-	if (allowBirdAccount) {
-		args.push("--allow-bird-account");
-	}
-	if (cacheTtlSeconds !== undefined) {
-		args.push("--cache-ttl", String(cacheTtlSeconds));
-	}
-	if (envFile) {
-		const resolvedEnvFile = resolvePath(envFile);
-		return [
-			"/bin/bash",
-			"-lc",
-			[
-				"set -a",
-				`[ ! -f ${shellQuote(resolvedEnvFile)} ] || . ${shellQuote(resolvedEnvFile)}`,
-				"set +a",
-				`exec ${args.map(shellQuote).join(" ")}`,
-			].join("; "),
-		];
-	}
-	return args;
-}
-
-export function buildAccountSyncLaunchAgentPlist(
-	options: AccountSyncLaunchAgentOptions = {},
-) {
-	const label = options.label ?? DEFAULT_ACCOUNT_SYNC_LABEL;
-	const intervalSeconds =
-		options.intervalSeconds ?? DEFAULT_ACCOUNT_SYNC_INTERVAL_SECONDS;
-	const logPath = resolvePath(
-		options.logPath ?? getDefaultAccountSyncAuditLogPath(),
-	);
-	const stdoutPath = resolvePath(
-		options.stdoutPath ??
-			path.join(getBirdclawPaths().rootDir, "logs", "account-sync.out.log"),
-	);
-	const stderrPath = resolvePath(
-		options.stderrPath ??
-			path.join(getBirdclawPaths().rootDir, "logs", "account-sync.err.log"),
-	);
-	const programArguments = buildProgramArguments({ ...options, logPath });
-	const plist = `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-  <key>Label</key>
-  ${stringEntry(label)}
-  <key>ProgramArguments</key>
-  <array>
-    ${programArguments.map(stringEntry).join("\n    ")}
-  </array>
-  <key>StartInterval</key>
-  <integer>${String(intervalSeconds)}</integer>
-  <key>RunAtLoad</key>
-  <true/>
-  <key>StandardOutPath</key>
-  ${stringEntry(stdoutPath)}
-  <key>StandardErrorPath</key>
-  ${stringEntry(stderrPath)}
-  <key>EnvironmentVariables</key>
-  <dict>
-    <key>PATH</key>
-    ${stringEntry(DEFAULT_LAUNCHD_PATH)}
-  </dict>
-</dict>
-</plist>
-`;
-	return {
-		label,
-		intervalSeconds,
-		logPath,
-		...(options.envFile ? { envFile: resolvePath(options.envFile) } : {}),
-		stdoutPath,
-		stderrPath,
-		programArguments,
-		plist,
-	};
-}
-
-export async function installAccountSyncLaunchAgent(
-	options: AccountSyncLaunchAgentOptions = {},
-): Promise<AccountSyncLaunchAgentInstallResult> {
-	ensureBirdclawDirs();
-	const agent = buildAccountSyncLaunchAgentPlist(options);
-	const launchAgentsDir = resolvePath(
-		options.launchAgentsDir ?? "~/Library/LaunchAgents",
-	);
-	const plistPath = path.join(launchAgentsDir, `${agent.label}.plist`);
-	await fs.mkdir(launchAgentsDir, { recursive: true });
-	await fs.mkdir(path.dirname(agent.logPath), { recursive: true });
-	await fs.mkdir(path.dirname(agent.stdoutPath), { recursive: true });
-	await fs.mkdir(path.dirname(agent.stderrPath), { recursive: true });
-	await fs.writeFile(plistPath, agent.plist, "utf8");
-
-	let loaded = false;
-	if (options.load !== false) {
-		await execFileAsync("launchctl", ["unload", plistPath]).catch(() => {});
-		await execFileAsync("launchctl", ["load", "-w", plistPath]);
-		loaded = true;
-	}
-
-	return {
-		ok: true,
-		label: agent.label,
-		plistPath,
-		loaded,
-		programArguments: agent.programArguments,
-		logPath: agent.logPath,
-		stdoutPath: agent.stdoutPath,
-		stderrPath: agent.stderrPath,
-		intervalSeconds: agent.intervalSeconds,
-		...(agent.envFile ? { envFile: agent.envFile } : {}),
-	};
-}
-
-export function parseAccountSyncSteps(value: string | undefined) {
-	if (!value) return undefined;
-	const valid = new Set<AccountSyncStepKind>(DEFAULT_STEPS);
-	const steps = value
-		.split(",")
-		.map((step) => step.trim())
-		.filter(Boolean);
-	if (steps.length === 0) return undefined;
-	for (const step of steps) {
-		if (!valid.has(step as AccountSyncStepKind)) {
-			throw new Error(`--steps must contain ${Array.from(valid).join(", ")}`);
-		}
-	}
-	return steps as AccountSyncStepKind[];
-}