birdclaw 0.3.0 → 0.4.1

package/src/lib/sqlite.ts

@@ -0,0 +1,143 @@
+import { DatabaseSync, type StatementSync } from "node:sqlite";
+
+export type Database = NativeSqliteDatabase;
+
+type DatabaseOptions = {
+	readonly?: boolean;
+	fileMustExist?: boolean;
+};
+
+type PragmaOptions = {
+	simple?: boolean;
+};
+
+type RunResult = {
+	changes: number;
+	lastInsertRowid: number;
+};
+
+function bindArgs(parameters: unknown[]) {
+	if (parameters.length === 1 && Array.isArray(parameters[0])) {
+		return parameters[0];
+	}
+	return parameters;
+}
+
+function normalizeValue(value: unknown): unknown {
+	if (value instanceof Uint8Array && !Buffer.isBuffer(value)) {
+		return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+	}
+	return value;
+}
+
+function normalizeRow(row: unknown): unknown {
+	if (
+		!row ||
+		typeof row !== "object" ||
+		Array.isArray(row) ||
+		Buffer.isBuffer(row)
+	) {
+		return normalizeValue(row);
+	}
+	return Object.fromEntries(
+		Object.entries(row as Record<string, unknown>).map(([key, value]) => [
+			key,
+			normalizeValue(value),
+		]),
+	);
+}
+
+class NativeSqliteStatement {
+	readonly reader: boolean;
+
+	constructor(private readonly statement: StatementSync) {
+		this.reader = statement.columns().length > 0;
+	}
+
+	all(...parameters: unknown[]): unknown[] {
+		return this.statement.all(...bindArgs(parameters)).map(normalizeRow);
+	}
+
+	get(...parameters: unknown[]): unknown {
+		return normalizeRow(this.statement.get(...bindArgs(parameters)));
+	}
+
+	run(...parameters: unknown[]): RunResult {
+		const result = this.statement.run(...bindArgs(parameters));
+		return {
+			changes: Number(result.changes),
+			lastInsertRowid: Number(result.lastInsertRowid),
+		};
+	}
+
+	iterate(...parameters: unknown[]): IterableIterator<unknown> {
+		const rows = this.statement.iterate(...bindArgs(parameters));
+		return (function* () {
+			for (const row of rows) {
+				yield normalizeRow(row);
+			}
+		})();
+	}
+}
+
+export class NativeSqliteDatabase {
+	private transactionDepth = 0;
+	private readonly db: DatabaseSync;
+
+	constructor(path: string, options: DatabaseOptions = {}) {
+		this.db = new DatabaseSync(path, {
+			readOnly: options.readonly,
+		});
+	}
+
+	close(): void {
+		if (!this.db.isOpen) {
+			return;
+		}
+		this.db.close();
+	}
+
+	exec(sql: string): void {
+		this.db.exec(sql);
+	}
+
+	pragma(sql: string, options: PragmaOptions = {}): unknown {
+		const rows = this.prepare(`pragma ${sql}`).all() as Array<
+			Record<string, unknown>
+		>;
+		if (!options.simple) {
+			return rows;
+		}
+		const first = rows[0];
+		return first ? Object.values(first)[0] : undefined;
+	}
+
+	prepare(sql: string): NativeSqliteStatement {
+		return new NativeSqliteStatement(this.db.prepare(sql));
+	}
+
+	transaction<TArgs extends unknown[], TResult>(
+		fn: (...args: TArgs) => TResult,
+	): (...args: TArgs) => TResult {
+		return (...args: TArgs) => {
+			const nested = this.db.isTransaction;
+			const savepoint = `__birdclaw_tx_${++this.transactionDepth}`;
+			this.exec(nested ? `savepoint ${savepoint}` : "begin");
+			try {
+				const result = fn(...args);
+				this.exec(nested ? `release ${savepoint}` : "commit");
+				return result;
+			} catch (error) {
+				if (nested) {
+					this.exec(`rollback to ${savepoint}`);
+					this.exec(`release ${savepoint}`);
+				} else {
+					this.exec("rollback");
+				}
+				throw error;
+			}
+		};
+	}
+}
+
+export default NativeSqliteDatabase;

package/src/lib/timeline-collections-live.ts

@@ -1,4 +1,4 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listBookmarkedTweetsViaBird, listLikedTweetsViaBird } from "./bird";
 import { getNativeDb } from "./db";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
@@ -50,7 +50,7 @@ function assertXurlLimit(limit: number) {
 	}
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db
 				.prepare(
@@ -97,7 +97,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -150,7 +150,7 @@ function mergePayloads(pages: XurlMentionsResponse[]): XurlMentionsResponse {
 }
 
 function mergeTimelineCollectionIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	kind: TimelineCollectionKind,
 	payload: XurlMentionsResponse,
@@ -170,7 +170,7 @@ function mergeTimelineCollectionIntoLocalStore(
       entities_json, media_json, quoted_tweet_id
     ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '[]', ?)
     on conflict(id) do update set
-      account_id = excluded.account_id,
+      account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
       kind = case
         when tweets.kind in ('home', 'mention') then tweets.kind
@@ -185,8 +185,8 @@ function mergeTimelineCollectionIntoLocalStore(
       is_replied = max(tweets.is_replied, excluded.is_replied),
       reply_to_id = coalesce(excluded.reply_to_id, tweets.reply_to_id),
       quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id),
-      bookmarked = max(tweets.bookmarked, excluded.bookmarked),
-      liked = max(tweets.liked, excluded.liked)
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked
     `,
 	);
 	const upsertCollection = db.prepare(`

package/src/lib/timeline-live.ts

@@ -1,8 +1,9 @@
-import type Database from "better-sqlite3";
+import type { Database } from "./sqlite";
 import { listHomeTimelineViaBird } from "./bird";
 import { getNativeDb } from "./db";
 import { readSyncCache, writeSyncCache } from "./sync-cache";
 import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import { upsertTweetAccountEdge } from "./tweet-account-edges";
 import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
 
 const DEFAULT_TIMELINE_CACHE_TTL_MS = 2 * 60_000;
@@ -20,7 +21,7 @@ function assertLimit(limit: number) {
 	}
 }
 
-function resolveAccount(db: Database.Database, accountId?: string) {
+function resolveAccount(db: Database, accountId?: string) {
 	const row = accountId
 		? (db.prepare("select id from accounts where id = ?").get(accountId) as
 				| { id: string }
@@ -53,7 +54,7 @@ function getMediaCount(tweet: XurlMentionData) {
 	).length;
 }
 
-function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
+function replaceTweetFts(db: Database, tweetId: string, text: string) {
 	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
 	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
 		tweetId,
@@ -62,7 +63,7 @@ function replaceTweetFts(db: Database.Database, tweetId: string, text: string) {
 }
 
 function mergeHomeTimelineIntoLocalStore(
-	db: Database.Database,
+	db: Database,
 	accountId: string,
 	payload: XurlMentionsResponse,
 ) {
@@ -77,12 +78,9 @@ function mergeHomeTimelineIntoLocalStore(
       entities_json, media_json, quoted_tweet_id
     ) values (?, ?, ?, 'home', ?, ?, 0, null, ?, ?, 0, 0, ?, '[]', null)
     on conflict(id) do update set
-      account_id = excluded.account_id,
+      account_id = tweets.account_id,
       author_profile_id = excluded.author_profile_id,
-      kind = case
-        when tweets.kind = 'mention' then tweets.kind
-        else 'home'
-      end,
+      kind = tweets.kind,
       text = excluded.text,
       created_at = excluded.created_at,
       like_count = excluded.like_count,
@@ -95,6 +93,7 @@ function mergeHomeTimelineIntoLocalStore(
 	);
 
 	db.transaction(() => {
+		const seenAt = new Date().toISOString();
 		for (const tweet of payload.data) {
 			const author =
 				usersById.get(tweet.author_id) ??
@@ -116,6 +115,14 @@ function mergeHomeTimelineIntoLocalStore(
 				getMediaCount(tweet),
 				JSON.stringify(tweet.entities ?? {}),
 			);
+			upsertTweetAccountEdge(db, {
+				accountId,
+				tweetId: tweet.id,
+				kind: "home",
+				source: "bird",
+				seenAt,
+				rawJson: JSON.stringify(tweet),
+			});
 			replaceTweetFts(db, tweet.id, tweet.text);
 		}
 	})();

package/src/lib/tweet-account-edges.ts

@@ -0,0 +1,39 @@
+import type { Database } from "./sqlite";
+
+export type TweetAccountEdgeKind = "home" | "mention";
+
+export function upsertTweetAccountEdge(
+	db: Database,
+	{
+		accountId,
+		tweetId,
+		kind,
+		source,
+		seenAt,
+		rawJson = "{}",
+	}: {
+		accountId: string;
+		tweetId: string;
+		kind: TweetAccountEdgeKind;
+		source: string;
+		seenAt: string;
+		rawJson?: string;
+	},
+) {
+	db.prepare(`
+    insert into tweet_account_edges (
+      account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count,
+      source, raw_json, updated_at
+    ) values (?, ?, ?, ?, ?, 1, ?, ?, ?)
+    on conflict(account_id, tweet_id, kind) do update set
+      first_seen_at = min(tweet_account_edges.first_seen_at, excluded.first_seen_at),
+      last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
+      seen_count = tweet_account_edges.seen_count + 1,
+      source = coalesce(nullif(excluded.source, ''), tweet_account_edges.source),
+      raw_json = case
+        when excluded.raw_json not in ('', '{}', 'null') then excluded.raw_json
+        else tweet_account_edges.raw_json
+      end,
+      updated_at = max(tweet_account_edges.updated_at, excluded.updated_at)
+  `).run(accountId, tweetId, kind, seenAt, seenAt, source, rawJson, seenAt);
+}

package/src/lib/types.ts

@@ -23,9 +23,55 @@ export interface ProfileRecord {
 	followingCount?: number;
 	avatarHue: number;
 	avatarUrl?: string;
+	location?: string;
+	url?: string;
+	verifiedType?: string;
+	entities?: Record<string, unknown>;
+	affiliations?: ProfileAffiliation[];
+	primaryAffiliation?: ProfileAffiliation;
 	createdAt: string;
 }
 
+export interface ProfileAffiliation {
+	organizationProfileId: string;
+	organizationName?: string;
+	organizationHandle?: string;
+	badgeUrl?: string | null;
+	url?: string | null;
+	label?: string | null;
+	source: string;
+	firstSeenAt: string;
+	lastSeenAt: string;
+	isActive: boolean;
+}
+
+export interface ProfileSnapshot {
+	profileId: string;
+	snapshotHash: string;
+	observedAt: string;
+	lastSeenAt: string;
+	source: string;
+	handle: string;
+	displayName: string;
+	bio: string;
+	location: string | null;
+	url: string | null;
+	verifiedType: string | null;
+	followersCount: number;
+	followingCount: number;
+	affiliations: unknown[];
+}
+
+export interface ProfileBioEntity {
+	profileId: string;
+	kind: "handle" | "domain" | "company_phrase";
+	value: string;
+	source: string;
+	firstSeenAt: string;
+	lastSeenAt: string;
+	isActive: boolean;
+}
+
 export interface TweetMentionEntity {
 	username: string;
 	id?: string;
@@ -125,6 +171,59 @@ export interface DmMessageItem {
 	sender: ProfileRecord;
 }
 
+export interface UrlExpansionItem {
+	url: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	source: "cache" | "network";
+	title?: string;
+	description?: string | null;
+	error?: string;
+	updatedAt: string;
+}
+
+export interface LinkOccurrenceItem {
+	sourceKind: "dm" | "tweet";
+	sourceId: string;
+	sourcePosition: number;
+	shortUrl: string;
+	accountId?: string | null;
+	conversationId?: string | null;
+	direction?: string | null;
+	createdAt: string;
+}
+
+export interface LinkIndexItem {
+	shortUrl: string;
+	expandedUrl: string;
+	finalUrl: string;
+	status: "hit" | "miss" | "error";
+	expandedTweetId?: string | null;
+	expandedHandle?: string | null;
+	title?: string | null;
+	description?: string | null;
+	error?: string | null;
+	source: string;
+	updatedAt: string;
+}
+
+export interface LinkSearchItem {
+	occurrence: LinkOccurrenceItem;
+	expansion: LinkIndexItem;
+	sourceText: string;
+	sourceAuthor?: ProfileRecord | null;
+	participant?: ProfileRecord | null;
+	linkedTweet?: TimelineItem | null;
+}
+
+export interface DmSearchMatchItem {
+	message: DmMessageItem;
+	before: DmMessageItem[];
+	after: DmMessageItem[];
+	urlExpansions?: UrlExpansionItem[];
+}
+
 export interface DmConversationItem {
 	id: string;
 	accountId: string;
@@ -138,6 +237,7 @@ export interface DmConversationItem {
 	influenceScore: number;
 	influenceLabel: string;
 	participant: ProfileRecord;
+	matches?: DmSearchMatchItem[];
 }
 
 export interface TimelineQuery {
@@ -158,6 +258,7 @@ export interface TimelineQuery {
 
 export interface DmQuery {
 	account?: string;
+	conversationIds?: string[];
 	participant?: string;
 	search?: string;
 	replyFilter?: ReplyFilter;
@@ -166,6 +267,7 @@ export interface DmQuery {
 	minInfluenceScore?: number;
 	maxInfluenceScore?: number;
 	sort?: "recent" | "influence";
+	context?: number;
 	limit?: number;
 }
 
@@ -266,7 +368,13 @@ export interface XurlMentionUser {
 	name: string;
 	username: string;
 	description?: string;
+	location?: string;
+	url?: string;
+	verified?: boolean;
+	verified_type?: string;
 	profile_image_url?: string;
+	entities?: Record<string, unknown>;
+	affiliation?: Record<string, unknown>;
 	public_metrics?: XurlPublicMetrics;
 	created_at?: string;
 }

package/src/lib/url-expansion.ts

@@ -0,0 +1,155 @@
+import { readSyncCache, writeSyncCache } from "./sync-cache";
+import type { UrlExpansionItem } from "./types";
+
+const SUCCESS_CACHE_TTL_MS = 365 * 24 * 60 * 60 * 1000;
+const FAILURE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
+const URL_REGEX = /https?:\/\/[^\s<>"')\]]+/g;
+
+interface CachedUrlExpansion {
+	expandedUrl: string;
+	finalUrl: string;
+	status: UrlExpansionItem["status"];
+	title?: string;
+	description?: string | null;
+	error?: string;
+}
+
+export interface ExpandUrlsOptions {
+	refresh?: boolean;
+	successMaxAgeMs?: number;
+	failureMaxAgeMs?: number;
+	fetchImpl?: typeof fetch;
+	timeoutMs?: number;
+}
+
+function cacheKeyForUrl(url: string) {
+	return `url:expand:${url}`;
+}
+
+function isFresh(updatedAt: string, maxAgeMs: number) {
+	return Date.now() - new Date(updatedAt).getTime() <= maxAgeMs;
+}
+
+function trimTrailingPunctuation(url: string) {
+	return url.replace(/[.,;:!?]+$/g, "");
+}
+
+export function extractUrls(text: string) {
+	return Array.from(
+		new Set(
+			Array.from(text.matchAll(URL_REGEX), (match) =>
+				trimTrailingPunctuation(match[0]),
+			),
+		),
+	).filter((url) => url.length > 0);
+}
+
+function toExpansionItem(
+	url: string,
+	value: CachedUrlExpansion,
+	source: UrlExpansionItem["source"],
+	updatedAt: string,
+): UrlExpansionItem {
+	return {
+		url,
+		expandedUrl: value.expandedUrl,
+		finalUrl: value.finalUrl,
+		status: value.status,
+		source,
+		...(value.title ? { title: value.title } : {}),
+		...(value.description !== undefined
+			? { description: value.description }
+			: {}),
+		...(value.error ? { error: value.error } : {}),
+		updatedAt,
+	};
+}
+
+async function fetchExpansion(
+	url: string,
+	fetchImpl: typeof fetch,
+	timeoutMs: number,
+): Promise<CachedUrlExpansion> {
+	const requestInit = {
+		redirect: "follow",
+		headers: { "user-agent": "birdclaw/0.3 url-expander" },
+		signal: AbortSignal.timeout(timeoutMs),
+	} satisfies RequestInit;
+
+	try {
+		let response = await fetchImpl(url, {
+			...requestInit,
+			method: "HEAD",
+		});
+
+		if (!response.url || response.url === url || response.status >= 400) {
+			response = await fetchImpl(url, {
+				...requestInit,
+				method: "GET",
+			});
+		}
+
+		const finalUrl = response.url || url;
+		return {
+			expandedUrl: finalUrl,
+			finalUrl,
+			status: response.ok || finalUrl !== url ? "hit" : "miss",
+			...(response.ok ? {} : { error: `HTTP ${response.status}` }),
+		};
+	} catch (error) {
+		return {
+			expandedUrl: url,
+			finalUrl: url,
+			status: "error",
+			error: error instanceof Error ? error.message : String(error),
+		};
+	}
+}
+
+export async function expandUrls(
+	urls: string[],
+	options: ExpandUrlsOptions = {},
+): Promise<UrlExpansionItem[]> {
+	const uniqueUrls = Array.from(new Set(urls));
+	const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+	const timeoutMs = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
+	const results: UrlExpansionItem[] = [];
+
+	for (const url of uniqueUrls) {
+		const cached = readSyncCache<CachedUrlExpansion>(cacheKeyForUrl(url));
+		if (cached && !options.refresh) {
+			const maxAge =
+				cached.value.status === "hit"
+					? (options.successMaxAgeMs ?? SUCCESS_CACHE_TTL_MS)
+					: (options.failureMaxAgeMs ?? FAILURE_CACHE_TTL_MS);
+			if (isFresh(cached.updatedAt, maxAge)) {
+				results.push(
+					toExpansionItem(url, cached.value, "cache", cached.updatedAt),
+				);
+				continue;
+			}
+		}
+
+		const value = await fetchExpansion(url, fetchImpl, timeoutMs);
+		const updatedAt = writeSyncCache(cacheKeyForUrl(url), value);
+		results.push(toExpansionItem(url, value, "network", updatedAt));
+	}
+
+	return results;
+}
+
+export async function expandUrlsFromTexts(
+	texts: string[],
+	options: ExpandUrlsOptions = {},
+) {
+	return expandUrls(
+		texts.flatMap((text) => extractUrls(text)),
+		options,
+	);
+}
+
+export const __test__ = {
+	cacheKeyForUrl,
+	trimTrailingPunctuation,
+};