birdclaw 0.8.1 → 0.8.3

package/src/lib/tweet-render.ts

@@ -1,4 +1,5 @@
 import type {
+	TweetArticle,
 	TweetEntities,
 	TweetHashtagEntity,
 	TweetMentionEntity,
@@ -57,6 +58,34 @@ export function displayUrlForLink(url: string) {
 	}
 }
 
+function comparableUrl(value: string) {
+	try {
+		const parsed = new URL(value);
+		return `${parsed.protocol}//${parsed.hostname.replace(/^www\./, "")}${parsed.pathname}`;
+	} catch {
+		return value.split("?")[0] ?? value;
+	}
+}
+
+export function isTweetArticleUrlEntity(
+	entry: TweetUrlEntity,
+	article: TweetArticle,
+) {
+	if (comparableUrl(entry.expandedUrl) === comparableUrl(article.url)) {
+		return true;
+	}
+	try {
+		const parsed = new URL(entry.expandedUrl);
+		const host = parsed.hostname.replace(/^www\./, "");
+		return (
+			(host === "x.com" || host === "twitter.com") &&
+			parsed.pathname.startsWith("/i/article/")
+		);
+	} catch {
+		return false;
+	}
+}
+
 function asRecord(value: unknown) {
 	return value && typeof value === "object"
 		? (value as Record<string, unknown>)
@@ -68,6 +97,9 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 	const rawMentions = Array.isArray(entities.mentions) ? entities.mentions : [];
 	const rawUrls = Array.isArray(entities.urls) ? entities.urls : [];
 	const rawHashtags = Array.isArray(entities.hashtags) ? entities.hashtags : [];
+	const rawArticle = asRecord(entities.article);
+	const articleTitle = String(rawArticle.title ?? "").trim();
+	const articleUrl = String(rawArticle.url ?? "").trim();
 
 	return {
 		...(rawMentions.length
@@ -102,6 +134,23 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 							),
 							start: Number(value.start ?? 0),
 							end: Number(value.end ?? 0),
+							...(typeof value.title === "string"
+								? { title: value.title }
+								: {}),
+							...(typeof value.description === "string" ||
+							value.description === null
+								? { description: value.description }
+								: {}),
+							...(typeof value.imageUrl === "string"
+								? { imageUrl: value.imageUrl }
+								: typeof value.image_url === "string"
+									? { imageUrl: value.image_url }
+									: {}),
+							...(typeof value.siteName === "string"
+								? { siteName: value.siteName }
+								: typeof value.site_name === "string"
+									? { siteName: value.site_name }
+									: {}),
 						};
 					}),
 				}
@@ -118,6 +167,35 @@ export function tweetEntitiesFromXurl(raw: unknown): TweetEntities {
 					}),
 				}
 			: {}),
+		...(articleTitle && articleUrl
+			? {
+					article: {
+						title: articleTitle,
+						url: articleUrl,
+						...(typeof (rawArticle.previewText ?? rawArticle.preview_text) ===
+							"string" &&
+						String(rawArticle.previewText ?? rawArticle.preview_text).trim()
+							? {
+									previewText: String(
+										rawArticle.previewText ?? rawArticle.preview_text,
+									).trim(),
+								}
+							: {}),
+						...(typeof (
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url
+						) === "string" &&
+						String(
+							rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+						).trim()
+							? {
+									coverImageUrl: String(
+										rawArticle.coverImageUrl ?? rawArticle.cover_image_url,
+									).trim(),
+								}
+							: {}),
+					},
+				}
+			: {}),
 	};
 }
 

package/src/lib/tweet-repository.ts

@@ -0,0 +1,156 @@
+import type { Database } from "./sqlite";
+import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
+import { tweetEntitiesFromXurl } from "./tweet-render";
+import type { XurlMentionData, XurlMentionsResponse } from "./types";
+import {
+	type TweetAccountEdgeKind,
+	upsertTweetAccountEdge,
+} from "./tweet-account-edges";
+import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
+
+export type CanonicalTweetKind =
+	| "authored"
+	| "bookmark"
+	| "home"
+	| "like"
+	| "mention"
+	| "search"
+	| "thread"
+	| "thread_context";
+
+export interface IngestTweetPayloadOptions {
+	accountId: string;
+	payload: XurlMentionsResponse;
+	kind: CanonicalTweetKind;
+	source: string;
+	edgeKind?: TweetAccountEdgeKind;
+	collectionKind?: "likes" | "bookmarks";
+	markRepliesAsReplied?: boolean;
+	replaceSecondaryKind?: boolean;
+}
+
+function getReferencedTweetId(tweet: XurlMentionData, type: string) {
+	return (
+		tweet.referenced_tweets?.find((item) => item.type === type)?.id ?? null
+	);
+}
+
+export function replaceTweetFts(db: Database, tweetId: string, text: string) {
+	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
+	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
+		tweetId,
+		text,
+	);
+}
+
+export function ingestTweetPayload(
+	db: Database,
+	{
+		accountId,
+		payload,
+		kind,
+		source,
+		edgeKind,
+		collectionKind,
+		markRepliesAsReplied = false,
+		replaceSecondaryKind = false,
+	}: IngestTweetPayloadOptions,
+) {
+	const usersById = new Map(
+		(payload.includes?.users ?? []).map((user) => [user.id, user]),
+	);
+	const upsertTweet = db.prepare(`
+    insert into tweets (
+      id, account_id, author_profile_id, kind, text, created_at,
+      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
+      entities_json, media_json, quoted_tweet_id
+    ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    on conflict(id) do update set
+      account_id = tweets.account_id,
+      author_profile_id = excluded.author_profile_id,
+      kind = case
+        when ? = 1
+          and tweets.kind not in ('authored', 'home', 'mention')
+          then excluded.kind
+        else tweets.kind
+      end,
+      text = excluded.text,
+      created_at = excluded.created_at,
+      is_replied = max(tweets.is_replied, excluded.is_replied),
+      reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
+      like_count = excluded.like_count,
+      media_count = max(tweets.media_count, excluded.media_count),
+      entities_json = excluded.entities_json,
+      media_json = case
+        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
+        else tweets.media_json
+      end,
+      bookmarked = tweets.bookmarked,
+      liked = tweets.liked,
+      quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id)
+  `);
+	const upsertCollection = collectionKind
+		? db.prepare(`
+        insert into tweet_collections (
+          account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
+        ) values (?, ?, ?, null, ?, ?, ?)
+        on conflict(account_id, tweet_id, kind) do update set
+          source = excluded.source,
+          raw_json = excluded.raw_json,
+          updated_at = excluded.updated_at
+      `)
+		: undefined;
+	const tweetIds: string[] = [];
+
+	db.transaction(() => {
+		const observedAt = new Date().toISOString();
+		for (const tweet of payload.data) {
+			const author = usersById.get(tweet.author_id);
+			const profile = author
+				? upsertProfileFromXUser(db, author)
+				: ensureStubProfileForXUser(db, tweet.author_id);
+			const replyToId = getReferencedTweetId(tweet, "replied_to");
+			const quotedTweetId = getReferencedTweetId(tweet, "quoted");
+			upsertTweet.run(
+				tweet.id,
+				accountId,
+				profile.profile.id,
+				kind,
+				tweet.text,
+				tweet.created_at,
+				markRepliesAsReplied && replyToId ? 1 : 0,
+				replyToId,
+				Number(tweet.public_metrics?.like_count ?? 0),
+				countTweetMedia(tweet),
+				collectionKind === "bookmarks" ? 1 : 0,
+				collectionKind === "likes" ? 1 : 0,
+				JSON.stringify(tweetEntitiesFromXurl(tweet.entities)),
+				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
+				quotedTweetId,
+				replaceSecondaryKind ? 1 : 0,
+			);
+			if (edgeKind) {
+				upsertTweetAccountEdge(db, {
+					accountId,
+					tweetId: tweet.id,
+					kind: edgeKind,
+					source,
+					seenAt: observedAt,
+					rawJson: JSON.stringify(tweet),
+				});
+			}
+			upsertCollection?.run(
+				accountId,
+				tweet.id,
+				collectionKind,
+				source,
+				JSON.stringify(tweet),
+				observedAt,
+			);
+			replaceTweetFts(db, tweet.id, tweet.text);
+			tweetIds.push(tweet.id);
+		}
+	})();
+
+	return tweetIds;
+}

package/src/lib/tweet-search-live.ts

@@ -1,14 +1,15 @@
 import { Effect } from "effect";
-import { searchTweetsViaBirdEffect } from "./bird";
 import type { Database } from "./sqlite";
 import { getNativeDb } from "./db";
 import { runEffectPromise } from "./effect-runtime";
-import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
-import { readSyncCache, writeSyncCache } from "./sync-cache";
+import { liveTransportGateway } from "./live-transport-gateway";
+import {
+	normalizeCacheTtlMs,
+	resolveLiveSyncAccount,
+	runCachedLiveSyncEffect,
+} from "./live-sync-engine";
 import type { XurlMentionsResponse, XurlTweetsResponse } from "./types";
-import { upsertTweetAccountEdge } from "./tweet-account-edges";
-import { ensureStubProfileForXUser, upsertProfileFromXUser } from "./x-profile";
-import { searchRecentTweetsEffect } from "./xurl";
+import { ingestTweetPayload } from "./tweet-repository";
 
 export type TweetSearchMode = "auto" | "bird" | "xurl" | "local";
 
@@ -84,130 +85,19 @@ function normalizeTime(value: string | undefined, optionName: string) {
 	return date.toISOString();
 }
 
-function normalizeCacheTtlMs(value: number | undefined) {
-	if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
-		return DEFAULT_CACHE_TTL_MS;
-	}
-	return Math.floor(value);
-}
-
-function resolveAccount(db: Database, accountId?: string) {
-	const row = accountId
-		? (db
-				.prepare("select id, handle from accounts where id = ?")
-				.get(accountId) as { id: string; handle: string } | undefined)
-		: (db
-				.prepare(
-					`
-          select id, handle
-          from accounts
-          order by is_default desc, created_at asc
-          limit 1
-          `,
-				)
-				.get() as { id: string; handle: string } | undefined);
-
-	if (!row) {
-		throw new Error(`Unknown account: ${accountId ?? "default"}`);
-	}
-
-	return {
-		accountId: row.id,
-		username: row.handle.replace(/^@/, ""),
-	};
-}
-
-function replaceTweetFts(db: Database, tweetId: string, text: string) {
-	db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId);
-	db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run(
-		tweetId,
-		text,
-	);
-}
-
 function mergeTweetSearchIntoLocalStore(
 	db: Database,
 	accountId: string,
 	payload: XurlMentionsResponse,
 	source: "bird" | "xurl" | "cache",
 ) {
-	const usersById = new Map(
-		(payload.includes?.users ?? []).map((user) => [user.id, user]),
-	);
-	const upsertTweet = db.prepare(
-		`
-    insert into tweets (
-      id, account_id, author_profile_id, kind, text, created_at,
-      is_replied, reply_to_id, like_count, media_count, bookmarked, liked,
-      entities_json, media_json, quoted_tweet_id
-    ) values (?, ?, ?, 'search', ?, ?, 0, ?, ?, ?, 0, 0, ?, ?, ?)
-    on conflict(id) do update set
-      account_id = tweets.account_id,
-      author_profile_id = excluded.author_profile_id,
-      kind = tweets.kind,
-      text = excluded.text,
-      created_at = excluded.created_at,
-      reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id),
-      like_count = excluded.like_count,
-      media_count = max(tweets.media_count, excluded.media_count),
-      entities_json = excluded.entities_json,
-      media_json = case
-        when excluded.media_json not in ('', '[]', 'null') then excluded.media_json
-        else tweets.media_json
-      end,
-      quoted_tweet_id = coalesce(tweets.quoted_tweet_id, excluded.quoted_tweet_id),
-      bookmarked = tweets.bookmarked,
-      liked = tweets.liked
-    `,
-	);
-	const tweetIds: string[] = [];
-
-	db.transaction(() => {
-		const seenAt = new Date().toISOString();
-		for (const tweet of payload.data) {
-			const author =
-				usersById.get(tweet.author_id) ??
-				({
-					id: tweet.author_id,
-					username: `user_${tweet.author_id}`,
-					name: `user_${tweet.author_id}`,
-				} as const);
-			const profile = usersById.has(tweet.author_id)
-				? upsertProfileFromXUser(db, author)
-				: ensureStubProfileForXUser(db, tweet.author_id);
-			const replyToId =
-				tweet.referenced_tweets?.find((item) => item.type === "replied_to")
-					?.id ?? null;
-			const quotedTweetId =
-				tweet.referenced_tweets?.find((item) => item.type === "quoted")?.id ??
-				null;
-			upsertTweet.run(
-				tweet.id,
-				accountId,
-				profile.profile.id,
-				tweet.text,
-				tweet.created_at,
-				replyToId,
-				Number(tweet.public_metrics?.like_count ?? 0),
-				countTweetMedia(tweet),
-				JSON.stringify(tweet.entities ?? {}),
-				buildMediaJsonFromIncludes(tweet, payload.includes?.media),
-				quotedTweetId,
-			);
-			upsertTweetAccountEdge(db, {
-				accountId,
-				tweetId: tweet.id,
-				kind: "search",
-				source,
-				seenAt,
-				rawJson: JSON.stringify(tweet),
-			});
-			replaceTweetFts(db, tweet.id, tweet.text);
-			tweetIds.push(tweet.id);
-		}
-	})();
-
-	return tweetIds;
+	return ingestTweetPayload(db, {
+		accountId,
+		payload,
+		kind: "search",
+		edgeKind: "search",
+		source,
+	});
 }
 
 function toMentionsResponse(payload: XurlTweetsResponse): XurlMentionsResponse {
@@ -301,7 +191,7 @@ function fetchBirdSearchEffect({
 	limit: number;
 	maxPages: number;
 }) {
-	return searchTweetsViaBirdEffect(query, {
+	return liveTransportGateway.bird.searchTweets(query, {
 		maxResults: Math.min(limit, XURL_PAGE_SIZE),
 		all: maxPages > 1 || limit > XURL_PAGE_SIZE,
 		maxPages,
@@ -331,13 +221,16 @@ function fetchXurlSearchEffect({
 				limit -
 				responses.reduce((total, response) => total + response.data.length, 0);
 			if (remaining <= 0) break;
-			const response = yield* searchRecentTweetsEffect(query, {
-				maxResults: Math.max(10, Math.min(XURL_PAGE_SIZE, remaining)),
-				paginationToken: nextToken,
-				startTime: since,
-				endTime: until,
-				timeoutMs,
-			});
+			const response = yield* liveTransportGateway.xurl.searchRecentTweets(
+				query,
+				{
+					maxResults: Math.max(10, Math.min(XURL_PAGE_SIZE, remaining)),
+					paginationToken: nextToken,
+					startTime: since,
+					endTime: until,
+					timeoutMs,
+				},
+			);
 			responses.push(toMentionsResponse(response));
 			nextToken =
 				typeof response.meta?.next_token === "string"
@@ -367,42 +260,44 @@ function runModeEffect(
 	return Effect.gen(function* () {
 		const db = getNativeDb();
 		const key = cacheKey({ ...options, mode });
-		const cached = yield* trySync(() =>
-			readSyncCache<XurlMentionsResponse>(key, db),
-		);
-		const ageMs = cached
-			? Date.now() - new Date(cached.updatedAt).getTime()
-			: Number.POSITIVE_INFINITY;
-		const payload =
-			!options.refresh && cached && ageMs <= options.cacheTtlMs
-				? cached.value
-				: yield* (
-						mode === "bird"
-							? fetchBirdSearchEffect(options).pipe(Effect.mapError(toError))
-							: fetchXurlSearchEffect(options)
-					).pipe(
+		const fetch =
+			mode === "bird"
+				? fetchBirdSearchEffect(options).pipe(Effect.mapError(toError))
+				: fetchXurlSearchEffect(options);
+		const syncResult = yield* runCachedLiveSyncEffect({
+			db,
+			cacheKey: key,
+			refresh: options.refresh,
+			cacheTtlMs: options.cacheTtlMs,
+			transports: [
+				{
+					source: mode,
+					fetch: fetch.pipe(
 						Effect.map((response) => limitResponse(response, options.limit)),
-					);
-		if (!cached || options.refresh || ageMs > options.cacheTtlMs) {
-			yield* trySync(() => writeSyncCache(key, payload, db));
-		}
-		const tweetIds = yield* trySync(() =>
-			mergeTweetSearchIntoLocalStore(
-				db,
-				options.accountId,
-				payload,
-				!options.refresh && cached && ageMs <= options.cacheTtlMs
-					? "cache"
-					: mode,
-			),
-		);
+					),
+				},
+			],
+			persistLive: (writeDb, payload, source) =>
+				mergeTweetSearchIntoLocalStore(
+					writeDb,
+					options.accountId,
+					payload,
+					source,
+				),
+			persistCached: (writeDb, payload) =>
+				mergeTweetSearchIntoLocalStore(
+					writeDb,
+					options.accountId,
+					payload,
+					"cache",
+				),
+		});
+		const { payload, source } = syncResult;
+		const tweetIds = syncResult.persisted ?? [];
 
 		return {
 			ok: true,
-			source:
-				!options.refresh && cached && ageMs <= options.cacheTtlMs
-					? "cache"
-					: mode,
+			source,
 			accountId: options.accountId,
 			query: options.query,
 			count: tweetIds.length,
@@ -478,9 +373,11 @@ export function syncTweetSearchEffect({
 		const normalizedUntil = yield* trySync(() =>
 			normalizeTime(until, "--until"),
 		);
-		const ttlMs = normalizeCacheTtlMs(cacheTtlMs);
+		const ttlMs = normalizeCacheTtlMs(cacheTtlMs, DEFAULT_CACHE_TTL_MS);
 		const db = getNativeDb();
-		const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
+		const resolvedAccount = yield* trySync(() =>
+			resolveLiveSyncAccount(db, account),
+		);
 		const accountId = resolvedAccount.accountId;
 		if (mode === "local") {
 			return {

package/src/lib/types.ts

@@ -101,10 +101,18 @@ export interface TweetHashtagEntity {
 	end: number;
 }
 
+export interface TweetArticle {
+	title: string;
+	previewText?: string;
+	url: string;
+	coverImageUrl?: string;
+}
+
 export interface TweetEntities {
 	mentions?: TweetMentionEntity[];
 	urls?: TweetUrlEntity[];
 	hashtags?: TweetHashtagEntity[];
+	article?: TweetArticle;
 }
 
 export interface TweetMediaItem {

package/src/lib/ui.ts

@@ -311,7 +311,7 @@ export const profilePreviewTriggerClass =
 	"profile-preview-trigger inline-flex text-inherit";
 
 export const profilePreviewCardClass =
-	"pointer-events-none absolute left-0 z-30 grid w-[280px] gap-2 rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 opacity-0 shadow-[0_8px_28px_var(--shadow-strong)] transition-all duration-150 group-hover:pointer-events-auto group-hover:opacity-100 group-focus-within:pointer-events-auto group-focus-within:opacity-100";
+	"fixed z-40 w-[280px] overflow-y-auto rounded-2xl border border-[var(--line)] bg-[var(--bg-elevated)] p-3 shadow-[0_8px_28px_var(--shadow-strong)]";
 
 export const profilePreviewHeaderClass = "flex items-center gap-3";